feat: 新增本地UI模型AI拾取与识别实验台

src/main/capabilities/cv-vision.ts

@@ -1,5 +1,5 @@
 import { register } from "../engine/registry";
-import { RuntimeError } from "../engine/types";
+import { RuntimeError, type CapabilityHandler } from "../engine/types";
 import { powershell } from "./desktop";
 import fs from "node:fs";
 import os from "node:os";
@@ -57,4 +57,95 @@ export async function locateTextOnScreen(target: string, nodeId: string): Promis
   const cx = (Math.min(...xs) + Math.max(...xs)) / 2;
   const cy = (Math.min(...ys) + Math.max(...ys)) / 2;
   return { target: keyword, text: hit.text, cx, cy };
-}
+}
+function screenUser32(): string {
+  return [
+    "Add-Type -MemberDefinition '[DllImport(\"user32.dll\")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value); [DllImport(\"user32.dll\")] public static extern bool SetCursorPos(int X, int Y); [DllImport(\"user32.dll\")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra);' -Name RpaCvInput -Namespace RpaCv | Out-Null;",
+    "try { [RpaCv.RpaCvInput]::SetProcessDpiAwarenessContext([IntPtr](-4)) | Out-Null } catch {}"
+  ].join("\n");
+}
+
+function coordinateScript(x: number, y: number, action: "hover" | "click" | "doubleClick" | "paste", value = ""): string {
+  const lines = [
+    screenUser32(),
+    "if (-not [RpaCv.RpaCvInput]::SetCursorPos(" + x + ", " + y + ")) { throw 'cursor_move_failed' }",
+    "Start-Sleep -Milliseconds 100"
+  ];
+  if (action !== "hover") {
+    lines.push(
+      "[RpaCv.RpaCvInput]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)",
+      "Start-Sleep -Milliseconds 40",
+      "[RpaCv.RpaCvInput]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)"
+    );
+  }
+  if (action === "doubleClick") {
+    lines.push(
+      "Start-Sleep -Milliseconds 70",
+      "[RpaCv.RpaCvInput]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)",
+      "Start-Sleep -Milliseconds 40",
+      "[RpaCv.RpaCvInput]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)"
+    );
+  }
+  if (action === "paste") {
+    lines.push(
+      "Add-Type -AssemblyName System.Windows.Forms",
+      "Start-Sleep -Milliseconds 220",
+      "$saved = $null",
+      "try { $saved = [System.Windows.Forms.Clipboard]::GetText() } catch {}",
+      "try { [System.Windows.Forms.Clipboard]::SetText(" + JSON.stringify(value) + ") } catch { throw 'clipboard_failed' }",
+      "Start-Sleep -Milliseconds 80",
+      "[System.Windows.Forms.SendKeys]::SendWait('^a')",
+      "Start-Sleep -Milliseconds 60",
+      "[System.Windows.Forms.SendKeys]::SendWait('^v')",
+      "Start-Sleep -Milliseconds 280",
+      "try { if ($saved) { [System.Windows.Forms.Clipboard]::SetText($saved) } else { [System.Windows.Forms.Clipboard]::Clear() } } catch {}"
+    );
+  }
+  lines.push("Write-Output (@{ ok = $true; x = " + x + "; y = " + y + " } | ConvertTo-Json -Compress)");
+  return lines.join("\n");
+}
+
+async function locateFor(args: Record<string, unknown>, node: { id: string; componentId: string }) {
+  const target = cvTarget(args, node.id);
+  const located = await locateTextOnScreen(target, node.id);
+  const offsetX = Number(args.offsetX ?? args.offset_x ?? 0) || 0;
+  const offsetY = Number(args.offsetY ?? args.offset_y ?? 0) || 0;
+  return { located, x: Math.round(located.cx + offsetX), y: Math.round(located.cy + offsetY) };
+}
+
+const getTextGround: CapabilityHandler = async (ctx, node, args) => {
+  const target = cvTarget(args, node.id);
+  const result = await locateTextOnScreen(target, node.id);
+  ctx.log("CV_GetText found: " + result.text, "info", node.id);
+  return result.text;
+};
+
+const setTextGround: CapabilityHandler = async (ctx, node, args) => {
+  const value = String(args.value ?? args.text ?? "");
+  if (!value) throw new RuntimeError("validation_error", "CV 文本输入内容为空", { nodeId: node.id, componentId: node.componentId });
+  const { x, y } = await locateFor(args, node);
+  await powershell(coordinateScript(x, y, "paste", value), 30000);
+  ctx.log("CV_SetText input at " + x + "," + y, "info", node.id);
+  return true;
+};
+
+const mouseHoverGround: CapabilityHandler = async (ctx, node, args) => {
+  const { x, y } = await locateFor(args, node);
+  await powershell(coordinateScript(x, y, "hover"), 15000);
+  ctx.log("CV_MouseHover hover at " + x + "," + y, "info", node.id);
+  return true;
+};
+
+const mouseClickGround: CapabilityHandler = async (ctx, node, args) => {
+  const { x, y } = await locateFor(args, node);
+  const clickway = String(args.clickway ?? "0");
+  const action = clickway === "1" || /double/i.test(clickway) ? "doubleClick" as const : "click" as const;
+  await powershell(coordinateScript(x, y, action), 15000);
+  ctx.log("CV_MouseClick " + action + " at " + x + "," + y, "info", node.id);
+  return true;
+};
+
+register("CV_GetText", getTextGround);
+register("CV_SetText", setTextGround);
+register("CV_MouseHover", mouseHoverGround);
+register("CV_MouseClick", mouseClickGround);