PRA-only-demo/src/main/capabilities/cv-vision.ts

import { register } from "../engine/registry";
import { RuntimeError, type CapabilityHandler } from "../engine/types";
import { powershell } from "./desktop";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";

interface CvTarget { text: string; cx: number; cy: number }
let paddleOcr: { detect: (image: string) => Promise<{ texts: Array<{ text: string; box?: number[][]; mean?: number }> }> } | null = null;

async function getOcr() {
  if (paddleOcr) return paddleOcr;
  const mod = await import("@gutenye/ocr-node");
  paddleOcr = await mod.default.create();
  return paddleOcr as { detect: (image: string) => Promise<{ texts: Array<{ text: string; box?: number[][]; mean?: number }> }> };
}

function cvTarget(args: Record<string, unknown>, nodeId: string): string {
  for (const key of ["cvSelector", "target", "text"]) {
    const v = String(args[key] ?? "").trim();
    if (v) return v.replace(/^text:/, "").replace(/^图片:/, "");
  }
  throw new RuntimeError("validation_error", "CV 目标为空", { nodeId, componentId: nodeId, recommendation: "填写要在屏幕上查找的目标文本" });
}

async function grabScreen(imagePath: string): Promise<void> {
  fs.mkdirSync(path.dirname(imagePath), { recursive: true });
  const psScript = [
    // 与点击脚本一致: 先声明 DPI 感知, 否则缩放非 100% 时 Bounds 是逻辑像素, 截屏只覆盖屏幕左上角
    "Add-Type -MemberDefinition '[DllImport(\"user32.dll\")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value);' -Name RpaCvDpi -Namespace RpaCv | Out-Null;",
    "try { [RpaCv.RpaCvDpi]::SetProcessDpiAwarenessContext([IntPtr](-4)) | Out-Null } catch {}",
    "Add-Type -AssemblyName System.Drawing | Out-Null;",
    "Add-Type -AssemblyName System.Windows.Forms | Out-Null;",
    "$b = [System.Windows.Forms.SystemInformation]::VirtualScreen;",
    "$bmp = New-Object System.Drawing.Bitmap($b.Width, $b.Height);",
    "$g = [System.Drawing.Graphics]::FromImage($bmp);",
    "$g.CopyFromScreen($b.X, $b.Y, 0, 0, $b.Size);",
    "$g.Dispose();",
    "$bmp.Save(" + JSON.stringify(imagePath) + ");",
    "$bmp.Dispose();"
  ].join(" ");
  await powershell(psScript, 30000);
}

interface OcrLine { text: string; box?: number[][]; mean?: number }

export async function locateTextOnScreen(target: string, nodeId: string): Promise<{ target: string; text: string; cx: number; cy: number }> {
  const shotPath = path.join(os.tmpdir(), "rpa-cv-screen-" + Date.now() + ".png");
  await grabScreen(shotPath);
  const ocr = await getOcr();
  const result = await ocr.detect(shotPath);
  const texts = result.texts ?? [];
  const keyword = target.trim();
  const hit = texts.find(t => t.text && t.text.includes(keyword));
  if (!hit || !hit.box || hit.box.length < 4) {
    // 找不到目标时保留截屏, 便于排查(截屏范围/遮挡/清晰度)
    throw new RuntimeError("cv_not_found", "屏幕上未找到目标文本：" + keyword, {
      nodeId,
      componentId: nodeId,
      recommendation: "确认目标文本在屏幕上可见且清晰；本次截屏已保留: " + shotPath
    });
  }
  try { fs.unlinkSync(shotPath); } catch {}
  const xs = hit.box.map(p => p[0]);
  const ys = hit.box.map(p => p[1]);
  const cx = (Math.min(...xs) + Math.max(...xs)) / 2;
  const cy = (Math.min(...ys) + Math.max(...ys)) / 2;
  return { target: keyword, text: hit.text, cx, cy };
}
function screenUser32(): string {
  return [
    "Add-Type -MemberDefinition '[DllImport(\"user32.dll\")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value); [DllImport(\"user32.dll\")] public static extern bool SetCursorPos(int X, int Y); [DllImport(\"user32.dll\")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra);' -Name RpaCvInput -Namespace RpaCv | Out-Null;",
    "try { [RpaCv.RpaCvInput]::SetProcessDpiAwarenessContext([IntPtr](-4)) | Out-Null } catch {}"
  ].join("\n");
}

function coordinateScript(x: number, y: number, action: "hover" | "click" | "doubleClick" | "paste", value = ""): string {
  const lines = [
    screenUser32(),
    "if (-not [RpaCv.RpaCvInput]::SetCursorPos(" + x + ", " + y + ")) { throw 'cursor_move_failed' }",
    "Start-Sleep -Milliseconds 100"
  ];
  if (action !== "hover") {
    lines.push(
      "[RpaCv.RpaCvInput]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)",
      "Start-Sleep -Milliseconds 40",
      "[RpaCv.RpaCvInput]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)"
    );
  }
  if (action === "doubleClick") {
    lines.push(
      "Start-Sleep -Milliseconds 70",
      "[RpaCv.RpaCvInput]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)",
      "Start-Sleep -Milliseconds 40",
      "[RpaCv.RpaCvInput]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)"
    );
  }
  if (action === "paste") {
    lines.push(
      "Add-Type -AssemblyName System.Windows.Forms",
      "Start-Sleep -Milliseconds 220",
      "$saved = $null",
      "try { $saved = [System.Windows.Forms.Clipboard]::GetText() } catch {}",
      "try { [System.Windows.Forms.Clipboard]::SetText(" + JSON.stringify(value) + ") } catch { throw 'clipboard_failed' }",
      "Start-Sleep -Milliseconds 80",
      "[System.Windows.Forms.SendKeys]::SendWait('^a')",
      "Start-Sleep -Milliseconds 60",
      "[System.Windows.Forms.SendKeys]::SendWait('^v')",
      "Start-Sleep -Milliseconds 280",
      "try { if ($saved) { [System.Windows.Forms.Clipboard]::SetText($saved) } else { [System.Windows.Forms.Clipboard]::Clear() } } catch {}"
    );
  }
  lines.push("Write-Output (@{ ok = $true; x = " + x + "; y = " + y + " } | ConvertTo-Json -Compress)");
  return lines.join("\n");
}

async function locateFor(args: Record<string, unknown>, node: { id: string; componentId: string }) {
  const target = cvTarget(args, node.id);
  const located = await locateTextOnScreen(target, node.id);
  const offsetX = Number(args.offsetX ?? args.offset_x ?? 0) || 0;
  const offsetY = Number(args.offsetY ?? args.offset_y ?? 0) || 0;
  return { located, x: Math.round(located.cx + offsetX), y: Math.round(located.cy + offsetY) };
}

const getTextGround: CapabilityHandler = async (ctx, node, args) => {
  const target = cvTarget(args, node.id);
  const result = await locateTextOnScreen(target, node.id);
  ctx.log("CV_GetText found: " + result.text, "info", node.id);
  return result.text;
};

const setTextGround: CapabilityHandler = async (ctx, node, args) => {
  const value = String(args.value ?? args.text ?? "");
  if (!value) throw new RuntimeError("validation_error", "CV 文本输入内容为空", { nodeId: node.id, componentId: node.componentId });
  const { x, y } = await locateFor(args, node);
  await powershell(coordinateScript(x, y, "paste", value), 30000);
  ctx.log("CV_SetText input at " + x + "," + y, "info", node.id);
  return true;
};

const mouseHoverGround: CapabilityHandler = async (ctx, node, args) => {
  const { x, y } = await locateFor(args, node);
  await powershell(coordinateScript(x, y, "hover"), 15000);
  ctx.log("CV_MouseHover hover at " + x + "," + y, "info", node.id);
  return true;
};

const mouseClickGround: CapabilityHandler = async (ctx, node, args) => {
  const { x, y } = await locateFor(args, node);
  const clickway = String(args.clickway ?? "0");
  const action = clickway === "1" || /double/i.test(clickway) ? "doubleClick" as const : "click" as const;
  await powershell(coordinateScript(x, y, action), 15000);
  ctx.log("CV_MouseClick " + action + " at " + x + "," + y, "info", node.id);
  return true;
};

register("CV_GetText", getTextGround);
register("CV_SetText", setTextGround);
register("CV_MouseHover", mouseHoverGround);
register("CV_MouseClick", mouseClickGround);