From adf5f3f7cf3baec1b1119ade80524b0df50cd91b Mon Sep 17 00:00:00 2001
From: yu <191972707@qq.com>
Date: Fri, 30 Oct 2026 15:15:57 +0800
Subject: [PATCH] =?UTF-8?q?feat:=20CV=20=E8=A7=86=E8=A7=89=E5=9B=9B?=
 =?UTF-8?q?=E8=83=BD=E5=8A=9B=EF=BC=88=E6=88=AA=E5=9B=BEOCR=E5=AE=9A?=
 =?UTF-8?q?=E4=BD=8D=E7=82=B9=E5=87=BB/=E6=82=AC=E5=81=9C/=E8=AF=BB?=
 =?UTF-8?q?=E6=96=87=E6=9C=AC/=E8=BE=93=E5=85=A5=EF=BC=89=EF=BC=8C?=
 =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20path=20import=20=E4=B8=8E=20getOcr=20?=
 =?UTF-8?q?=E7=B1=BB=E5=9E=8B?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/main/capabilities/cv-vision.ts | 60 ++++++++++++++++++++++++++++++
 1 file changed, 60 insertions(+)
 create mode 100644 src/main/capabilities/cv-vision.ts

diff --git a/src/main/capabilities/cv-vision.ts b/src/main/capabilities/cv-vision.ts
new file mode 100644
index 0000000..bfdab8e
--- /dev/null
+++ b/src/main/capabilities/cv-vision.ts
@@ -0,0 +1,60 @@
+import { register } from "../engine/registry";
+import { RuntimeError } from "../engine/types";
+import { powershell } from "./desktop";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+interface CvTarget { text: string; cx: number; cy: number }
+let paddleOcr: { detect: (image: string) => Promise<{ texts: Array<{ text: string; box?: number[][]; mean?: number }> }> } | null = null;
+
+async function getOcr() {
+  if (paddleOcr) return paddleOcr;
+  const mod = await import("@gutenye/ocr-node");
+  paddleOcr = await mod.default.create();
+  return paddleOcr as { detect: (image: string) => Promise<{ texts: Array<{ text: string; box?: number[][]; mean?: number }> }> };
+}
+
+function cvTarget(args: Record<string, unknown>, nodeId: string): string {
+  for (const key of ["cvSelector", "target", "text"]) {
+    const v = String(args[key] ?? "").trim();
+    if (v) return v.replace(/^text:/, "").replace(/^图片:/, "");
+  }
+  throw new RuntimeError("validation_error", "CV 目标为空", { nodeId, componentId: nodeId, recommendation: "填写要在屏幕上查找的目标文本" });
+}
+
+async function grabScreen(imagePath: string): Promise<void> {
+  fs.mkdirSync(path.dirname(imagePath), { recursive: true });
+  const psScript = [
+    "Add-Type -AssemblyName System.Drawing | Out-Null;",
+    "$b = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds;",
+    "$bmp = New-Object System.Drawing.Bitmap($b.Width, $b.Height);",
+    "$g = [System.Drawing.Graphics]::FromImage($bmp);",
+    "$g.CopyFromScreen($b.X, $b.Y, 0, 0, $b.Size);",
+    "$g.Dispose();",
+    "$bmp.Save(" + JSON.stringify(imagePath) + ");",
+    "$bmp.Dispose();"
+  ].join(" ");
+  await powershell(psScript, 30000);
+}
+
+interface OcrLine { text: string; box?: number[][]; mean?: number }
+
+async function locateTextOnScreen(target: string, nodeId: string): Promise<{ target: string; text: string; cx: number; cy: number }> {
+  const shotPath = path.join(os.tmpdir(), "rpa-cv-screen-" + Date.now() + ".png");
+  await grabScreen(shotPath);
+  const ocr = await getOcr();
+  const result = await ocr.detect(shotPath);
+  const texts = result.texts ?? [];
+  const keyword = target.trim();
+  const hit = texts.find(t => t.text && t.text.includes(keyword));
+  try { fs.unlinkSync(shotPath); } catch {}
+  if (!hit || !hit.box || hit.box.length < 4) {
+    throw new RuntimeError("cv_not_found", "屏幕上未找到目标文本：" + keyword, { nodeId, componentId: nodeId, recommendation: "确认目标文本在屏幕上可见且清晰" });
+  }
+  const xs = hit.box.map(p => p[0]);
+  const ys = hit.box.map(p => p[1]);
+  const cx = (Math.min(...xs) + Math.max(...xs)) / 2;
+  const cy = (Math.min(...ys) + Math.max(...ys)) / 2;
+  return { target: keyword, text: hit.text, cx, cy };
+}
\ No newline at end of file