From 5aa113ea0e89530e307157ed8647b9127adf905d Mon Sep 17 00:00:00 2001
From: yu <191972707@qq.com>
Date: Mon, 2 Nov 2026 17:55:18 +0800
Subject: [PATCH] =?UTF-8?q?feat:=20CV=20=E7=BB=84=E4=BB=B6=E6=94=AF?=
 =?UTF-8?q?=E6=8C=81=E5=8F=AF=E9=80=89=E6=90=9C=E7=B4=A2=E5=8C=BA=E5=9F=9F?=
 =?UTF-8?q?(=E7=AC=AC=E4=B8=80=E9=98=B6=E6=AE=B5)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- locateTextOnScreen 支持区域参数: 框选截图(imagePath)+屏幕偏移(x,y) 或纯坐标范围过滤
- locateFor 读取 searchRegion/region 参数; parseRegionArg 兼容 JSON 与图片路径
- 框选截图时 OCR 只识别该图, 命中坐标加偏移得到屏幕绝对坐标, 避免同名文本干扰
- pickRegionToField: region 类型字段存完整 {x,y,width,height,imagePath}
- 明日继续: catalog 给 CV 四组件加 searchRegion 字段 + region 分支屏幕框选按钮
---
 src/main/capabilities/cv-vision.ts     | 82 ++++++++++++++++++++------
 src/renderer/src/panels/Properties.tsx | 10 +++-
 2 files changed, 72 insertions(+), 20 deletions(-)

diff --git a/src/main/capabilities/cv-vision.ts b/src/main/capabilities/cv-vision.ts
index 84ebc5d..8a809be 100644
--- a/src/main/capabilities/cv-vision.ts
+++ b/src/main/capabilities/cv-vision.ts
@@ -44,28 +44,72 @@ async function grabScreen(imagePath: string): Promise<void> {
 
 interface OcrLine { text: string; box?: number[][]; mean?: number }
 
-export async function locateTextOnScreen(target: string, nodeId: string): Promise<{ target: string; text: string; cx: number; cy: number }> {
-  const shotPath = path.join(os.tmpdir(), "rpa-cv-screen-" + Date.now() + ".png");
-  await grabScreen(shotPath);
+/** 搜索区域: 框选生成时含 imagePath(区域截图)+x/y(屏幕偏移); 也可只给坐标范围 */
+interface CvRegion { imagePath?: string; x?: number; y?: number; width?: number; height?: number }
+
+function parseRegionArg(value: unknown): CvRegion | null {
+  if (!value) return null;
+  if (typeof value === "object") return value as CvRegion;
+  const s = String(value).trim();
+  if (!s) return null;
+  if (s.startsWith("{")) {
+    try { return JSON.parse(s) as CvRegion; } catch { return null; }
+  }
+  if (/\.(png|jpe?g|bmp)$/i.test(s) || fs.existsSync(s)) return { imagePath: s };
+  return null;
+}
+
+function boxCenter(box: number[][]): { cx: number; cy: number } {
+  const xs = box.map(p => p[0]);
+  const ys = box.map(p => p[1]);
+  return { cx: (Math.min(...xs) + Math.max(...xs)) / 2, cy: (Math.min(...ys) + Math.max(...ys)) / 2 };
+}
+
+export async function locateTextOnScreen(
+  target: string,
+  nodeId: string,
+  region?: CvRegion | null
+): Promise<{ target: string; text: string; cx: number; cy: number }> {
+  let shotPath: string | null = null;
+  let imagePath: string;
+  let offsetX = 0;
+  let offsetY = 0;
+  if (region?.imagePath && fs.existsSync(region.imagePath)) {
+    // 框选截图: OCR 只识别该图, 命中坐标加上框选时的屏幕偏移得到绝对坐标
+    imagePath = region.imagePath;
+    offsetX = Number(region.x ?? 0) || 0;
+    offsetY = Number(region.y ?? 0) || 0;
+  } else {
+    shotPath = path.join(os.tmpdir(), "rpa-cv-screen-" + Date.now() + ".png");
+    await grabScreen(shotPath);
+    imagePath = shotPath;
+  }
   const ocr = await getOcr();
-  const result = await ocr.detect(shotPath);
+  const result = await ocr.detect(imagePath);
   const texts = result.texts ?? [];
   const keyword = target.trim();
-  const hit = texts.find(t => t.text && t.text.includes(keyword));
-  if (!hit || !hit.box || hit.box.length < 4) {
-    // 找不到目标时保留截屏, 便于排查(截屏范围/遮挡/清晰度)
-    throw new RuntimeError("cv_not_found", "屏幕上未找到目标文本：" + keyword, {
-      nodeId,
-      componentId: nodeId,
-      recommendation: "确认目标文本在屏幕上可见且清晰；本次截屏已保留: " + shotPath
+  const candidates = texts.filter(t => t.text && t.text.includes(keyword) && t.box && t.box.length >= 4);
+  let hit: OcrLine | undefined = candidates[0];
+  // 未用框选图但给了坐标范围时, 过滤中心落在区域内的命中
+  if (hit && !region?.imagePath && region?.width && region?.height) {
+    const rx = Number(region.x ?? 0) || 0;
+    const ry = Number(region.y ?? 0) || 0;
+    hit = candidates.find(t => {
+      const c = boxCenter(t.box!);
+      return c.cx >= rx && c.cx <= rx + region.width! && c.cy >= ry && c.cy <= ry + region.height!;
     });
   }
-  try { fs.unlinkSync(shotPath); } catch {}
-  const xs = hit.box.map(p => p[0]);
-  const ys = hit.box.map(p => p[1]);
-  const cx = (Math.min(...xs) + Math.max(...xs)) / 2;
-  const cy = (Math.min(...ys) + Math.max(...ys)) / 2;
-  return { target: keyword, text: hit.text, cx, cy };
+  if (!hit || !hit.box || hit.box.length < 4) {
+    // 找不到目标时保留截屏, 便于排查(截屏范围/遮挡/清晰度)
+    throw new RuntimeError("cv_not_found", "屏幕上未找到目标文本：" + keyword + (region?.imagePath ? "（搜索范围: 框选区域）" : ""), {
+      nodeId,
+      componentId: nodeId,
+      recommendation: "确认目标文本在" + (region?.imagePath ? "框选区域内" : "屏幕上") + "可见且清晰；本次识别图片已保留: " + (region?.imagePath ?? shotPath)
+    });
+  }
+  if (shotPath) { try { fs.unlinkSync(shotPath); } catch {} }
+  const c = boxCenter(hit.box);
+  return { target: keyword, text: hit.text, cx: c.cx + offsetX, cy: c.cy + offsetY };
 }
 function screenUser32(): string {
   return [
@@ -116,7 +160,9 @@ function coordinateScript(x: number, y: number, action: "hover" | "click" | "dou
 
 async function locateFor(args: Record<string, unknown>, node: { id: string; componentId: string }) {
   const target = cvTarget(args, node.id);
-  const located = await locateTextOnScreen(target, node.id);
+  // 可选搜索区域: 框选生成 {x,y,width,height,imagePath}, 限定 OCR 查找范围(字段明天接入 UI)
+  const region = parseRegionArg(args.searchRegion ?? args.region);
+  const located = await locateTextOnScreen(target, node.id, region);
   const offsetX = Number(args.offsetX ?? args.offset_x ?? 0) || 0;
   const offsetY = Number(args.offsetY ?? args.offset_y ?? 0) || 0;
   return { located, x: Math.round(located.cx + offsetX), y: Math.round(located.cy + offsetY) };
diff --git a/src/renderer/src/panels/Properties.tsx b/src/renderer/src/panels/Properties.tsx
index 2587097..7c43473 100644
--- a/src/renderer/src/panels/Properties.tsx
+++ b/src/renderer/src/panels/Properties.tsx
@@ -609,8 +609,14 @@ function FieldControl({
     setPicking(true);
     try {
       const region = await window.rpa.desktop.pickRegion();
-      onChange(field.key, region.imagePath);
-      onToast("已框选并截图：" + region.width + "×" + region.height);
+      if (field.widget === "region") {
+        // region 字段存完整信息: 坐标+尺寸+截图路径, 运行时可据此定位区域内容在屏幕上的绝对位置
+        onChange(field.key, JSON.stringify({ x: region.x, y: region.y, width: region.width, height: region.height, imagePath: region.imagePath }));
+        onToast("已框选区域：" + region.width + "×" + region.height + " @(" + region.x + "," + region.y + ")");
+      } else {
+        onChange(field.key, region.imagePath);
+        onToast("已框选并截图：" + region.width + "×" + region.height);
+      }
     } catch (error) {
       onToast(error instanceof Error ? error.message : String(error), "error");
     } finally {