feat: 框选与 CV 查找支持多显示器副屏

- pickScreenRegion 每个显示器各开一个框选覆盖层, 副屏可直接框选; 返回屏幕绝对 DIP 坐标+该屏 scaleFactor
- cv-vision 全屏查找改为逐显示器 desktopCapturer 截屏(含负坐标副屏), 命中换算为绝对 DIP 坐标; 区域截图命中按 scaleFactor 换算
- 点击/悬停/输入坐标经 screen.dipToScreenPoint 转物理像素, 修复缩放非 100% 时点击偏移
- 新增 2 个多显示器测试用例(副屏命中映射/区域图缩放换算), 11/11 通过

src/main/capabilities/cv-vision.ts

@@ -41,30 +41,43 @@ async function gdiGrabScreen(imagePath: string): Promise<void> {
   await powershell(psScript, 30000);
 }
 
-async function grabScreen(imagePath: string): Promise<void> {
-  fs.mkdirSync(path.dirname(imagePath), { recursive: true });
-  // GDI CopyFromScreen 截不到 GPU 合成窗口(Electron 自身界面, 会截出全黑),
-  // 优先用 desktopCapturer 走系统合成器; 不可用时(如单测环境)回退 GDI 脚本
+/** 一次屏幕截图: 图片左上角在虚拟桌面上的绝对 DIP 坐标 + 物理/DIP 缩放比 */
+interface ScreenShot { imagePath: string; dipX: number; dipY: number; scale: number }
+
+/** desktopCapturer 逐显示器截屏(每屏原生分辨率), 不可用时回退 GDI 全虚拟桌面单图 */
+async function grabScreenAll(baseDir: string): Promise<ScreenShot[]> {
+  fs.mkdirSync(baseDir, { recursive: true });
   try {
     const { desktopCapturer, screen } = await import("electron");
-    const display = screen.getPrimaryDisplay();
-    const sources = await desktopCapturer.getSources({
-      types: ["screen"],
-      thumbnailSize: { width: display.size.width, height: display.size.height }
-    });
-    const source = sources.find(item => item.display_id === String(display.id)) || sources[0];
-    if (source && !source.thumbnail.isEmpty()) {
-      fs.writeFileSync(imagePath, source.thumbnail.toPNG());
-      return;
+    if (desktopCapturer && screen?.getAllDisplays) {
+      const shots: ScreenShot[] = [];
+      const displays = screen.getAllDisplays();
+      for (const display of displays) {
+        const scale = display.scaleFactor || 1;
+        const sources = await desktopCapturer.getSources({
+          types: ["screen"],
+          thumbnailSize: { width: Math.round(display.bounds.width * scale), height: Math.round(display.bounds.height * scale) }
+        });
+        const source = sources.find(item => item.display_id === String(display.id))
+          || sources.find(item => item.thumbnail.getSize().width === Math.round(display.bounds.width * scale))
+          || sources[0];
+        if (!source || source.thumbnail.isEmpty()) continue;
+        const imagePath = path.join(baseDir, `rpa-cv-screen-${display.id}-${Date.now()}.png`);
+        fs.writeFileSync(imagePath, source.thumbnail.toPNG());
+        shots.push({ imagePath, dipX: display.bounds.x, dipY: display.bounds.y, scale });
+      }
+      if (shots.length) return shots;
     }
   } catch {}
+  const imagePath = path.join(baseDir, `rpa-cv-screen-${Date.now()}.png`);
   await gdiGrabScreen(imagePath);
+  return [{ imagePath, dipX: 0, dipY: 0, scale: 1 }];
 }
 
 interface OcrLine { text: string; box?: number[][]; mean?: number }
 
-/** 搜索区域: 框选生成时含 imagePath(区域截图)+x/y(屏幕偏移); 也可只给坐标范围 */
-interface CvRegion { imagePath?: string; x?: number; y?: number; width?: number; height?: number }
+/** 搜索区域: 框选生成时含 imagePath(区域截图)+x/y(屏幕绝对 DIP 坐标)+scale(该屏缩放); 也可只给坐标范围 */
+interface CvRegion { imagePath?: string; x?: number; y?: number; width?: number; height?: number; scale?: number }
 
 function parseRegionArg(value: unknown): CvRegion | null {
   if (!value) return null;
@@ -89,46 +102,63 @@ export async function locateTextOnScreen(
   nodeId: string,
   region?: CvRegion | null
 ): Promise<{ target: string; text: string; cx: number; cy: number }> {
-  let shotPath: string | null = null;
-  let imagePath: string;
-  let offsetX = 0;
-  let offsetY = 0;
-  if (region?.imagePath && fs.existsSync(region.imagePath)) {
-    // 框选截图: OCR 只识别该图, 命中坐标加上框选时的屏幕偏移得到绝对坐标
-    imagePath = region.imagePath;
-    offsetX = Number(region.x ?? 0) || 0;
-    offsetY = Number(region.y ?? 0) || 0;
-  } else {
-    shotPath = path.join(os.tmpdir(), "rpa-cv-screen-" + Date.now() + ".png");
-    await grabScreen(shotPath);
-    imagePath = shotPath;
-  }
   const ocr = await getOcr();
-  const result = await ocr.detect(imagePath);
-  const texts = result.texts ?? [];
   const keyword = target.trim();
-  const candidates = texts.filter(t => t.text && t.text.includes(keyword) && t.box && t.box.length >= 4);
-  let hit: OcrLine | undefined = candidates[0];
-  // 未用框选图但给了坐标范围时, 过滤中心落在区域内的命中
-  if (hit && !region?.imagePath && region?.width && region?.height) {
-    const rx = Number(region.x ?? 0) || 0;
-    const ry = Number(region.y ?? 0) || 0;
-    hit = candidates.find(t => {
-      const c = boxCenter(t.box!);
-      return c.cx >= rx && c.cx <= rx + region.width! && c.cy >= ry && c.cy <= ry + region.height!;
-    });
+  // 命中中心换算为屏幕绝对 DIP 坐标
+  let found: { text: string; cx: number; cy: number } | null = null;
+  const cleanups: string[] = [];
+
+  if (region?.imagePath && fs.existsSync(region.imagePath)) {
+    // 框选截图: OCR 只识别该图; 图内物理坐标 ÷ 该屏缩放 + 框选绝对 DIP 原点
+    const scale = Number(region.scale ?? 1) || 1;
+    const result = await ocr.detect(region.imagePath);
+    const hit = (result.texts ?? []).find(t => t.text && t.text.includes(keyword) && t.box && t.box.length >= 4);
+    if (hit?.box) {
+      const c = boxCenter(hit.box);
+      found = {
+        text: hit.text,
+        cx: (Number(region.x ?? 0) || 0) + c.cx / scale,
+        cy: (Number(region.y ?? 0) || 0) + c.cy / scale
+      };
+    }
+  } else {
+    // 全屏查找: 逐显示器截屏识别, 覆盖副屏(含负坐标屏); 命中即停
+    const shots = await grabScreenAll(path.join(os.tmpdir(), "rpa-cv-shots"));
+    for (const shot of shots) {
+      const result = await ocr.detect(shot.imagePath);
+      const candidates = (result.texts ?? []).filter(t => t.text && t.text.includes(keyword) && t.box && t.box.length >= 4);
+      let hit: OcrLine | undefined = candidates[0];
+      // 给了坐标范围(绝对 DIP)时, 过滤中心落在区域内的命中
+      if (hit && region?.width && region?.height) {
+        const rx = Number(region.x ?? 0) || 0;
+        const ry = Number(region.y ?? 0) || 0;
+        hit = candidates.find(t => {
+          const c = boxCenter(t.box!);
+          const dx = shot.dipX + c.cx / shot.scale;
+          const dy = shot.dipY + c.cy / shot.scale;
+          return dx >= rx && dx <= rx + region.width! && dy >= ry && dy <= ry + region.height!;
+        });
+      }
+      if (hit?.box) {
+        const c = boxCenter(hit.box);
+        found = { text: hit.text, cx: shot.dipX + c.cx / shot.scale, cy: shot.dipY + c.cy / shot.scale };
+        break;
+      }
+      cleanups.push(shot.imagePath);
+    }
+    // 成功命中时清理全部截图, 未命中保留全部以便排查
+    if (found) for (const p of shots.map(s => s.imagePath)) { try { fs.unlinkSync(p); } catch {} }
   }
-  if (!hit || !hit.box || hit.box.length < 4) {
-    // 找不到目标时保留截屏, 便于排查(截屏范围/遮挡/清晰度)
+
+  if (!found) {
     throw new RuntimeError("cv_not_found", "屏幕上未找到目标文本：" + keyword + (region?.imagePath ? "（搜索范围: 框选区域）" : ""), {
       nodeId,
       componentId: nodeId,
-      recommendation: "确认目标文本在" + (region?.imagePath ? "框选区域内" : "屏幕上") + "可见且清晰；本次识别图片已保留: " + (region?.imagePath ?? shotPath)
+      recommendation: "确认目标文本在" + (region?.imagePath ? "框选区域内" : "屏幕上(含副屏)") + "可见且清晰"
     });
   }
-  if (shotPath) { try { fs.unlinkSync(shotPath); } catch {} }
-  const c = boxCenter(hit.box);
-  return { target: keyword, text: hit.text, cx: c.cx + offsetX, cy: c.cy + offsetY };
+  for (const p of cleanups) { try { fs.unlinkSync(p); } catch {} }
+  return { target: keyword, text: found.text, cx: found.cx, cy: found.cy };
 }
 function screenUser32(): string {
   return [
@@ -177,9 +207,21 @@ function coordinateScript(x: number, y: number, action: "hover" | "click" | "dou
   return lines.join("\n");
 }
 
+/** DIP → 物理像素(SetCursorPos 在 DPI 感知进程里用物理坐标); 无 Electron API 时按原值(缩放 100% 等价) */
+async function toPhysical(x: number, y: number): Promise<{ x: number; y: number }> {
+  try {
+    const { screen } = await import("electron");
+    if (screen && typeof screen.dipToScreenPoint === "function") {
+      const p = screen.dipToScreenPoint({ x: Math.round(x), y: Math.round(y) });
+      return { x: p.x, y: p.y };
+    }
+  } catch {}
+  return { x: Math.round(x), y: Math.round(y) };
+}
+
 async function locateFor(args: Record<string, unknown>, node: { id: string; componentId: string }) {
   const target = cvTarget(args, node.id);
-  // 可选搜索区域: 框选生成 {x,y,width,height,imagePath}, 限定 OCR 查找范围(字段明天接入 UI)
+  // 可选搜索区域: 框选生成 {x,y,width,height,imagePath,scale}, 限定 OCR 查找范围
   const region = parseRegionArg(args.searchRegion ?? args.region);
   const located = await locateTextOnScreen(target, node.id, region);
   const offsetX = Number(args.offsetX ?? args.offset_x ?? 0) || 0;
@@ -198,24 +240,27 @@ const setTextGround: CapabilityHandler = async (ctx, node, args) => {
   const value = String(args.value ?? args.text ?? "");
   if (!value) throw new RuntimeError("validation_error", "CV 文本输入内容为空", { nodeId: node.id, componentId: node.componentId });
   const { x, y } = await locateFor(args, node);
-  await powershell(coordinateScript(x, y, "paste", value), 30000);
-  ctx.log("CV_SetText input at " + x + "," + y, "info", node.id);
+  const phys = await toPhysical(x, y);
+  await powershell(coordinateScript(phys.x, phys.y, "paste", value), 30000);
+  ctx.log("CV_SetText input at " + phys.x + "," + phys.y, "info", node.id);
   return true;
 };
 
 const mouseHoverGround: CapabilityHandler = async (ctx, node, args) => {
   const { x, y } = await locateFor(args, node);
-  await powershell(coordinateScript(x, y, "hover"), 15000);
-  ctx.log("CV_MouseHover hover at " + x + "," + y, "info", node.id);
+  const phys = await toPhysical(x, y);
+  await powershell(coordinateScript(phys.x, phys.y, "hover"), 15000);
+  ctx.log("CV_MouseHover hover at " + phys.x + "," + phys.y, "info", node.id);
   return true;
 };
 
 const mouseClickGround: CapabilityHandler = async (ctx, node, args) => {
   const { x, y } = await locateFor(args, node);
+  const phys = await toPhysical(x, y);
   const clickway = String(args.clickway ?? "0");
   const action = clickway === "1" || /double/i.test(clickway) ? "doubleClick" as const : "click" as const;
-  await powershell(coordinateScript(x, y, action), 15000);
-  ctx.log("CV_MouseClick " + action + " at " + x + "," + y, "info", node.id);
+  await powershell(coordinateScript(phys.x, phys.y, action), 15000);
+  ctx.log("CV_MouseClick " + action + " at " + phys.x + "," + phys.y, "info", node.id);
   return true;
 };