diff --git a/src/main/capabilities/cv-vision.test.ts b/src/main/capabilities/cv-vision.test.ts index 125cbfe..31c52c4 100644 --- a/src/main/capabilities/cv-vision.test.ts +++ b/src/main/capabilities/cv-vision.test.ts @@ -1,4 +1,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; const powershellMock = vi.hoisted(() => vi.fn(async () => '{"ok":true}')); const detectMock = vi.hoisted(() => vi.fn(async () => ({ @@ -7,11 +10,26 @@ const detectMock = vi.hoisted(() => vi.fn(async () => ({ { text: "确认", box: [[100, 200], [130, 200], [130, 210], [100, 210]] } ] }))); +// desktopCapturer 模拟: 默认无画面(走 GDI 回退), 多屏用例按请求宽度提供对应显示器画面 +const electronState = vi.hoisted(() => ({ sources: [] as Array<{ display_id: string; thumbnail: any }> })); vi.mock("./desktop", () => ({ powershell: powershellMock })); vi.mock("@gutenye/ocr-node", () => ({ default: { create: vi.fn(async () => ({ detect: detectMock })) } })); +vi.mock("electron", () => ({ + desktopCapturer: { + getSources: vi.fn(async (opts: { thumbnailSize: { width: number } }) => + electronState.sources.filter(s => s.thumbnail.getSize().width === opts.thumbnailSize.width)) + }, + screen: { + getAllDisplays: vi.fn(() => [ + { id: 1, bounds: { x: 0, y: 0, width: 1920, height: 1080 }, scaleFactor: 1 }, + { id: 2, bounds: { x: 1920, y: 0, width: 1280, height: 720 }, scaleFactor: 2 } + ]), + dipToScreenPoint: vi.fn((p: { x: number; y: number }) => ({ x: p.x, y: p.y })) + } +})); import { getHandler } from "../engine/registry"; import "./cv-vision"; @@ -20,6 +38,7 @@ describe("CV vision capabilities", () => { beforeEach(() => { powershellMock.mockClear(); detectMock.mockClear(); + electronState.sources = []; }); it.each(["CV_GetText", "CV_SetText", "CV_MouseHover", "CV_MouseClick"])("registers %s", (id) => { @@ -67,4 +86,41 @@ describe("CV vision capabilities", () => { searchRegion: JSON.stringify({ x: 500, y: 500, width: 50, height: 50 }) }, {} as any)).rejects.toThrow("未找到目标文本"); }); + + it("locates targets on a secondary display and maps DIP coordinates", async () => { + // 主屏无画面, 副屏(bounds 1920,0 1280x720 缩放2)物理中心(450,210) → DIP(2145,105) + const fakeThumb = (w: number, h: number): any => ({ + isEmpty: () => false, + getSize: () => ({ width: w, height: h }), + toPNG: () => Buffer.from([137, 80, 78, 71]) + }); + electronState.sources = [{ display_id: "2", thumbnail: fakeThumb(2560, 1440) }]; + detectMock.mockImplementationOnce(async () => ({ + texts: [{ text: "副屏按钮", box: [[400, 200], [500, 200], [500, 220], [400, 220]] }] + })); + await expect(getHandler("CV_MouseClick")!({ log: vi.fn() } as any, { id: "n5", componentId: "CV_MouseClick" } as any, { + cvSelector: "副屏按钮" + }, {} as any)).resolves.toBe(true); + const script = String((powershellMock.mock.calls as unknown as Array<[string]>).at(-1)?.[0] ?? ""); + expect(script).toContain("SetCursorPos(2145, 105)"); + }); + + it("maps region-image hits through the display scale", async () => { + // 框选图来自缩放2的副屏: 图内物理中心(120,90) + 框选DIP原点(1920,0) → DIP(1980,45) + const regionShot = path.join(os.tmpdir(), "rpa-cv-region-test-" + Date.now() + ".png"); + fs.writeFileSync(regionShot, Buffer.from([137, 80, 78, 71])); + try { + detectMock.mockImplementationOnce(async () => ({ + texts: [{ text: "确认", box: [[100, 80], [140, 80], [140, 100], [100, 100]] }] + })); + await expect(getHandler("CV_MouseHover")!({ log: vi.fn() } as any, { id: "n6", componentId: "CV_MouseHover" } as any, { + cvSelector: "确认", + searchRegion: JSON.stringify({ x: 1920, y: 0, width: 640, height: 360, scale: 2, imagePath: regionShot }) + }, {} as any)).resolves.toBe(true); + const script = String((powershellMock.mock.calls as unknown as Array<[string]>).at(-1)?.[0] ?? ""); + expect(script).toContain("SetCursorPos(1980, 45)"); + } finally { + try { fs.unlinkSync(regionShot); } catch {} + } + }); }); diff --git a/src/main/capabilities/cv-vision.ts b/src/main/capabilities/cv-vision.ts index 434e9da..50ad6ce 100644 --- a/src/main/capabilities/cv-vision.ts +++ b/src/main/capabilities/cv-vision.ts @@ -41,30 +41,43 @@ async function gdiGrabScreen(imagePath: string): Promise { await powershell(psScript, 30000); } -async function grabScreen(imagePath: string): Promise { - fs.mkdirSync(path.dirname(imagePath), { recursive: true }); - // GDI CopyFromScreen 截不到 GPU 合成窗口(Electron 自身界面, 会截出全黑), - // 优先用 desktopCapturer 走系统合成器; 不可用时(如单测环境)回退 GDI 脚本 +/** 一次屏幕截图: 图片左上角在虚拟桌面上的绝对 DIP 坐标 + 物理/DIP 缩放比 */ +interface ScreenShot { imagePath: string; dipX: number; dipY: number; scale: number } + +/** desktopCapturer 逐显示器截屏(每屏原生分辨率), 不可用时回退 GDI 全虚拟桌面单图 */ +async function grabScreenAll(baseDir: string): Promise { + fs.mkdirSync(baseDir, { recursive: true }); try { const { desktopCapturer, screen } = await import("electron"); - const display = screen.getPrimaryDisplay(); - const sources = await desktopCapturer.getSources({ - types: ["screen"], - thumbnailSize: { width: display.size.width, height: display.size.height } - }); - const source = sources.find(item => item.display_id === String(display.id)) || sources[0]; - if (source && !source.thumbnail.isEmpty()) { - fs.writeFileSync(imagePath, source.thumbnail.toPNG()); - return; + if (desktopCapturer && screen?.getAllDisplays) { + const shots: ScreenShot[] = []; + const displays = screen.getAllDisplays(); + for (const display of displays) { + const scale = display.scaleFactor || 1; + const sources = await desktopCapturer.getSources({ + types: ["screen"], + thumbnailSize: { width: Math.round(display.bounds.width * scale), height: Math.round(display.bounds.height * scale) } + }); + const source = sources.find(item => item.display_id === String(display.id)) + || sources.find(item => item.thumbnail.getSize().width === Math.round(display.bounds.width * scale)) + || sources[0]; + if (!source || source.thumbnail.isEmpty()) continue; + const imagePath = path.join(baseDir, `rpa-cv-screen-${display.id}-${Date.now()}.png`); + fs.writeFileSync(imagePath, source.thumbnail.toPNG()); + shots.push({ imagePath, dipX: display.bounds.x, dipY: display.bounds.y, scale }); + } + if (shots.length) return shots; } } catch {} + const imagePath = path.join(baseDir, `rpa-cv-screen-${Date.now()}.png`); await gdiGrabScreen(imagePath); + return [{ imagePath, dipX: 0, dipY: 0, scale: 1 }]; } interface OcrLine { text: string; box?: number[][]; mean?: number } -/** 搜索区域: 框选生成时含 imagePath(区域截图)+x/y(屏幕偏移); 也可只给坐标范围 */ -interface CvRegion { imagePath?: string; x?: number; y?: number; width?: number; height?: number } +/** 搜索区域: 框选生成时含 imagePath(区域截图)+x/y(屏幕绝对 DIP 坐标)+scale(该屏缩放); 也可只给坐标范围 */ +interface CvRegion { imagePath?: string; x?: number; y?: number; width?: number; height?: number; scale?: number } function parseRegionArg(value: unknown): CvRegion | null { if (!value) return null; @@ -89,46 +102,63 @@ export async function locateTextOnScreen( nodeId: string, region?: CvRegion | null ): Promise<{ target: string; text: string; cx: number; cy: number }> { - let shotPath: string | null = null; - let imagePath: string; - let offsetX = 0; - let offsetY = 0; - if (region?.imagePath && fs.existsSync(region.imagePath)) { - // 框选截图: OCR 只识别该图, 命中坐标加上框选时的屏幕偏移得到绝对坐标 - imagePath = region.imagePath; - offsetX = Number(region.x ?? 0) || 0; - offsetY = Number(region.y ?? 0) || 0; - } else { - shotPath = path.join(os.tmpdir(), "rpa-cv-screen-" + Date.now() + ".png"); - await grabScreen(shotPath); - imagePath = shotPath; - } const ocr = await getOcr(); - const result = await ocr.detect(imagePath); - const texts = result.texts ?? []; const keyword = target.trim(); - const candidates = texts.filter(t => t.text && t.text.includes(keyword) && t.box && t.box.length >= 4); - let hit: OcrLine | undefined = candidates[0]; - // 未用框选图但给了坐标范围时, 过滤中心落在区域内的命中 - if (hit && !region?.imagePath && region?.width && region?.height) { - const rx = Number(region.x ?? 0) || 0; - const ry = Number(region.y ?? 0) || 0; - hit = candidates.find(t => { - const c = boxCenter(t.box!); - return c.cx >= rx && c.cx <= rx + region.width! && c.cy >= ry && c.cy <= ry + region.height!; - }); + // 命中中心换算为屏幕绝对 DIP 坐标 + let found: { text: string; cx: number; cy: number } | null = null; + const cleanups: string[] = []; + + if (region?.imagePath && fs.existsSync(region.imagePath)) { + // 框选截图: OCR 只识别该图; 图内物理坐标 ÷ 该屏缩放 + 框选绝对 DIP 原点 + const scale = Number(region.scale ?? 1) || 1; + const result = await ocr.detect(region.imagePath); + const hit = (result.texts ?? []).find(t => t.text && t.text.includes(keyword) && t.box && t.box.length >= 4); + if (hit?.box) { + const c = boxCenter(hit.box); + found = { + text: hit.text, + cx: (Number(region.x ?? 0) || 0) + c.cx / scale, + cy: (Number(region.y ?? 0) || 0) + c.cy / scale + }; + } + } else { + // 全屏查找: 逐显示器截屏识别, 覆盖副屏(含负坐标屏); 命中即停 + const shots = await grabScreenAll(path.join(os.tmpdir(), "rpa-cv-shots")); + for (const shot of shots) { + const result = await ocr.detect(shot.imagePath); + const candidates = (result.texts ?? []).filter(t => t.text && t.text.includes(keyword) && t.box && t.box.length >= 4); + let hit: OcrLine | undefined = candidates[0]; + // 给了坐标范围(绝对 DIP)时, 过滤中心落在区域内的命中 + if (hit && region?.width && region?.height) { + const rx = Number(region.x ?? 0) || 0; + const ry = Number(region.y ?? 0) || 0; + hit = candidates.find(t => { + const c = boxCenter(t.box!); + const dx = shot.dipX + c.cx / shot.scale; + const dy = shot.dipY + c.cy / shot.scale; + return dx >= rx && dx <= rx + region.width! && dy >= ry && dy <= ry + region.height!; + }); + } + if (hit?.box) { + const c = boxCenter(hit.box); + found = { text: hit.text, cx: shot.dipX + c.cx / shot.scale, cy: shot.dipY + c.cy / shot.scale }; + break; + } + cleanups.push(shot.imagePath); + } + // 成功命中时清理全部截图, 未命中保留全部以便排查 + if (found) for (const p of shots.map(s => s.imagePath)) { try { fs.unlinkSync(p); } catch {} } } - if (!hit || !hit.box || hit.box.length < 4) { - // 找不到目标时保留截屏, 便于排查(截屏范围/遮挡/清晰度) + + if (!found) { throw new RuntimeError("cv_not_found", "屏幕上未找到目标文本:" + keyword + (region?.imagePath ? "(搜索范围: 框选区域)" : ""), { nodeId, componentId: nodeId, - recommendation: "确认目标文本在" + (region?.imagePath ? "框选区域内" : "屏幕上") + "可见且清晰;本次识别图片已保留: " + (region?.imagePath ?? shotPath) + recommendation: "确认目标文本在" + (region?.imagePath ? "框选区域内" : "屏幕上(含副屏)") + "可见且清晰" }); } - if (shotPath) { try { fs.unlinkSync(shotPath); } catch {} } - const c = boxCenter(hit.box); - return { target: keyword, text: hit.text, cx: c.cx + offsetX, cy: c.cy + offsetY }; + for (const p of cleanups) { try { fs.unlinkSync(p); } catch {} } + return { target: keyword, text: found.text, cx: found.cx, cy: found.cy }; } function screenUser32(): string { return [ @@ -177,9 +207,21 @@ function coordinateScript(x: number, y: number, action: "hover" | "click" | "dou return lines.join("\n"); } +/** DIP → 物理像素(SetCursorPos 在 DPI 感知进程里用物理坐标); 无 Electron API 时按原值(缩放 100% 等价) */ +async function toPhysical(x: number, y: number): Promise<{ x: number; y: number }> { + try { + const { screen } = await import("electron"); + if (screen && typeof screen.dipToScreenPoint === "function") { + const p = screen.dipToScreenPoint({ x: Math.round(x), y: Math.round(y) }); + return { x: p.x, y: p.y }; + } + } catch {} + return { x: Math.round(x), y: Math.round(y) }; +} + async function locateFor(args: Record, node: { id: string; componentId: string }) { const target = cvTarget(args, node.id); - // 可选搜索区域: 框选生成 {x,y,width,height,imagePath}, 限定 OCR 查找范围(字段明天接入 UI) + // 可选搜索区域: 框选生成 {x,y,width,height,imagePath,scale}, 限定 OCR 查找范围 const region = parseRegionArg(args.searchRegion ?? args.region); const located = await locateTextOnScreen(target, node.id, region); const offsetX = Number(args.offsetX ?? args.offset_x ?? 0) || 0; @@ -198,24 +240,27 @@ const setTextGround: CapabilityHandler = async (ctx, node, args) => { const value = String(args.value ?? args.text ?? ""); if (!value) throw new RuntimeError("validation_error", "CV 文本输入内容为空", { nodeId: node.id, componentId: node.componentId }); const { x, y } = await locateFor(args, node); - await powershell(coordinateScript(x, y, "paste", value), 30000); - ctx.log("CV_SetText input at " + x + "," + y, "info", node.id); + const phys = await toPhysical(x, y); + await powershell(coordinateScript(phys.x, phys.y, "paste", value), 30000); + ctx.log("CV_SetText input at " + phys.x + "," + phys.y, "info", node.id); return true; }; const mouseHoverGround: CapabilityHandler = async (ctx, node, args) => { const { x, y } = await locateFor(args, node); - await powershell(coordinateScript(x, y, "hover"), 15000); - ctx.log("CV_MouseHover hover at " + x + "," + y, "info", node.id); + const phys = await toPhysical(x, y); + await powershell(coordinateScript(phys.x, phys.y, "hover"), 15000); + ctx.log("CV_MouseHover hover at " + phys.x + "," + phys.y, "info", node.id); return true; }; const mouseClickGround: CapabilityHandler = async (ctx, node, args) => { const { x, y } = await locateFor(args, node); + const phys = await toPhysical(x, y); const clickway = String(args.clickway ?? "0"); const action = clickway === "1" || /double/i.test(clickway) ? "doubleClick" as const : "click" as const; - await powershell(coordinateScript(x, y, action), 15000); - ctx.log("CV_MouseClick " + action + " at " + x + "," + y, "info", node.id); + await powershell(coordinateScript(phys.x, phys.y, action), 15000); + ctx.log("CV_MouseClick " + action + " at " + phys.x + "," + phys.y, "info", node.id); return true; }; diff --git a/src/main/desktop-pick.ts b/src/main/desktop-pick.ts index b7d7931..e3fb8a6 100644 --- a/src/main/desktop-pick.ts +++ b/src/main/desktop-pick.ts @@ -696,17 +696,8 @@ export async function listVisibleWindows(): Promise>(stdout, "["); } -export async function pickScreenRegion(): Promise<{ x: number; y: number; width: number; height: number; imagePath: string }> { - const appWindows = BrowserWindow.getAllWindows().filter(w => !w.isDestroyed()); - for (const w of appWindows) w.minimize(); - const display = screen.getPrimaryDisplay(); - const sx = display.bounds.x, sy = display.bounds.y, sw = display.bounds.width, sh = display.bounds.height; - const win = new BrowserWindow({ - x: sx, y: sy, width: sw, height: sh, - frame: false, transparent: true, alwaysOnTop: true, skipTaskbar: true, resizable: false, movable: false, - webPreferences: { contextIsolation: true, nodeIntegration: false, sandbox: true } - }); - const htmlParts = [ +function regionOverlayHtml(): string { + return [ "