import { register } from "../engine/registry"; import { RuntimeError, type CapabilityHandler } from "../engine/types"; import { powershell } from "./desktop"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; interface CvTarget { text: string; cx: number; cy: number } let paddleOcr: { detect: (image: string) => Promise<{ texts: Array<{ text: string; box?: number[][]; mean?: number }> }> } | null = null; async function getOcr() { if (paddleOcr) return paddleOcr; const mod = await import("@gutenye/ocr-node"); paddleOcr = await mod.default.create(); return paddleOcr as { detect: (image: string) => Promise<{ texts: Array<{ text: string; box?: number[][]; mean?: number }> }> }; } function cvTarget(args: Record, nodeId: string): string { for (const key of ["cvSelector", "target", "text"]) { const v = String(args[key] ?? "").trim(); if (v) return v.replace(/^text:/, "").replace(/^图片:/, ""); } throw new RuntimeError("validation_error", "CV 目标为空", { nodeId, componentId: nodeId, recommendation: "填写要在屏幕上查找的目标文本" }); } async function gdiGrabScreen(imagePath: string): Promise { const psScript = [ // 与点击脚本一致: 先声明 DPI 感知, 否则缩放非 100% 时 Bounds 是逻辑像素, 截屏只覆盖屏幕左上角 "Add-Type -MemberDefinition '[DllImport(\"user32.dll\")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value);' -Name RpaCvDpi -Namespace RpaCv | Out-Null;", "try { [RpaCv.RpaCvDpi]::SetProcessDpiAwarenessContext([IntPtr](-4)) | Out-Null } catch {}", "Add-Type -AssemblyName System.Drawing | Out-Null;", "Add-Type -AssemblyName System.Windows.Forms | Out-Null;", "$b = [System.Windows.Forms.SystemInformation]::VirtualScreen;", "$bmp = New-Object System.Drawing.Bitmap($b.Width, $b.Height);", "$g = [System.Drawing.Graphics]::FromImage($bmp);", "$g.CopyFromScreen($b.X, $b.Y, 0, 0, $b.Size);", "$g.Dispose();", "$bmp.Save(" + JSON.stringify(imagePath) + ");", "$bmp.Dispose();" ].join(" "); await powershell(psScript, 30000); } /** 一次屏幕截图: 图片左上角在虚拟桌面上的绝对 DIP 坐标 + 物理/DIP 缩放比 */ interface ScreenShot { imagePath: string; dipX: number; dipY: number; scale: number } /** desktopCapturer 逐显示器截屏(每屏原生分辨率), 不可用时回退 GDI 全虚拟桌面单图 */ async function grabScreenAll(baseDir: string): Promise { fs.mkdirSync(baseDir, { recursive: true }); try { const { desktopCapturer, screen } = await import("electron"); if (desktopCapturer && screen?.getAllDisplays) { const shots: ScreenShot[] = []; const displays = screen.getAllDisplays(); for (const display of displays) { const scale = display.scaleFactor || 1; const sources = await desktopCapturer.getSources({ types: ["screen"], thumbnailSize: { width: Math.round(display.bounds.width * scale), height: Math.round(display.bounds.height * scale) } }); const source = sources.find(item => item.display_id === String(display.id)) || sources.find(item => item.thumbnail.getSize().width === Math.round(display.bounds.width * scale)) || sources[0]; if (!source || source.thumbnail.isEmpty()) continue; const imagePath = path.join(baseDir, `rpa-cv-screen-${display.id}-${Date.now()}.png`); fs.writeFileSync(imagePath, source.thumbnail.toPNG()); shots.push({ imagePath, dipX: display.bounds.x, dipY: display.bounds.y, scale }); } if (shots.length) return shots; } } catch {} const imagePath = path.join(baseDir, `rpa-cv-screen-${Date.now()}.png`); await gdiGrabScreen(imagePath); return [{ imagePath, dipX: 0, dipY: 0, scale: 1 }]; } interface OcrLine { text: string; box?: number[][]; mean?: number } /** 搜索区域: 框选生成时含 imagePath(区域截图)+x/y(屏幕绝对 DIP 坐标)+scale(该屏缩放); 也可只给坐标范围 */ interface CvRegion { imagePath?: string; x?: number; y?: number; width?: number; height?: number; scale?: number } function parseRegionArg(value: unknown): CvRegion | null { if (!value) return null; if (typeof value === "object") return value as CvRegion; const s = String(value).trim(); if (!s) return null; if (s.startsWith("{")) { try { return JSON.parse(s) as CvRegion; } catch { return null; } } if (/\.(png|jpe?g|bmp)$/i.test(s) || fs.existsSync(s)) return { imagePath: s }; return null; } function boxCenter(box: number[][]): { cx: number; cy: number } { const xs = box.map(p => p[0]); const ys = box.map(p => p[1]); return { cx: (Math.min(...xs) + Math.max(...xs)) / 2, cy: (Math.min(...ys) + Math.max(...ys)) / 2 }; } export async function locateTextOnScreen( target: string, nodeId: string, region?: CvRegion | null ): Promise<{ target: string; text: string; cx: number; cy: number }> { const ocr = await getOcr(); const keyword = target.trim(); // 命中中心换算为屏幕绝对 DIP 坐标 let found: { text: string; cx: number; cy: number } | null = null; const cleanups: string[] = []; if (region?.imagePath && fs.existsSync(region.imagePath)) { // 框选截图: OCR 只识别该图; 图内物理坐标 ÷ 该屏缩放 + 框选绝对 DIP 原点 const scale = Number(region.scale ?? 1) || 1; const result = await ocr.detect(region.imagePath); const hit = (result.texts ?? []).find(t => t.text && t.text.includes(keyword) && t.box && t.box.length >= 4); if (hit?.box) { const c = boxCenter(hit.box); found = { text: hit.text, cx: (Number(region.x ?? 0) || 0) + c.cx / scale, cy: (Number(region.y ?? 0) || 0) + c.cy / scale }; } } else { // 全屏查找: 逐显示器截屏识别, 覆盖副屏(含负坐标屏); 命中即停 const shots = await grabScreenAll(path.join(os.tmpdir(), "rpa-cv-shots")); for (const shot of shots) { const result = await ocr.detect(shot.imagePath); const candidates = (result.texts ?? []).filter(t => t.text && t.text.includes(keyword) && t.box && t.box.length >= 4); let hit: OcrLine | undefined = candidates[0]; // 给了坐标范围(绝对 DIP)时, 过滤中心落在区域内的命中 if (hit && region?.width && region?.height) { const rx = Number(region.x ?? 0) || 0; const ry = Number(region.y ?? 0) || 0; hit = candidates.find(t => { const c = boxCenter(t.box!); const dx = shot.dipX + c.cx / shot.scale; const dy = shot.dipY + c.cy / shot.scale; return dx >= rx && dx <= rx + region.width! && dy >= ry && dy <= ry + region.height!; }); } if (hit?.box) { const c = boxCenter(hit.box); found = { text: hit.text, cx: shot.dipX + c.cx / shot.scale, cy: shot.dipY + c.cy / shot.scale }; break; } cleanups.push(shot.imagePath); } // 成功命中时清理全部截图, 未命中保留全部以便排查 if (found) for (const p of shots.map(s => s.imagePath)) { try { fs.unlinkSync(p); } catch {} } } if (!found) { throw new RuntimeError("cv_not_found", "屏幕上未找到目标文本:" + keyword + (region?.imagePath ? "(搜索范围: 框选区域)" : ""), { nodeId, componentId: nodeId, recommendation: "确认目标文本在" + (region?.imagePath ? "框选区域内" : "屏幕上(含副屏)") + "可见且清晰" }); } for (const p of cleanups) { try { fs.unlinkSync(p); } catch {} } return { target: keyword, text: found.text, cx: found.cx, cy: found.cy }; } function screenUser32(): string { return [ "Add-Type -MemberDefinition '[DllImport(\"user32.dll\")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value); [DllImport(\"user32.dll\")] public static extern bool SetCursorPos(int X, int Y); [DllImport(\"user32.dll\")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra);' -Name RpaCvInput -Namespace RpaCv | Out-Null;", "try { [RpaCv.RpaCvInput]::SetProcessDpiAwarenessContext([IntPtr](-4)) | Out-Null } catch {}" ].join("\n"); } function coordinateScript(x: number, y: number, action: "hover" | "click" | "doubleClick" | "paste", value = ""): string { const lines = [ screenUser32(), "if (-not [RpaCv.RpaCvInput]::SetCursorPos(" + x + ", " + y + ")) { throw 'cursor_move_failed' }", "Start-Sleep -Milliseconds 100" ]; if (action !== "hover") { lines.push( "[RpaCv.RpaCvInput]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)", "Start-Sleep -Milliseconds 40", "[RpaCv.RpaCvInput]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)" ); } if (action === "doubleClick") { lines.push( "Start-Sleep -Milliseconds 70", "[RpaCv.RpaCvInput]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)", "Start-Sleep -Milliseconds 40", "[RpaCv.RpaCvInput]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)" ); } if (action === "paste") { lines.push( "Add-Type -AssemblyName System.Windows.Forms", "Start-Sleep -Milliseconds 220", "$saved = $null", "try { $saved = [System.Windows.Forms.Clipboard]::GetText() } catch {}", "try { [System.Windows.Forms.Clipboard]::SetText(" + JSON.stringify(value) + ") } catch { throw 'clipboard_failed' }", "Start-Sleep -Milliseconds 80", "[System.Windows.Forms.SendKeys]::SendWait('^a')", "Start-Sleep -Milliseconds 60", "[System.Windows.Forms.SendKeys]::SendWait('^v')", "Start-Sleep -Milliseconds 280", "try { if ($saved) { [System.Windows.Forms.Clipboard]::SetText($saved) } else { [System.Windows.Forms.Clipboard]::Clear() } } catch {}" ); } lines.push("Write-Output (@{ ok = $true; x = " + x + "; y = " + y + " } | ConvertTo-Json -Compress)"); return lines.join("\n"); } /** DIP → 物理像素(SetCursorPos 在 DPI 感知进程里用物理坐标); 无 Electron API 时按原值(缩放 100% 等价) */ async function toPhysical(x: number, y: number): Promise<{ x: number; y: number }> { try { const { screen } = await import("electron"); if (screen && typeof screen.dipToScreenPoint === "function") { const p = screen.dipToScreenPoint({ x: Math.round(x), y: Math.round(y) }); return { x: p.x, y: p.y }; } } catch {} return { x: Math.round(x), y: Math.round(y) }; } async function locateFor(args: Record, node: { id: string; componentId: string }) { const target = cvTarget(args, node.id); // 可选搜索区域: 框选生成 {x,y,width,height,imagePath,scale}, 限定 OCR 查找范围 const region = parseRegionArg(args.searchRegion ?? args.region); const located = await locateTextOnScreen(target, node.id, region); const offsetX = Number(args.offsetX ?? args.offset_x ?? 0) || 0; const offsetY = Number(args.offsetY ?? args.offset_y ?? 0) || 0; return { located, x: Math.round(located.cx + offsetX), y: Math.round(located.cy + offsetY) }; } const getTextGround: CapabilityHandler = async (ctx, node, args) => { const target = cvTarget(args, node.id); const result = await locateTextOnScreen(target, node.id); ctx.log("CV_GetText found: " + result.text, "info", node.id); return result.text; }; const setTextGround: CapabilityHandler = async (ctx, node, args) => { const value = String(args.value ?? args.text ?? ""); if (!value) throw new RuntimeError("validation_error", "CV 文本输入内容为空", { nodeId: node.id, componentId: node.componentId }); const { x, y } = await locateFor(args, node); const phys = await toPhysical(x, y); await powershell(coordinateScript(phys.x, phys.y, "paste", value), 30000); ctx.log("CV_SetText input at " + phys.x + "," + phys.y, "info", node.id); return true; }; const mouseHoverGround: CapabilityHandler = async (ctx, node, args) => { const { x, y } = await locateFor(args, node); const phys = await toPhysical(x, y); await powershell(coordinateScript(phys.x, phys.y, "hover"), 15000); ctx.log("CV_MouseHover hover at " + phys.x + "," + phys.y, "info", node.id); return true; }; const mouseClickGround: CapabilityHandler = async (ctx, node, args) => { const { x, y } = await locateFor(args, node); const phys = await toPhysical(x, y); const clickway = String(args.clickway ?? "0"); const action = clickway === "1" || /double/i.test(clickway) ? "doubleClick" as const : "click" as const; await powershell(coordinateScript(phys.x, phys.y, action), 15000); ctx.log("CV_MouseClick " + action + " at " + phys.x + "," + phys.y, "info", node.id); return true; }; register("CV_GetText", getTextGround); register("CV_SetText", setTextGround); register("CV_MouseHover", mouseHoverGround); register("CV_MouseClick", mouseClickGround);