import { register } from "../engine/registry"; import { RuntimeError, type CapabilityHandler } from "../engine/types"; import { powershell } from "./desktop"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; interface CvTarget { text: string; cx: number; cy: number } let paddleOcr: { detect: (image: string) => Promise<{ texts: Array<{ text: string; box?: number[][]; mean?: number }> }> } | null = null; async function getOcr() { if (paddleOcr) return paddleOcr; const mod = await import("@gutenye/ocr-node"); paddleOcr = await mod.default.create(); return paddleOcr as { detect: (image: string) => Promise<{ texts: Array<{ text: string; box?: number[][]; mean?: number }> }> }; } function cvTarget(args: Record, nodeId: string): string { for (const key of ["cvSelector", "target", "text"]) { const v = String(args[key] ?? "").trim(); if (v) return v.replace(/^text:/, "").replace(/^图片:/, ""); } throw new RuntimeError("validation_error", "CV 目标为空", { nodeId, componentId: nodeId, recommendation: "填写要在屏幕上查找的目标文本" }); } async function grabScreen(imagePath: string): Promise { fs.mkdirSync(path.dirname(imagePath), { recursive: true }); const psScript = [ // 与点击脚本一致: 先声明 DPI 感知, 否则缩放非 100% 时 Bounds 是逻辑像素, 截屏只覆盖屏幕左上角 "Add-Type -MemberDefinition '[DllImport(\"user32.dll\")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value);' -Name RpaCvDpi -Namespace RpaCv | Out-Null;", "try { [RpaCv.RpaCvDpi]::SetProcessDpiAwarenessContext([IntPtr](-4)) | Out-Null } catch {}", "Add-Type -AssemblyName System.Drawing | Out-Null;", "Add-Type -AssemblyName System.Windows.Forms | Out-Null;", "$b = [System.Windows.Forms.SystemInformation]::VirtualScreen;", "$bmp = New-Object System.Drawing.Bitmap($b.Width, $b.Height);", "$g = [System.Drawing.Graphics]::FromImage($bmp);", "$g.CopyFromScreen($b.X, $b.Y, 0, 0, $b.Size);", "$g.Dispose();", "$bmp.Save(" + JSON.stringify(imagePath) + ");", "$bmp.Dispose();" ].join(" "); await powershell(psScript, 30000); } interface OcrLine { text: string; box?: number[][]; mean?: number } export async function locateTextOnScreen(target: string, nodeId: string): Promise<{ target: string; text: string; cx: number; cy: number }> { const shotPath = path.join(os.tmpdir(), "rpa-cv-screen-" + Date.now() + ".png"); await grabScreen(shotPath); const ocr = await getOcr(); const result = await ocr.detect(shotPath); const texts = result.texts ?? []; const keyword = target.trim(); const hit = texts.find(t => t.text && t.text.includes(keyword)); if (!hit || !hit.box || hit.box.length < 4) { // 找不到目标时保留截屏, 便于排查(截屏范围/遮挡/清晰度) throw new RuntimeError("cv_not_found", "屏幕上未找到目标文本:" + keyword, { nodeId, componentId: nodeId, recommendation: "确认目标文本在屏幕上可见且清晰;本次截屏已保留: " + shotPath }); } try { fs.unlinkSync(shotPath); } catch {} const xs = hit.box.map(p => p[0]); const ys = hit.box.map(p => p[1]); const cx = (Math.min(...xs) + Math.max(...xs)) / 2; const cy = (Math.min(...ys) + Math.max(...ys)) / 2; return { target: keyword, text: hit.text, cx, cy }; } function screenUser32(): string { return [ "Add-Type -MemberDefinition '[DllImport(\"user32.dll\")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value); [DllImport(\"user32.dll\")] public static extern bool SetCursorPos(int X, int Y); [DllImport(\"user32.dll\")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra);' -Name RpaCvInput -Namespace RpaCv | Out-Null;", "try { [RpaCv.RpaCvInput]::SetProcessDpiAwarenessContext([IntPtr](-4)) | Out-Null } catch {}" ].join("\n"); } function coordinateScript(x: number, y: number, action: "hover" | "click" | "doubleClick" | "paste", value = ""): string { const lines = [ screenUser32(), "if (-not [RpaCv.RpaCvInput]::SetCursorPos(" + x + ", " + y + ")) { throw 'cursor_move_failed' }", "Start-Sleep -Milliseconds 100" ]; if (action !== "hover") { lines.push( "[RpaCv.RpaCvInput]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)", "Start-Sleep -Milliseconds 40", "[RpaCv.RpaCvInput]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)" ); } if (action === "doubleClick") { lines.push( "Start-Sleep -Milliseconds 70", "[RpaCv.RpaCvInput]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)", "Start-Sleep -Milliseconds 40", "[RpaCv.RpaCvInput]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)" ); } if (action === "paste") { lines.push( "Add-Type -AssemblyName System.Windows.Forms", "Start-Sleep -Milliseconds 220", "$saved = $null", "try { $saved = [System.Windows.Forms.Clipboard]::GetText() } catch {}", "try { [System.Windows.Forms.Clipboard]::SetText(" + JSON.stringify(value) + ") } catch { throw 'clipboard_failed' }", "Start-Sleep -Milliseconds 80", "[System.Windows.Forms.SendKeys]::SendWait('^a')", "Start-Sleep -Milliseconds 60", "[System.Windows.Forms.SendKeys]::SendWait('^v')", "Start-Sleep -Milliseconds 280", "try { if ($saved) { [System.Windows.Forms.Clipboard]::SetText($saved) } else { [System.Windows.Forms.Clipboard]::Clear() } } catch {}" ); } lines.push("Write-Output (@{ ok = $true; x = " + x + "; y = " + y + " } | ConvertTo-Json -Compress)"); return lines.join("\n"); } async function locateFor(args: Record, node: { id: string; componentId: string }) { const target = cvTarget(args, node.id); const located = await locateTextOnScreen(target, node.id); const offsetX = Number(args.offsetX ?? args.offset_x ?? 0) || 0; const offsetY = Number(args.offsetY ?? args.offset_y ?? 0) || 0; return { located, x: Math.round(located.cx + offsetX), y: Math.round(located.cy + offsetY) }; } const getTextGround: CapabilityHandler = async (ctx, node, args) => { const target = cvTarget(args, node.id); const result = await locateTextOnScreen(target, node.id); ctx.log("CV_GetText found: " + result.text, "info", node.id); return result.text; }; const setTextGround: CapabilityHandler = async (ctx, node, args) => { const value = String(args.value ?? args.text ?? ""); if (!value) throw new RuntimeError("validation_error", "CV 文本输入内容为空", { nodeId: node.id, componentId: node.componentId }); const { x, y } = await locateFor(args, node); await powershell(coordinateScript(x, y, "paste", value), 30000); ctx.log("CV_SetText input at " + x + "," + y, "info", node.id); return true; }; const mouseHoverGround: CapabilityHandler = async (ctx, node, args) => { const { x, y } = await locateFor(args, node); await powershell(coordinateScript(x, y, "hover"), 15000); ctx.log("CV_MouseHover hover at " + x + "," + y, "info", node.id); return true; }; const mouseClickGround: CapabilityHandler = async (ctx, node, args) => { const { x, y } = await locateFor(args, node); const clickway = String(args.clickway ?? "0"); const action = clickway === "1" || /double/i.test(clickway) ? "doubleClick" as const : "click" as const; await powershell(coordinateScript(x, y, action), 15000); ctx.log("CV_MouseClick " + action + " at " + x + "," + y, "info", node.id); return true; }; register("CV_GetText", getTextGround); register("CV_SetText", setTextGround); register("CV_MouseHover", mouseHoverGround); register("CV_MouseClick", mouseClickGround);