import { register } from "../engine/registry"; import { RuntimeError } from "../engine/types"; import { powershell } from "./desktop"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; interface CvTarget { text: string; cx: number; cy: number } let paddleOcr: { detect: (image: string) => Promise<{ texts: Array<{ text: string; box?: number[][]; mean?: number }> }> } | null = null; async function getOcr() { if (paddleOcr) return paddleOcr; const mod = await import("@gutenye/ocr-node"); paddleOcr = await mod.default.create(); return paddleOcr as { detect: (image: string) => Promise<{ texts: Array<{ text: string; box?: number[][]; mean?: number }> }> }; } function cvTarget(args: Record, nodeId: string): string { for (const key of ["cvSelector", "target", "text"]) { const v = String(args[key] ?? "").trim(); if (v) return v.replace(/^text:/, "").replace(/^图片:/, ""); } throw new RuntimeError("validation_error", "CV 目标为空", { nodeId, componentId: nodeId, recommendation: "填写要在屏幕上查找的目标文本" }); } async function grabScreen(imagePath: string): Promise { fs.mkdirSync(path.dirname(imagePath), { recursive: true }); const psScript = [ "Add-Type -AssemblyName System.Drawing | Out-Null;", "$b = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds;", "$bmp = New-Object System.Drawing.Bitmap($b.Width, $b.Height);", "$g = [System.Drawing.Graphics]::FromImage($bmp);", "$g.CopyFromScreen($b.X, $b.Y, 0, 0, $b.Size);", "$g.Dispose();", "$bmp.Save(" + JSON.stringify(imagePath) + ");", "$bmp.Dispose();" ].join(" "); await powershell(psScript, 30000); } interface OcrLine { text: string; box?: number[][]; mean?: number } export async function locateTextOnScreen(target: string, nodeId: string): Promise<{ target: string; text: string; cx: number; cy: number }> { const shotPath = path.join(os.tmpdir(), "rpa-cv-screen-" + Date.now() + ".png"); await grabScreen(shotPath); const ocr = await getOcr(); const result = await ocr.detect(shotPath); const texts = result.texts ?? []; const keyword = target.trim(); const hit = texts.find(t => t.text && t.text.includes(keyword)); try { fs.unlinkSync(shotPath); } catch {} if (!hit || !hit.box || hit.box.length < 4) { throw new RuntimeError("cv_not_found", "屏幕上未找到目标文本:" + keyword, { nodeId, componentId: nodeId, recommendation: "确认目标文本在屏幕上可见且清晰" }); } const xs = hit.box.map(p => p[0]); const ys = hit.box.map(p => p[1]); const cx = (Math.min(...xs) + Math.max(...xs)) / 2; const cy = (Math.min(...ys) + Math.max(...ys)) / 2; return { target: keyword, text: hit.text, cx, cy }; }