60 lines
2.7 KiB
TypeScript
60 lines
2.7 KiB
TypeScript
import { register } from "../engine/registry";
|
|
import { RuntimeError } from "../engine/types";
|
|
import { powershell } from "./desktop";
|
|
import fs from "node:fs";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
|
|
interface CvTarget { text: string; cx: number; cy: number }
|
|
let paddleOcr: { detect: (image: string) => Promise<{ texts: Array<{ text: string; box?: number[][]; mean?: number }> }> } | null = null;
|
|
|
|
async function getOcr() {
|
|
if (paddleOcr) return paddleOcr;
|
|
const mod = await import("@gutenye/ocr-node");
|
|
paddleOcr = await mod.default.create();
|
|
return paddleOcr as { detect: (image: string) => Promise<{ texts: Array<{ text: string; box?: number[][]; mean?: number }> }> };
|
|
}
|
|
|
|
function cvTarget(args: Record<string, unknown>, nodeId: string): string {
|
|
for (const key of ["cvSelector", "target", "text"]) {
|
|
const v = String(args[key] ?? "").trim();
|
|
if (v) return v.replace(/^text:/, "").replace(/^图片:/, "");
|
|
}
|
|
throw new RuntimeError("validation_error", "CV 目标为空", { nodeId, componentId: nodeId, recommendation: "填写要在屏幕上查找的目标文本" });
|
|
}
|
|
|
|
async function grabScreen(imagePath: string): Promise<void> {
|
|
fs.mkdirSync(path.dirname(imagePath), { recursive: true });
|
|
const psScript = [
|
|
"Add-Type -AssemblyName System.Drawing | Out-Null;",
|
|
"$b = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds;",
|
|
"$bmp = New-Object System.Drawing.Bitmap($b.Width, $b.Height);",
|
|
"$g = [System.Drawing.Graphics]::FromImage($bmp);",
|
|
"$g.CopyFromScreen($b.X, $b.Y, 0, 0, $b.Size);",
|
|
"$g.Dispose();",
|
|
"$bmp.Save(" + JSON.stringify(imagePath) + ");",
|
|
"$bmp.Dispose();"
|
|
].join(" ");
|
|
await powershell(psScript, 30000);
|
|
}
|
|
|
|
interface OcrLine { text: string; box?: number[][]; mean?: number }
|
|
|
|
export async function locateTextOnScreen(target: string, nodeId: string): Promise<{ target: string; text: string; cx: number; cy: number }> {
|
|
const shotPath = path.join(os.tmpdir(), "rpa-cv-screen-" + Date.now() + ".png");
|
|
await grabScreen(shotPath);
|
|
const ocr = await getOcr();
|
|
const result = await ocr.detect(shotPath);
|
|
const texts = result.texts ?? [];
|
|
const keyword = target.trim();
|
|
const hit = texts.find(t => t.text && t.text.includes(keyword));
|
|
try { fs.unlinkSync(shotPath); } catch {}
|
|
if (!hit || !hit.box || hit.box.length < 4) {
|
|
throw new RuntimeError("cv_not_found", "屏幕上未找到目标文本：" + keyword, { nodeId, componentId: nodeId, recommendation: "确认目标文本在屏幕上可见且清晰" });
|
|
}
|
|
const xs = hit.box.map(p => p[0]);
|
|
const ys = hit.box.map(p => p[1]);
|
|
const cx = (Math.min(...xs) + Math.max(...xs)) / 2;
|
|
const cy = (Math.min(...ys) + Math.max(...ys)) / 2;
|
|
return { target: keyword, text: hit.text, cx, cy };
|
|
} |