yu
b40c15d5b2
- pickScreenRegion 每个显示器各开一个框选覆盖层, 副屏可直接框选; 返回屏幕绝对 DIP 坐标+该屏 scaleFactor - cv-vision 全屏查找改为逐显示器 desktopCapturer 截屏(含负坐标副屏), 命中换算为绝对 DIP 坐标; 区域截图命中按 scaleFactor 换算 - 点击/悬停/输入坐标经 screen.dipToScreenPoint 转物理像素, 修复缩放非 100% 时点击偏移 - 新增 2 个多显示器测试用例(副屏命中映射/区域图缩放换算), 11/11 通过
271 lines
13 KiB
TypeScript
271 lines
13 KiB
TypeScript
import { register } from "../engine/registry";
|
|
import { RuntimeError, type CapabilityHandler } from "../engine/types";
|
|
import { powershell } from "./desktop";
|
|
import fs from "node:fs";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
|
|
interface CvTarget { text: string; cx: number; cy: number }
|
|
let paddleOcr: { detect: (image: string) => Promise<{ texts: Array<{ text: string; box?: number[][]; mean?: number }> }> } | null = null;
|
|
|
|
async function getOcr() {
|
|
if (paddleOcr) return paddleOcr;
|
|
const mod = await import("@gutenye/ocr-node");
|
|
paddleOcr = await mod.default.create();
|
|
return paddleOcr as { detect: (image: string) => Promise<{ texts: Array<{ text: string; box?: number[][]; mean?: number }> }> };
|
|
}
|
|
|
|
function cvTarget(args: Record<string, unknown>, nodeId: string): string {
|
|
for (const key of ["cvSelector", "target", "text"]) {
|
|
const v = String(args[key] ?? "").trim();
|
|
if (v) return v.replace(/^text:/, "").replace(/^图片:/, "");
|
|
}
|
|
throw new RuntimeError("validation_error", "CV 目标为空", { nodeId, componentId: nodeId, recommendation: "填写要在屏幕上查找的目标文本" });
|
|
}
|
|
|
|
async function gdiGrabScreen(imagePath: string): Promise<void> {
|
|
const psScript = [
|
|
// 与点击脚本一致: 先声明 DPI 感知, 否则缩放非 100% 时 Bounds 是逻辑像素, 截屏只覆盖屏幕左上角
|
|
"Add-Type -MemberDefinition '[DllImport(\"user32.dll\")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value);' -Name RpaCvDpi -Namespace RpaCv | Out-Null;",
|
|
"try { [RpaCv.RpaCvDpi]::SetProcessDpiAwarenessContext([IntPtr](-4)) | Out-Null } catch {}",
|
|
"Add-Type -AssemblyName System.Drawing | Out-Null;",
|
|
"Add-Type -AssemblyName System.Windows.Forms | Out-Null;",
|
|
"$b = [System.Windows.Forms.SystemInformation]::VirtualScreen;",
|
|
"$bmp = New-Object System.Drawing.Bitmap($b.Width, $b.Height);",
|
|
"$g = [System.Drawing.Graphics]::FromImage($bmp);",
|
|
"$g.CopyFromScreen($b.X, $b.Y, 0, 0, $b.Size);",
|
|
"$g.Dispose();",
|
|
"$bmp.Save(" + JSON.stringify(imagePath) + ");",
|
|
"$bmp.Dispose();"
|
|
].join(" ");
|
|
await powershell(psScript, 30000);
|
|
}
|
|
|
|
/** 一次屏幕截图: 图片左上角在虚拟桌面上的绝对 DIP 坐标 + 物理/DIP 缩放比 */
|
|
interface ScreenShot { imagePath: string; dipX: number; dipY: number; scale: number }
|
|
|
|
/** desktopCapturer 逐显示器截屏(每屏原生分辨率), 不可用时回退 GDI 全虚拟桌面单图 */
|
|
async function grabScreenAll(baseDir: string): Promise<ScreenShot[]> {
|
|
fs.mkdirSync(baseDir, { recursive: true });
|
|
try {
|
|
const { desktopCapturer, screen } = await import("electron");
|
|
if (desktopCapturer && screen?.getAllDisplays) {
|
|
const shots: ScreenShot[] = [];
|
|
const displays = screen.getAllDisplays();
|
|
for (const display of displays) {
|
|
const scale = display.scaleFactor || 1;
|
|
const sources = await desktopCapturer.getSources({
|
|
types: ["screen"],
|
|
thumbnailSize: { width: Math.round(display.bounds.width * scale), height: Math.round(display.bounds.height * scale) }
|
|
});
|
|
const source = sources.find(item => item.display_id === String(display.id))
|
|
|| sources.find(item => item.thumbnail.getSize().width === Math.round(display.bounds.width * scale))
|
|
|| sources[0];
|
|
if (!source || source.thumbnail.isEmpty()) continue;
|
|
const imagePath = path.join(baseDir, `rpa-cv-screen-${display.id}-${Date.now()}.png`);
|
|
fs.writeFileSync(imagePath, source.thumbnail.toPNG());
|
|
shots.push({ imagePath, dipX: display.bounds.x, dipY: display.bounds.y, scale });
|
|
}
|
|
if (shots.length) return shots;
|
|
}
|
|
} catch {}
|
|
const imagePath = path.join(baseDir, `rpa-cv-screen-${Date.now()}.png`);
|
|
await gdiGrabScreen(imagePath);
|
|
return [{ imagePath, dipX: 0, dipY: 0, scale: 1 }];
|
|
}
|
|
|
|
interface OcrLine { text: string; box?: number[][]; mean?: number }
|
|
|
|
/** 搜索区域: 框选生成时含 imagePath(区域截图)+x/y(屏幕绝对 DIP 坐标)+scale(该屏缩放); 也可只给坐标范围 */
|
|
interface CvRegion { imagePath?: string; x?: number; y?: number; width?: number; height?: number; scale?: number }
|
|
|
|
function parseRegionArg(value: unknown): CvRegion | null {
|
|
if (!value) return null;
|
|
if (typeof value === "object") return value as CvRegion;
|
|
const s = String(value).trim();
|
|
if (!s) return null;
|
|
if (s.startsWith("{")) {
|
|
try { return JSON.parse(s) as CvRegion; } catch { return null; }
|
|
}
|
|
if (/\.(png|jpe?g|bmp)$/i.test(s) || fs.existsSync(s)) return { imagePath: s };
|
|
return null;
|
|
}
|
|
|
|
function boxCenter(box: number[][]): { cx: number; cy: number } {
|
|
const xs = box.map(p => p[0]);
|
|
const ys = box.map(p => p[1]);
|
|
return { cx: (Math.min(...xs) + Math.max(...xs)) / 2, cy: (Math.min(...ys) + Math.max(...ys)) / 2 };
|
|
}
|
|
|
|
export async function locateTextOnScreen(
|
|
target: string,
|
|
nodeId: string,
|
|
region?: CvRegion | null
|
|
): Promise<{ target: string; text: string; cx: number; cy: number }> {
|
|
const ocr = await getOcr();
|
|
const keyword = target.trim();
|
|
// 命中中心换算为屏幕绝对 DIP 坐标
|
|
let found: { text: string; cx: number; cy: number } | null = null;
|
|
const cleanups: string[] = [];
|
|
|
|
if (region?.imagePath && fs.existsSync(region.imagePath)) {
|
|
// 框选截图: OCR 只识别该图; 图内物理坐标 ÷ 该屏缩放 + 框选绝对 DIP 原点
|
|
const scale = Number(region.scale ?? 1) || 1;
|
|
const result = await ocr.detect(region.imagePath);
|
|
const hit = (result.texts ?? []).find(t => t.text && t.text.includes(keyword) && t.box && t.box.length >= 4);
|
|
if (hit?.box) {
|
|
const c = boxCenter(hit.box);
|
|
found = {
|
|
text: hit.text,
|
|
cx: (Number(region.x ?? 0) || 0) + c.cx / scale,
|
|
cy: (Number(region.y ?? 0) || 0) + c.cy / scale
|
|
};
|
|
}
|
|
} else {
|
|
// 全屏查找: 逐显示器截屏识别, 覆盖副屏(含负坐标屏); 命中即停
|
|
const shots = await grabScreenAll(path.join(os.tmpdir(), "rpa-cv-shots"));
|
|
for (const shot of shots) {
|
|
const result = await ocr.detect(shot.imagePath);
|
|
const candidates = (result.texts ?? []).filter(t => t.text && t.text.includes(keyword) && t.box && t.box.length >= 4);
|
|
let hit: OcrLine | undefined = candidates[0];
|
|
// 给了坐标范围(绝对 DIP)时, 过滤中心落在区域内的命中
|
|
if (hit && region?.width && region?.height) {
|
|
const rx = Number(region.x ?? 0) || 0;
|
|
const ry = Number(region.y ?? 0) || 0;
|
|
hit = candidates.find(t => {
|
|
const c = boxCenter(t.box!);
|
|
const dx = shot.dipX + c.cx / shot.scale;
|
|
const dy = shot.dipY + c.cy / shot.scale;
|
|
return dx >= rx && dx <= rx + region.width! && dy >= ry && dy <= ry + region.height!;
|
|
});
|
|
}
|
|
if (hit?.box) {
|
|
const c = boxCenter(hit.box);
|
|
found = { text: hit.text, cx: shot.dipX + c.cx / shot.scale, cy: shot.dipY + c.cy / shot.scale };
|
|
break;
|
|
}
|
|
cleanups.push(shot.imagePath);
|
|
}
|
|
// 成功命中时清理全部截图, 未命中保留全部以便排查
|
|
if (found) for (const p of shots.map(s => s.imagePath)) { try { fs.unlinkSync(p); } catch {} }
|
|
}
|
|
|
|
if (!found) {
|
|
throw new RuntimeError("cv_not_found", "屏幕上未找到目标文本：" + keyword + (region?.imagePath ? "（搜索范围: 框选区域）" : ""), {
|
|
nodeId,
|
|
componentId: nodeId,
|
|
recommendation: "确认目标文本在" + (region?.imagePath ? "框选区域内" : "屏幕上(含副屏)") + "可见且清晰"
|
|
});
|
|
}
|
|
for (const p of cleanups) { try { fs.unlinkSync(p); } catch {} }
|
|
return { target: keyword, text: found.text, cx: found.cx, cy: found.cy };
|
|
}
|
|
function screenUser32(): string {
|
|
return [
|
|
"Add-Type -MemberDefinition '[DllImport(\"user32.dll\")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value); [DllImport(\"user32.dll\")] public static extern bool SetCursorPos(int X, int Y); [DllImport(\"user32.dll\")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra);' -Name RpaCvInput -Namespace RpaCv | Out-Null;",
|
|
"try { [RpaCv.RpaCvInput]::SetProcessDpiAwarenessContext([IntPtr](-4)) | Out-Null } catch {}"
|
|
].join("\n");
|
|
}
|
|
|
|
function coordinateScript(x: number, y: number, action: "hover" | "click" | "doubleClick" | "paste", value = ""): string {
|
|
const lines = [
|
|
screenUser32(),
|
|
"if (-not [RpaCv.RpaCvInput]::SetCursorPos(" + x + ", " + y + ")) { throw 'cursor_move_failed' }",
|
|
"Start-Sleep -Milliseconds 100"
|
|
];
|
|
if (action !== "hover") {
|
|
lines.push(
|
|
"[RpaCv.RpaCvInput]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)",
|
|
"Start-Sleep -Milliseconds 40",
|
|
"[RpaCv.RpaCvInput]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)"
|
|
);
|
|
}
|
|
if (action === "doubleClick") {
|
|
lines.push(
|
|
"Start-Sleep -Milliseconds 70",
|
|
"[RpaCv.RpaCvInput]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)",
|
|
"Start-Sleep -Milliseconds 40",
|
|
"[RpaCv.RpaCvInput]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)"
|
|
);
|
|
}
|
|
if (action === "paste") {
|
|
lines.push(
|
|
"Add-Type -AssemblyName System.Windows.Forms",
|
|
"Start-Sleep -Milliseconds 220",
|
|
"$saved = $null",
|
|
"try { $saved = [System.Windows.Forms.Clipboard]::GetText() } catch {}",
|
|
"try { [System.Windows.Forms.Clipboard]::SetText(" + JSON.stringify(value) + ") } catch { throw 'clipboard_failed' }",
|
|
"Start-Sleep -Milliseconds 80",
|
|
"[System.Windows.Forms.SendKeys]::SendWait('^a')",
|
|
"Start-Sleep -Milliseconds 60",
|
|
"[System.Windows.Forms.SendKeys]::SendWait('^v')",
|
|
"Start-Sleep -Milliseconds 280",
|
|
"try { if ($saved) { [System.Windows.Forms.Clipboard]::SetText($saved) } else { [System.Windows.Forms.Clipboard]::Clear() } } catch {}"
|
|
);
|
|
}
|
|
lines.push("Write-Output (@{ ok = $true; x = " + x + "; y = " + y + " } | ConvertTo-Json -Compress)");
|
|
return lines.join("\n");
|
|
}
|
|
|
|
/** DIP → 物理像素(SetCursorPos 在 DPI 感知进程里用物理坐标); 无 Electron API 时按原值(缩放 100% 等价) */
|
|
async function toPhysical(x: number, y: number): Promise<{ x: number; y: number }> {
|
|
try {
|
|
const { screen } = await import("electron");
|
|
if (screen && typeof screen.dipToScreenPoint === "function") {
|
|
const p = screen.dipToScreenPoint({ x: Math.round(x), y: Math.round(y) });
|
|
return { x: p.x, y: p.y };
|
|
}
|
|
} catch {}
|
|
return { x: Math.round(x), y: Math.round(y) };
|
|
}
|
|
|
|
async function locateFor(args: Record<string, unknown>, node: { id: string; componentId: string }) {
|
|
const target = cvTarget(args, node.id);
|
|
// 可选搜索区域: 框选生成 {x,y,width,height,imagePath,scale}, 限定 OCR 查找范围
|
|
const region = parseRegionArg(args.searchRegion ?? args.region);
|
|
const located = await locateTextOnScreen(target, node.id, region);
|
|
const offsetX = Number(args.offsetX ?? args.offset_x ?? 0) || 0;
|
|
const offsetY = Number(args.offsetY ?? args.offset_y ?? 0) || 0;
|
|
return { located, x: Math.round(located.cx + offsetX), y: Math.round(located.cy + offsetY) };
|
|
}
|
|
|
|
const getTextGround: CapabilityHandler = async (ctx, node, args) => {
|
|
const target = cvTarget(args, node.id);
|
|
const result = await locateTextOnScreen(target, node.id);
|
|
ctx.log("CV_GetText found: " + result.text, "info", node.id);
|
|
return result.text;
|
|
};
|
|
|
|
const setTextGround: CapabilityHandler = async (ctx, node, args) => {
|
|
const value = String(args.value ?? args.text ?? "");
|
|
if (!value) throw new RuntimeError("validation_error", "CV 文本输入内容为空", { nodeId: node.id, componentId: node.componentId });
|
|
const { x, y } = await locateFor(args, node);
|
|
const phys = await toPhysical(x, y);
|
|
await powershell(coordinateScript(phys.x, phys.y, "paste", value), 30000);
|
|
ctx.log("CV_SetText input at " + phys.x + "," + phys.y, "info", node.id);
|
|
return true;
|
|
};
|
|
|
|
const mouseHoverGround: CapabilityHandler = async (ctx, node, args) => {
|
|
const { x, y } = await locateFor(args, node);
|
|
const phys = await toPhysical(x, y);
|
|
await powershell(coordinateScript(phys.x, phys.y, "hover"), 15000);
|
|
ctx.log("CV_MouseHover hover at " + phys.x + "," + phys.y, "info", node.id);
|
|
return true;
|
|
};
|
|
|
|
const mouseClickGround: CapabilityHandler = async (ctx, node, args) => {
|
|
const { x, y } = await locateFor(args, node);
|
|
const phys = await toPhysical(x, y);
|
|
const clickway = String(args.clickway ?? "0");
|
|
const action = clickway === "1" || /double/i.test(clickway) ? "doubleClick" as const : "click" as const;
|
|
await powershell(coordinateScript(phys.x, phys.y, action), 15000);
|
|
ctx.log("CV_MouseClick " + action + " at " + phys.x + "," + phys.y, "info", node.id);
|
|
return true;
|
|
};
|
|
|
|
register("CV_GetText", getTextGround);
|
|
register("CV_SetText", setTextGround);
|
|
register("CV_MouseHover", mouseHoverGround);
|
|
register("CV_MouseClick", mouseClickGround);
|