阶段1-rpa-core建包: engine+31个纯Node能力迁入packages/rpa-core, 领域类型单一源化, createEngine工厂收拢模块级状态
- engine/(registry/types/runtime/expression)物理迁入包, 解耦shared/types(域类型归包, 宿主type-only re-export) - 31个纯Node能力+27个测试迁入包; 宿主留守Electron/浏览器/AI依赖能力, index.ts经@rpa/core/capabilities拉起 - 新增createEngine()/defaultEngine: excel缓存/Excel句柄/db连接/FTP客户端/OCR实例/拟真设置挂引擎实例 - remaining-core子流程依赖改为host-bridge注入, 包内不再import宿主flows - 宿主main/index.ts启动时注入子流程源; electron-vite/vitest/tsconfig配@rpa/core源码别名 - 验证: typecheck绿, 快照909个id diff为零, 332测试过(2个桌面冒烟因会话无交互SetForegroundWindow被拒, 环境阻塞)
This commit is contained in:
yu authored and trae-solo-agent committed
1 parent
4139400dbd
commit
caa8e19332
111 files changed
+581
-337
No files matched your search
@@ -0,0 +1,557 @@
|
||||
import path from "node:path";
|
||||
import { promises as fs } from "node:fs";
|
||||
import { PNG } from "pngjs";
|
||||
import { register } from "../engine/registry";
|
||||
import { RuntimeError } from "../engine/types";
|
||||
import { powershell, psString } from "./desktop";
|
||||
|
||||
function text(args: Record<string, unknown>, keys: string[], fallback = ""): string {
|
||||
for (const key of keys) if (args[key] !== undefined && args[key] !== null && args[key] !== "") return String(args[key]);
|
||||
return fallback;
|
||||
}
|
||||
|
||||
/** 全虚拟桌面（所有显示器拼接）物理分辨率截图。
|
||||
* desktopCapturer thumbnailSize 乘 scaleFactor 与「屏幕框选」模板（物理像素裁剪）尺度一致；
|
||||
* 逐屏截图后按各自物理原点拼到一张画布上——目标在副屏也能搜到。
|
||||
* 返回 originX/originY = 拼接画布左上角对应的屏幕物理坐标（副屏在主屏左侧时为负），用于结果坐标平移。 */
|
||||
async function captureVirtualScreenPng(): Promise<{ png: PNG; originX: number; originY: number }> {
|
||||
const { desktopCapturer, screen } = await import("electron");
|
||||
const displays = screen.getAllDisplays();
|
||||
if (displays.length === 0) throw new RuntimeError("desktop_error", "无法获取屏幕信息");
|
||||
// 每屏按自己的物理分辨率截图（display_id 匹配 → 缩略图尺寸匹配 → 按物理原点排序对位）
|
||||
const captures: { display: Electron.Display; origin: { x: number; y: number }; png: PNG }[] = [];
|
||||
for (const display of displays) {
|
||||
const scale = display.scaleFactor || 1;
|
||||
const wantW = Math.round(display.bounds.width * scale);
|
||||
const wantH = Math.round(display.bounds.height * scale);
|
||||
const sources = await desktopCapturer.getSources({
|
||||
types: ["screen"],
|
||||
thumbnailSize: { width: wantW, height: wantH }
|
||||
});
|
||||
const source = sources.find(item => item.display_id === String(display.id))
|
||||
|| sources.find(item => item.thumbnail.getSize().width === wantW)
|
||||
|| sources[0];
|
||||
if (!source || source.thumbnail.isEmpty()) {
|
||||
throw new RuntimeError("desktop_error", "无法获取屏幕截图");
|
||||
}
|
||||
// 显示器 DIP 原点的物理坐标 = 该屏物理原点（屏内线性映射: 物理 = (DIP-原点)*缩放 + 物理原点）
|
||||
const origin = screen.dipToScreenPoint({ x: display.bounds.x, y: display.bounds.y });
|
||||
captures.push({ display, origin, png: PNG.sync.read(source.thumbnail.toPNG()) });
|
||||
}
|
||||
const originX = Math.min(...captures.map(c => c.origin.x));
|
||||
const originY = Math.min(...captures.map(c => c.origin.y));
|
||||
const endX = Math.max(...captures.map(c => c.origin.x + c.png.width));
|
||||
const endY = Math.max(...captures.map(c => c.origin.y + c.png.height));
|
||||
const width = endX - originX;
|
||||
const height = endY - originY;
|
||||
const { createCanvas } = await import("@napi-rs/canvas");
|
||||
const canvas = createCanvas(width, height);
|
||||
const context = canvas.getContext("2d");
|
||||
for (const capture of captures) {
|
||||
const sourceCanvas = createCanvas(capture.png.width, capture.png.height);
|
||||
const sourceContext = sourceCanvas.getContext("2d");
|
||||
const imageData = sourceContext.createImageData(capture.png.width, capture.png.height);
|
||||
imageData.data.set(capture.png.data);
|
||||
sourceContext.putImageData(imageData, 0, 0);
|
||||
context.drawImage(sourceCanvas, capture.origin.x - originX, capture.origin.y - originY);
|
||||
}
|
||||
const stitched = context.getImageData(0, 0, width, height);
|
||||
const png = new PNG({ width, height });
|
||||
png.data = Buffer.from(stitched.data);
|
||||
return { png, originX, originY };
|
||||
}
|
||||
|
||||
/** DIP 区域 → 物理像素区域：左上角经 dipToScreenPoint 换算（天然支持副屏/负坐标），宽高乘所在屏缩放比 */
|
||||
async function dipRegionToPhysical(region: { x: number; y: number; width: number; height: number }): Promise<{ x: number; y: number; width: number; height: number }> {
|
||||
const { screen } = await import("electron");
|
||||
const topLeft = screen.dipToScreenPoint({ x: Math.round(region.x), y: Math.round(region.y) });
|
||||
const display = screen.getDisplayMatching({
|
||||
x: region.x, y: region.y,
|
||||
width: Math.max(1, Math.round(region.width)), height: Math.max(1, Math.round(region.height))
|
||||
});
|
||||
const scale = display.scaleFactor || 1;
|
||||
return {
|
||||
x: topLeft.x,
|
||||
y: topLeft.y,
|
||||
width: Math.max(1, Math.round(region.width * scale)),
|
||||
height: Math.max(1, Math.round(region.height * scale))
|
||||
};
|
||||
}
|
||||
|
||||
/** 按魔数判断是否 PNG（89 50 4E 47） */
|
||||
function isPngBuffer(buffer: Buffer): boolean {
|
||||
return buffer.length > 8 && buffer[0] === 0x89 && buffer[1] === 0x50 && buffer[2] === 0x4e && buffer[3] === 0x47;
|
||||
}
|
||||
|
||||
/** 解码用户提供的图片文件为 pngjs PNG。pngjs 只认 PNG, 模板/输入图常是 JPG——
|
||||
* 非 PNG 时走 @napi-rs/canvas 解码(JPG/BMP/WebP 等), 否则报 "unrecognised content at end of stream" */
|
||||
async function decodeImageFile(filePath: string): Promise<PNG> {
|
||||
let buffer: Buffer;
|
||||
try {
|
||||
buffer = await fs.readFile(filePath);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException)?.code === "ENOENT") {
|
||||
throw new RuntimeError("validation_error", `找不到图片文件：${filePath}`, {
|
||||
recommendation: "确认路径正确，或直接点参数旁的「屏幕框选」截取一张"
|
||||
});
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (isPngBuffer(buffer)) return PNG.sync.read(buffer);
|
||||
const { loadImage, createCanvas } = await import("@napi-rs/canvas");
|
||||
const image = await loadImage(buffer);
|
||||
if (!image.width || !image.height) {
|
||||
throw new RuntimeError("validation_error", "图片解码失败", { detail: filePath });
|
||||
}
|
||||
const canvas = createCanvas(image.width, image.height);
|
||||
const context = canvas.getContext("2d");
|
||||
context.drawImage(image, 0, 0);
|
||||
const imageData = context.getImageData(0, 0, image.width, image.height);
|
||||
const png = new PNG({ width: image.width, height: image.height });
|
||||
png.data = Buffer.from(imageData.data);
|
||||
return png;
|
||||
}
|
||||
|
||||
/** 解析区域 JSON：对象 {x,y,width,height}（兼容 left/top、x1y1x2y2 角点）或数组 [x,y,width,height]，并夹取到图片边界内 */
|
||||
export function parseRegion(raw: string, image: PNG): { x: number; y: number; width: number; height: number } {
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(raw);
|
||||
} catch {
|
||||
throw new RuntimeError("validation_error", "区域 JSON 无法解析", { detail: raw.slice(0, 200) });
|
||||
}
|
||||
const numberOf = (value: unknown): number | null => {
|
||||
const n = Number(value);
|
||||
return Number.isFinite(n) ? n : null;
|
||||
};
|
||||
if (Array.isArray(parsed) && parsed.length === 4) {
|
||||
const [x, y, width, height] = parsed.map(numberOf);
|
||||
if (x !== null && y !== null && width !== null && height !== null) {
|
||||
return clampRegion({ x, y, width, height }, image);
|
||||
}
|
||||
}
|
||||
if (parsed && typeof parsed === "object") {
|
||||
const record = parsed as Record<string, unknown>;
|
||||
const x = numberOf(record.x ?? record.left);
|
||||
const y = numberOf(record.y ?? record.top);
|
||||
const width = numberOf(record.width);
|
||||
const height = numberOf(record.height);
|
||||
if (x !== null && y !== null && width !== null && height !== null) {
|
||||
return clampRegion({ x, y, width, height }, image);
|
||||
}
|
||||
const x2 = numberOf(record.x2 ?? record.right);
|
||||
const y2 = numberOf(record.y2 ?? record.bottom);
|
||||
if (x !== null && y !== null && x2 !== null && y2 !== null) {
|
||||
return clampRegion({ x, y, width: x2 - x, height: y2 - y }, image);
|
||||
}
|
||||
}
|
||||
throw new RuntimeError("validation_error", "区域 JSON 不符合要求", {
|
||||
detail: raw.slice(0, 200),
|
||||
recommendation: '使用 {"x":0,"y":0,"width":100,"height":80} 或 [x, y, width, height]'
|
||||
});
|
||||
}
|
||||
|
||||
export function clampRegion(region: { x: number; y: number; width: number; height: number }, image: PNG): { x: number; y: number; width: number; height: number } {
|
||||
const x = Math.max(0, Math.min(Math.round(region.x), image.width - 1));
|
||||
const y = Math.max(0, Math.min(Math.round(region.y), image.height - 1));
|
||||
const width = Math.max(1, Math.min(Math.round(region.width), image.width - x));
|
||||
const height = Math.max(1, Math.min(Math.round(region.height), image.height - y));
|
||||
return { x, y, width, height };
|
||||
}
|
||||
|
||||
export function cropPng(source: PNG, region: { x: number; y: number; width: number; height: number }): PNG {
|
||||
const cropped = new PNG({ width: region.width, height: region.height });
|
||||
PNG.bitblt(source, cropped, region.x, region.y, region.width, region.height, 0, 0);
|
||||
return cropped;
|
||||
}
|
||||
|
||||
/** 目标路径容错：填的是文件夹（已存在或以分隔符结尾）时，自动往里存默认文件名 */
|
||||
async function resolveOutputPath(outputPath: string, defaultName: string): Promise<string> {
|
||||
if (/[\\/]$/.test(outputPath)) return path.join(outputPath, defaultName);
|
||||
try {
|
||||
const stat = await fs.stat(outputPath);
|
||||
if (stat.isDirectory()) return path.join(outputPath, defaultName);
|
||||
} catch {
|
||||
// 路径不存在说明是要新建的文件，原样返回
|
||||
}
|
||||
return outputPath;
|
||||
}
|
||||
|
||||
register("Image_Screenshot", async (ctx, node, args) => {
|
||||
const folder = text(args, ["outImagePath", "folder"]);
|
||||
const name = text(args, ["outImageName", "name"], `screenshot_${Date.now()}.png`);
|
||||
const imageArg = text(args, ["image_path"]);
|
||||
const source = text(args, ["pictureSource"], "1");
|
||||
const range = text(args, ["shotRange"], "1");
|
||||
|
||||
let image: PNG;
|
||||
let output: string;
|
||||
let screenOrigin: { x: number; y: number } | null = null;
|
||||
if (source === "2") {
|
||||
// 指定图片文件来源：image_path 为输入图片；输出走 输出目录+文件名，两者均未配置时直接返回源文件
|
||||
if (!imageArg.trim()) {
|
||||
throw new RuntimeError("validation_error", "图像来源为「指定图片文件」时，截图保存路径需填写源图片路径", {
|
||||
recommendation: "在截图保存路径选择一张已存在的图片，或将图像来源改回「当前屏幕」"
|
||||
});
|
||||
}
|
||||
const sourceFile = path.resolve(imageArg);
|
||||
image = await decodeImageFile(sourceFile);
|
||||
output = folder ? path.resolve(path.join(folder, name)) : sourceFile;
|
||||
} else {
|
||||
// 当前屏幕来源：image_path 即保存路径（文件夹则自动追加默认文件名），缺省用 输出目录+文件名；再缺省落流程证据目录（路径可预期）
|
||||
// 截取整个虚拟桌面（含副屏），物理分辨率
|
||||
const shot = await captureVirtualScreenPng();
|
||||
image = shot.png;
|
||||
screenOrigin = { x: shot.originX, y: shot.originY };
|
||||
output = await resolveOutputPath(
|
||||
path.resolve(imageArg || (folder ? path.join(folder, name) : path.join(ctx.evidenceDir, name))),
|
||||
name
|
||||
);
|
||||
}
|
||||
|
||||
if (range === "2") {
|
||||
const regionRaw = text(args, ["position", "action", "position_img"]);
|
||||
if (!regionRaw.trim()) {
|
||||
throw new RuntimeError("validation_error", "截图范围为「框选区域」时，截图区域需填写区域 JSON", {
|
||||
recommendation: '在「截图区域」点屏幕框选，或填写 {"x":0,"y":0,"width":100,"height":80}，或将截图范围改回「整个屏幕」'
|
||||
});
|
||||
}
|
||||
if (screenOrigin) {
|
||||
// 屏幕来源: 区域 JSON 是 DIP 坐标（支持副屏），换算物理坐标后减去拼接画布原点再裁剪
|
||||
const phys = await dipRegionToPhysical(parseRegion(regionRaw, image));
|
||||
image = cropPng(image, clampRegion({
|
||||
x: phys.x - screenOrigin.x,
|
||||
y: phys.y - screenOrigin.y,
|
||||
width: phys.width,
|
||||
height: phys.height
|
||||
}, image));
|
||||
} else {
|
||||
// 图片文件来源: 区域按该图片自身像素坐标
|
||||
image = cropPng(image, parseRegion(regionRaw, image));
|
||||
}
|
||||
}
|
||||
|
||||
await fs.mkdir(path.dirname(output), { recursive: true });
|
||||
await fs.writeFile(output, PNG.sync.write(image));
|
||||
return output;
|
||||
});
|
||||
|
||||
interface MatchResult { x: number; y: number; width: number; height: number; score: number; }
|
||||
|
||||
interface WindowRect { x: number; y: number; width: number; height: number }
|
||||
|
||||
/** 按标题（精确→包含）或进程名查找窗口，前置并返回窗口矩形（物理屏幕坐标） */
|
||||
async function focusWindowRect(windowObj: string): Promise<WindowRect> {
|
||||
const quoted = psString(windowObj);
|
||||
const stdout = await powershell(`Add-Type @'
|
||||
using System; using System.Runtime.InteropServices;
|
||||
public static class WinRect {
|
||||
[DllImport("user32.dll")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value);
|
||||
[DllImport("user32.dll")] public static extern IntPtr FindWindow(string cls, string title);
|
||||
[DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr h);
|
||||
[DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr h, int cmd);
|
||||
[DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr h, out RECT r);
|
||||
public struct RECT { public int Left; public int Top; public int Right; public int Bottom; }
|
||||
}
|
||||
'@;
|
||||
# 不声明 DPI 感知时 GetWindowRect 返回虚拟化坐标, 与物理分辨率截图对不上
|
||||
try { [void][WinRect]::SetProcessDpiAwarenessContext([IntPtr](-4)) } catch {};
|
||||
$h = [WinRect]::FindWindow($null, ${quoted});
|
||||
if ($h -eq [IntPtr]::Zero) { $p = Get-Process | Where-Object { $_.MainWindowTitle -and ($_.MainWindowTitle -eq ${quoted} -or $_.MainWindowTitle.Contains(${quoted})) } | Select-Object -First 1; if ($p) { $h = $p.MainWindowHandle } }
|
||||
if ($h -eq [IntPtr]::Zero) { Write-Output 'NOT_FOUND'; exit }
|
||||
[WinRect]::ShowWindow($h, 9) | Out-Null;
|
||||
[WinRect]::SetForegroundWindow($h) | Out-Null;
|
||||
Start-Sleep -Milliseconds 300;
|
||||
$r = New-Object WinRect+RECT;
|
||||
[WinRect]::GetWindowRect($h, [ref]$r) | Out-Null;
|
||||
Write-Output ("{0},{1},{2},{3}" -f $r.Left, $r.Top, $r.Right, $r.Bottom)`);
|
||||
const line = stdout.trim().split(/\r?\n/).pop() ?? "";
|
||||
if (line === "NOT_FOUND") {
|
||||
throw new RuntimeError("not_found", `未找到窗口「${windowObj}」`, { recommendation: "确认窗口已打开，窗口对象可填完整标题、标题包含关键字或进程名" });
|
||||
}
|
||||
const parts = line.split(",").map(Number);
|
||||
if (parts.length !== 4 || parts.some((v) => !Number.isFinite(v))) {
|
||||
throw new RuntimeError("desktop_error", "窗口矩形获取失败", { detail: line.slice(0, 120) });
|
||||
}
|
||||
const [left, top, right, bottom] = parts;
|
||||
return { x: left, y: top, width: Math.max(1, right - left), height: Math.max(1, bottom - top) };
|
||||
}
|
||||
|
||||
/** 单通道噪声死区：JPG 压缩伪影/抗锯齿带来 ±10 级内的像素波动，不应计入差异 */
|
||||
const NOISE_FLOOR = 10;
|
||||
|
||||
function pixelDifference(screen: PNG, template: PNG, offsetX: number, offsetY: number, sampleStep: number, cutoff: number): number {
|
||||
let difference = 0;
|
||||
let samples = 0;
|
||||
for (let y = 0; y < template.height; y += sampleStep) {
|
||||
for (let x = 0; x < template.width; x += sampleStep) {
|
||||
const sourceIndex = ((offsetY + y) * screen.width + offsetX + x) * 4;
|
||||
const templateIndex = (y * template.width + x) * 4;
|
||||
for (let c = 0; c < 3; c++) {
|
||||
const d = Math.abs(screen.data[sourceIndex + c] - template.data[templateIndex + c]);
|
||||
difference += d <= NOISE_FLOOR ? 0 : d - NOISE_FLOOR;
|
||||
}
|
||||
samples += 3;
|
||||
// cutoff 是 0~1 的归一化差异上限; difference 是通道差累加值(0~255 每通道),
|
||||
// 提前退出必须按 255 折算——之前漏乘导致几乎所有位置在第一个像素就被剪掉, 得分全乱
|
||||
if (difference > cutoff * samples * 255) return difference / (samples * 255);
|
||||
}
|
||||
}
|
||||
return samples ? difference / (samples * 255) : 1;
|
||||
}
|
||||
|
||||
/** 在屏幕图里扫描模板的最佳位置（粗扫网格 + 精化） */
|
||||
function scanBest(screen: PNG, template: PNG, maximumDifference: number): MatchResult | null {
|
||||
const sampleStep = Math.max(1, Math.floor(Math.min(template.width, template.height) / 18));
|
||||
const scanStep = Math.max(1, Math.floor(Math.min(template.width, template.height) / 12));
|
||||
let best: MatchResult | null = null;
|
||||
for (let y = 0; y <= screen.height - template.height; y += scanStep) {
|
||||
for (let x = 0; x <= screen.width - template.width; x += scanStep) {
|
||||
// 尚无 best 时不剪枝(完整评估): 若初始就按 maximumDifference 剪枝,
|
||||
// 均匀背景位置在少数采样后返回"刚好压线"的部分均值抢占 best,
|
||||
// 而真位置(常不在网格上)附近的网格点因部分重叠被剪, 精化永远够不到——
|
||||
// 相似度 0.95 时实测 score=1.0 的真位置被判"未达到阈值"
|
||||
const cutoff = best ? 1 - best.score : 1;
|
||||
const difference = pixelDifference(screen, template, x, y, sampleStep, cutoff);
|
||||
const score = 1 - difference;
|
||||
if (!best || score > best.score) best = { x, y, width: template.width, height: template.height, score };
|
||||
}
|
||||
}
|
||||
if (!best) return null;
|
||||
// 注意: 阈值判定必须放在精化之后——粗扫网格(步长=模板短边/12)通常够不到精确位置,
|
||||
// 粗扫 best 常差在阈值边缘, 精化后才能提到满分; 先判阈值会把真命中误杀
|
||||
const refine = Math.max(1, scanStep);
|
||||
const minX = Math.max(0, best.x - refine);
|
||||
const maxX = Math.min(screen.width - template.width, best.x + refine);
|
||||
const minY = Math.max(0, best.y - refine);
|
||||
const maxY = Math.min(screen.height - template.height, best.y + refine);
|
||||
for (let y = minY; y <= maxY; y += 1) {
|
||||
for (let x = minX; x <= maxX; x += 1) {
|
||||
// 精化区域只有几百个位置, 完整评估避免同样的剪枝偏置
|
||||
const difference = pixelDifference(screen, template, x, y, sampleStep, 1);
|
||||
const score = 1 - difference;
|
||||
if (score > best.score) best = { x, y, width: template.width, height: template.height, score };
|
||||
}
|
||||
}
|
||||
// 阈值判定: 精化后的最终得分才与 maximumDifference 比较
|
||||
if (1 - best.score > maximumDifference) return null;
|
||||
return best;
|
||||
}
|
||||
|
||||
/** 用 canvas 双线性缩放模板（匹配尺度兜底用） */
|
||||
async function resizeTemplate(template: PNG, scale: number): Promise<PNG> {
|
||||
const { createCanvas } = await import("@napi-rs/canvas");
|
||||
const width = Math.max(1, Math.round(template.width * scale));
|
||||
const height = Math.max(1, Math.round(template.height * scale));
|
||||
const source = createCanvas(template.width, template.height);
|
||||
const sourceContext = source.getContext("2d");
|
||||
const imageData = sourceContext.createImageData(template.width, template.height);
|
||||
imageData.data.set(template.data);
|
||||
sourceContext.putImageData(imageData, 0, 0);
|
||||
const target = createCanvas(width, height);
|
||||
const targetContext = target.getContext("2d");
|
||||
targetContext.imageSmoothingEnabled = true;
|
||||
targetContext.imageSmoothingQuality = "high";
|
||||
targetContext.drawImage(source, 0, 0, width, height);
|
||||
const scaled = targetContext.getImageData(0, 0, width, height);
|
||||
const png = new PNG({ width, height });
|
||||
png.data = Buffer.from(scaled.data);
|
||||
return png;
|
||||
}
|
||||
|
||||
/**
|
||||
* 模板匹配。extraScales 提供尺度兜底：截图与模板的获取渠道分辨率不一致时
|
||||
* （如旧版 DIP 截图存下的模板 vs 现在的物理分辨率截图），按给定缩放比重试。
|
||||
*/
|
||||
export async function findImage(templatePath: string, screenshotPath: string, accuracy: number, extraScales: number[] = []): Promise<MatchResult | null> {
|
||||
const [screenBuffer, templateBuffer] = await Promise.all([fs.readFile(screenshotPath), fs.readFile(templatePath)]);
|
||||
const screen = PNG.sync.read(screenBuffer);
|
||||
// 模板图可能是 JPG 等非 PNG 格式, 按魔数选择解码器
|
||||
const template = isPngBuffer(templateBuffer) ? PNG.sync.read(templateBuffer) : await decodeImageFile(templatePath);
|
||||
if (template.width > screen.width || template.height > screen.height) {
|
||||
throw new RuntimeError("validation_error", `模板图片（${template.width}×${template.height}）比截图画面（${screen.width}×${screen.height}）还大，无法查找`, {
|
||||
recommendation: "模板应是屏幕上要寻找的小图案（按钮、图标、文字块），不要用整屏大图"
|
||||
});
|
||||
}
|
||||
const maximumDifference = Math.max(0.01, Math.min(0.95, accuracy || 0.2));
|
||||
const best = scanBest(screen, template, maximumDifference);
|
||||
if (best) return best;
|
||||
for (const scale of extraScales) {
|
||||
if (!Number.isFinite(scale) || scale === 1) continue;
|
||||
const scaled = await resizeTemplate(template, scale);
|
||||
if (scaled.width > screen.width || scaled.height > screen.height) continue;
|
||||
const scaledBest = scanBest(screen, scaled, maximumDifference);
|
||||
if (scaledBest) return scaledBest;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function locate(args: Record<string, unknown>, evidenceDir: string, nodeId: string): Promise<MatchResult> {
|
||||
const rawTemplate = text(args, ["image", "image_path", "template"]);
|
||||
if (!rawTemplate.trim()) throw new RuntimeError("validation_error", "模板图片路径为空", { nodeId });
|
||||
const template = path.resolve(rawTemplate);
|
||||
|
||||
// 查找范围：1 整个虚拟桌面（默认，含副屏）/ 2 指定窗口 / 3 指定区域；
|
||||
// 截图统一为物理分辨率拼接画布, 结果坐标加回画布原点即屏幕物理坐标（可直接用于点击）
|
||||
const rangeMode = text(args, ["find_range"], "1");
|
||||
const screenshot = path.join(evidenceDir, `${nodeId}_screen.png`);
|
||||
const shot = await captureVirtualScreenPng();
|
||||
let offsetX = 0;
|
||||
let offsetY = 0;
|
||||
if (rangeMode === "2") {
|
||||
const windowObj = text(args, ["window_obj"]);
|
||||
if (!windowObj.trim()) throw new RuntimeError("validation_error", "查找范围为「指定窗口」时，窗口对象不能为空", { nodeId, recommendation: "填写窗口标题（或标题关键字、进程名）" });
|
||||
const rect = await focusWindowRect(windowObj);
|
||||
const cropRect = clampRegion({ x: rect.x - shot.originX, y: rect.y - shot.originY, width: rect.width, height: rect.height }, shot.png);
|
||||
await fs.writeFile(screenshot, PNG.sync.write(cropPng(shot.png, cropRect)));
|
||||
offsetX = rect.x;
|
||||
offsetY = rect.y;
|
||||
} else if (rangeMode === "3" || rangeMode === "4") {
|
||||
const regionRaw = text(args, ["shot_position", "action", "position"]);
|
||||
if (!regionRaw.trim()) throw new RuntimeError("validation_error", "查找范围为「指定区域」时，需要填写区域 JSON", { nodeId, recommendation: '在 shot_position 填写 {"x":0,"y":0,"width":100,"height":80} 或 [x, y, width, height]' });
|
||||
// 区域 JSON 是 DIP 坐标（框选工具按 DIP 返回, 支持副屏负坐标），换算物理坐标后减画布原点裁剪
|
||||
const phys = await dipRegionToPhysical(parseRegion(regionRaw, shot.png));
|
||||
const cropRect = clampRegion({ x: phys.x - shot.originX, y: phys.y - shot.originY, width: phys.width, height: phys.height }, shot.png);
|
||||
await fs.writeFile(screenshot, PNG.sync.write(cropPng(shot.png, cropRect)));
|
||||
offsetX = phys.x;
|
||||
offsetY = phys.y;
|
||||
} else {
|
||||
await fs.writeFile(screenshot, PNG.sync.write(shot.png));
|
||||
// 拼接画布坐标 → 屏幕物理坐标必须加回画布原点(副屏在主屏左侧/上方时原点为负);
|
||||
// 漏加会导致点击偏移(实测副屏上移 177px 时点击正好偏下 177px)
|
||||
offsetX = shot.originX;
|
||||
offsetY = shot.originY;
|
||||
}
|
||||
|
||||
// accuracy 字段语义为「相似度」(0~1, 越大越严格, 0.9=至少90%像素一致)——与界面文案一致。
|
||||
// 引擎内部用最大允许差异 = 1 - 相似度。此前引擎直接把该值当差异阈值,
|
||||
// 用户填 0.9 反而放宽到"接受90%像素差异", 导致到处误命中。
|
||||
const rawAccuracy = Number(text(args, ["accuracy"], "0.8"));
|
||||
const similarity = Math.max(0.05, Math.min(1, Number.isFinite(rawAccuracy) && rawAccuracy > 0 ? rawAccuracy : 0.8));
|
||||
// 原生 1:1 匹配失败时, 按缩放比对模板放大/缩小重试——兼容旧版 DIP 截图存的模板与任意来源的图片
|
||||
const scale = (await import("electron")).screen.getPrimaryDisplay().scaleFactor || 1;
|
||||
const result = await findImage(template, screenshot, 1 - similarity, scale !== 1 ? [scale, 1 / scale] : []);
|
||||
if (!result) throw new RuntimeError("not_found", `屏幕中未找到目标图像（要求相似度 ${(similarity * 100).toFixed(0)}%，未达到阈值）`, { nodeId, recommendation: "重新框选模板（框小一点、只框目标图案），或适当调低相似度（如 0.8）；目标在副屏时确认其清晰可见" });
|
||||
return { ...result, x: result.x + offsetX, y: result.y + offsetY };
|
||||
}
|
||||
|
||||
register(["Image_Find", "Image_Find_V1"], async (ctx, node, args) => locate(args, ctx.evidenceDir, node.id));
|
||||
// 图像位置: 与查找图像同义(历史重复组件), 同管线返回坐标
|
||||
for (const id of ["Image_Location", "Image_Location_V1"]) register(id, async (ctx, node, args) => locate(args, ctx.evidenceDir, node.id));
|
||||
// 判断图像: 找到返回 true / 找不到返回 false(供条件分支), 仅有 not_found 转为 false;
|
||||
// 模板路径为空/文件不存在等配置错误仍然抛出, 避免静默 false 难排查。
|
||||
// timeout>0 时在窗口内每 300ms 重试(等待图像出现再判断的场景)
|
||||
for (const id of ["Image_Exists", "Image_Exists_V1"]) register(id, async (ctx, node, args) => {
|
||||
const rawTimeout = Number(text(args, ["timeout", "time_out"], "0"));
|
||||
const deadline = Date.now() + (Number.isFinite(rawTimeout) && rawTimeout > 0 ? rawTimeout : 0) * 1000;
|
||||
for (;;) {
|
||||
try {
|
||||
await locate(args, ctx.evidenceDir, node.id);
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (error instanceof RuntimeError && error.code === "not_found") {
|
||||
if (Date.now() >= deadline) return false;
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, 300));
|
||||
}
|
||||
});
|
||||
// 等待图像: 在超时窗口内每 300ms 重试一次查找, 命中即返回坐标, 超时仍无则抛 not_found
|
||||
for (const id of ["Image_Wait", "Image_Wait_V1"]) register(id, async (ctx, node, args) => {
|
||||
const rawTimeout = Number(text(args, ["timeout", "time_out"], "20"));
|
||||
const timeout = Number.isFinite(rawTimeout) && rawTimeout >= 0 ? rawTimeout : 20;
|
||||
const deadline = Date.now() + timeout * 1000;
|
||||
for (;;) {
|
||||
try {
|
||||
return await locate(args, ctx.evidenceDir, node.id);
|
||||
} catch (error) {
|
||||
if (!(error instanceof RuntimeError) || error.code !== "not_found" || Date.now() >= deadline) throw error;
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, 300));
|
||||
}
|
||||
});
|
||||
// 移动鼠标到图像: 定位后把光标移到匹配区域中心;
|
||||
// SetCursorPos 传物理坐标, 脚本必须声明 per-monitor DPI 感知(与 injectClick 同理), 否则被系统虚拟化导致偏移
|
||||
for (const id of ["Image_Move", "Image_Move_V1", "Image_Move_V2"]) register(id, async (ctx, node, args) => {
|
||||
const hit = await locate(args, ctx.evidenceDir, node.id);
|
||||
const point = clickPoint(hit, text(args, ["position"], "center"));
|
||||
const x = point.x;
|
||||
const y = point.y;
|
||||
await powershell("Add-Type -MemberDefinition '[DllImport(\"user32.dll\")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value); [DllImport(\"user32.dll\")] public static extern bool SetCursorPos(int X, int Y);' -Name RpaImageMoveInput -Namespace RpaImageMove | Out-Null; try { [void][RpaImageMove.RpaImageMoveInput]::SetProcessDpiAwarenessContext([IntPtr](-4)) } catch {}; [RpaImageMove.RpaImageMoveInput]::SetCursorPos(" + x + "," + y + ")");
|
||||
return { ...hit, x, y };
|
||||
});
|
||||
/** 点击位置锚点（相对匹配区域的比例坐标），支持九宫格取值 */
|
||||
const POSITION_ANCHORS: Record<string, [number, number]> = {
|
||||
center: [0.5, 0.5],
|
||||
leftTop: [0, 0], top: [0.5, 0], centerTop: [0.5, 0], rightTop: [1, 0],
|
||||
left: [0, 0.5], centerLeft: [0, 0.5], right: [1, 0.5], centerRight: [1, 0.5],
|
||||
leftBottom: [0, 1], bottom: [0.5, 1], centerBottom: [0.5, 1], rightBottom: [1, 1]
|
||||
};
|
||||
|
||||
export function clickPoint(result: MatchResult, position: string): { x: number; y: number } {
|
||||
const raw = position.trim();
|
||||
if (raw && raw.toLowerCase() !== "center") {
|
||||
if (raw.startsWith("{")) {
|
||||
try {
|
||||
const offset = JSON.parse(raw) as { x?: unknown; y?: unknown };
|
||||
const dx = Number(offset.x);
|
||||
const dy = Number(offset.y);
|
||||
if (Number.isFinite(dx) && Number.isFinite(dy)) {
|
||||
return { x: Math.round(result.x + dx), y: Math.round(result.y + dy) };
|
||||
}
|
||||
} catch {
|
||||
// 非法 JSON 落回中心
|
||||
}
|
||||
}
|
||||
const anchor = POSITION_ANCHORS[raw] ?? POSITION_ANCHORS[raw.toLowerCase()] ?? POSITION_ANCHORS[raw.replace(/-([a-z])/g, (_: string, char: string) => char.toUpperCase())];
|
||||
if (anchor) {
|
||||
return { x: Math.round(result.x + result.width * anchor[0]), y: Math.round(result.y + result.height * anchor[1]) };
|
||||
}
|
||||
}
|
||||
return { x: Math.round(result.x + result.width / 2), y: Math.round(result.y + result.height / 2) };
|
||||
}
|
||||
|
||||
const CLICK_ACTIONS: Record<string, string> = {
|
||||
left: "[Mouse]::mouse_event(2,0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 30; [Mouse]::mouse_event(4,0,0,0,[UIntPtr]::Zero)",
|
||||
right: "[Mouse]::mouse_event(8,0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 30; [Mouse]::mouse_event(16,0,0,0,[UIntPtr]::Zero)",
|
||||
middle: "[Mouse]::mouse_event(32,0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 30; [Mouse]::mouse_event(64,0,0,0,[UIntPtr]::Zero)",
|
||||
double: "[Mouse]::mouse_event(2,0,0,0,[UIntPtr]::Zero); [Mouse]::mouse_event(4,0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 60; [Mouse]::mouse_event(2,0,0,0,[UIntPtr]::Zero); [Mouse]::mouse_event(4,0,0,0,[UIntPtr]::Zero)"
|
||||
};
|
||||
|
||||
async function injectClick(x: number, y: number, clickKey: string, showTrack: boolean): Promise<void> {
|
||||
const action = CLICK_ACTIONS[clickKey] ?? CLICK_ACTIONS.left;
|
||||
// show_track=1 时从当前光标位置平滑移动过去（20 步插值），否则直接跳转
|
||||
const move = showTrack
|
||||
? `for ($i = 1; $i -le 20; $i++) { [Mouse]::SetCursorPos([int]($p.X + (${x} - $p.X) * $i / 20), [int]($p.Y + (${y} - $p.Y) * $i / 20)); Start-Sleep -Milliseconds 12 }`
|
||||
: `[Mouse]::SetCursorPos(${x}, ${y})`;
|
||||
await powershell(`Add-Type @'
|
||||
using System; using System.Runtime.InteropServices;
|
||||
public static class Mouse {
|
||||
[DllImport("user32.dll")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value);
|
||||
[DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y);
|
||||
[DllImport("user32.dll")] public static extern bool GetCursorPos(out POINT p);
|
||||
[DllImport("user32.dll")] public static extern void mouse_event(uint f, uint x, uint y, uint d, UIntPtr e);
|
||||
public struct POINT { public int X; public int Y; }
|
||||
}
|
||||
'@;
|
||||
# 传入的是物理像素坐标(与物理分辨率截图匹配所得), 必须声明 DPI 感知——
|
||||
# 否则进程被 DPI 虚拟化, 坐标再被系统放大一次, 点击成比例偏右下
|
||||
try { [void][Mouse]::SetProcessDpiAwarenessContext([IntPtr](-4)) } catch {};
|
||||
$p = New-Object Mouse+POINT;
|
||||
[Mouse]::GetCursorPos([ref]$p) | Out-Null;
|
||||
${move};
|
||||
Start-Sleep -Milliseconds 60;
|
||||
${action}`);
|
||||
}
|
||||
|
||||
register(["Image_Click", "Image_Click_V1", "Image_Click_V2", "Image_Click_V3"], async (ctx, node, args) => {
|
||||
const result = await locate(args, ctx.evidenceDir, node.id);
|
||||
const point = clickPoint(result, text(args, ["position"], "center"));
|
||||
const clickKey = text(args, ["click_key"], "left");
|
||||
const showTrack = String(args.show_track ?? "0") === "1" || args.show_track === true;
|
||||
await injectClick(point.x, point.y, clickKey, showTrack);
|
||||
ctx.log(`已${clickKey === "right" ? "右键" : clickKey === "double" ? "双击" : "点击"}图像 (${point.x}, ${point.y})，匹配得分 ${result.score.toFixed(3)}`, "info", node.id);
|
||||
return { ...result, click: point };
|
||||
});
|
||||
Reference in new issue
Block a user