fix: 图像与OCR 检测报告问题全量修复（P0/P1/P2/P3）
- P0-1: GetOcrTextNew 传图键补 uiSelector, 组件恢复可用 - P1-1: Image_Click 全版本 position 生效(九宫格锚点+自定义偏移 JSON), catalog 补下拉选项 - P1-2: Click_V3 click_key 生效(左/右/中键/双击), catalog 补选项 - P1-3: GetOcrTable 按垂直中心聚合成行列(行内按 x 排序), showTable 写流程数据表(与 Bivartable 共享 __rpaDatabook), isExcel 用 exceljs 另存并处理 res/randomString 占位 - P2-1: Find_V1/Click_V3 find_range 生效(整个屏幕/指定窗口/指定区域), 窗口模式前置+取矩形裁剪, 坐标平移回屏幕系 - P2-2(补): Image_Screenshot 空路径兜底到流程证据目录 - P3-1: accuracy2 无特征匹配层实现, catalog 隐藏并注明 - P3-2: show_track=1 时鼠标 20 步插值平滑移动后点击 - P3-3: 截屏/Find 换 desktopCapturer(去掉逐次 Add-Type); 新增 vision-helpers.test.ts 12 项单测(区域解析/边界夹取/裁剪/九宫格) - 回归: typecheck 通过; vitest 256 过(4 失败为桌面/PDF 环境遗留); 报告文档已标注修复状态
This commit is contained in:
yu
parent
342145cc46
commit
ba17d6dc2a
@ -7,6 +7,10 @@
|
||||
1. 静态核对——组件目录（catalog.json）字段定义 vs 实现代码（vision.ts / ocr-caps.ts）逐键比对
|
||||
2. 真实运行——GDI 生成含中英文的测试图、真实截屏、模板匹配直测、OCR 引擎实识别（飞桨 @gutenye/ocr-node）
|
||||
- **说明**：本次仅检测未修改任何代码，问题留给同事修复。
|
||||
- **修复状态（2026-11-02 更新）**：报告中的全部问题已由刘爱家修复完成——
|
||||
- 提交 `342145c`：P2-2（屏幕截图图像来源/截图范围，含区域裁剪与下拉选项）
|
||||
- 提交 `<本次>`：P0-1（GetOcrTextNew 传图键）、P1-1（点击位置九宫格/偏移）、P1-2（鼠标按键左/右/中/双击）、P1-3（GetOcrTable 行列聚类 + 写数据表 + 存 Excel）、P2-1（查找范围指定窗口/指定区域）、P3-2（鼠标移动轨迹动画）、P3-3（截图空路径落证据目录；表格行内按 x 排序）；P3-1（accuracy2）无特征匹配层实现，已在目录中隐藏并注明原因
|
||||
- 回归：typecheck 通过；vitest 256 过（4 失败为与本次无关的桌面/PDF 环境遗留项）；新增 vision-helpers 单测 12 项覆盖区域解析/裁剪/九宫格
|
||||
|
||||
---
|
||||
|
||||
|
||||
@ -61583,7 +61583,45 @@
|
||||
"placeholder": "",
|
||||
"description": "",
|
||||
"advanced": false,
|
||||
"valueTypes": []
|
||||
"valueTypes": [],
|
||||
"options": [
|
||||
{
|
||||
"label": "中心",
|
||||
"value": "center"
|
||||
},
|
||||
{
|
||||
"label": "左上",
|
||||
"value": "leftTop"
|
||||
},
|
||||
{
|
||||
"label": "上",
|
||||
"value": "top"
|
||||
},
|
||||
{
|
||||
"label": "右上",
|
||||
"value": "rightTop"
|
||||
},
|
||||
{
|
||||
"label": "左",
|
||||
"value": "left"
|
||||
},
|
||||
{
|
||||
"label": "右",
|
||||
"value": "right"
|
||||
},
|
||||
{
|
||||
"label": "左下",
|
||||
"value": "leftBottom"
|
||||
},
|
||||
{
|
||||
"label": "下",
|
||||
"value": "bottom"
|
||||
},
|
||||
{
|
||||
"label": "右下",
|
||||
"value": "rightBottom"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "accuracy",
|
||||
@ -61603,13 +61641,13 @@
|
||||
{
|
||||
"key": "accuracy2",
|
||||
"label": "特征相似度",
|
||||
"widget": "text",
|
||||
"widget": "hidden",
|
||||
"type": "string",
|
||||
"displayType": "text-expression",
|
||||
"required": true,
|
||||
"defaultValue": "0.2",
|
||||
"placeholder": "请输入相似度阈值",
|
||||
"description": "即需要查找的图片与模板图片的针对局部图像特征相似程度的阈值指标，与模板相似度共同决定组件运行结果。用户可自定义配置一个0~1区间的相似度取值（如0.55即55%相似度），0为完全不相似、1为完全一致，越大则命中阈值越高。（如无特殊需求，建议默认0.2）",
|
||||
"description": "【暂未启用】当前实现为纯像素模板匹配，无特征匹配层；字段保留以兼容旧流程",
|
||||
"advanced": false,
|
||||
"valueTypes": [
|
||||
"rpa_num"
|
||||
@ -62791,7 +62829,21 @@
|
||||
"placeholder": "",
|
||||
"description": "请选择查找目标图像的范围",
|
||||
"advanced": false,
|
||||
"valueTypes": []
|
||||
"valueTypes": [],
|
||||
"options": [
|
||||
{
|
||||
"label": "整个屏幕",
|
||||
"value": "1"
|
||||
},
|
||||
{
|
||||
"label": "指定窗口",
|
||||
"value": "2"
|
||||
},
|
||||
{
|
||||
"label": "指定区域",
|
||||
"value": "3"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "window_obj",
|
||||
@ -63297,7 +63349,21 @@
|
||||
"placeholder": "",
|
||||
"description": "请选择查找目标图像的范围",
|
||||
"advanced": false,
|
||||
"valueTypes": []
|
||||
"valueTypes": [],
|
||||
"options": [
|
||||
{
|
||||
"label": "整个屏幕",
|
||||
"value": "1"
|
||||
},
|
||||
{
|
||||
"label": "指定窗口",
|
||||
"value": "2"
|
||||
},
|
||||
{
|
||||
"label": "指定区域",
|
||||
"value": "3"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "window_obj",
|
||||
@ -63366,7 +63432,17 @@
|
||||
"placeholder": "",
|
||||
"description": "",
|
||||
"advanced": false,
|
||||
"valueTypes": []
|
||||
"valueTypes": [],
|
||||
"options": [
|
||||
{
|
||||
"label": "左键",
|
||||
"value": "left"
|
||||
},
|
||||
{
|
||||
"label": "右键",
|
||||
"value": "right"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "position",
|
||||
@ -63379,7 +63455,45 @@
|
||||
"placeholder": "",
|
||||
"description": "",
|
||||
"advanced": false,
|
||||
"valueTypes": []
|
||||
"valueTypes": [],
|
||||
"options": [
|
||||
{
|
||||
"label": "中心",
|
||||
"value": "center"
|
||||
},
|
||||
{
|
||||
"label": "左上",
|
||||
"value": "leftTop"
|
||||
},
|
||||
{
|
||||
"label": "上",
|
||||
"value": "top"
|
||||
},
|
||||
{
|
||||
"label": "右上",
|
||||
"value": "rightTop"
|
||||
},
|
||||
{
|
||||
"label": "左",
|
||||
"value": "left"
|
||||
},
|
||||
{
|
||||
"label": "右",
|
||||
"value": "right"
|
||||
},
|
||||
{
|
||||
"label": "左下",
|
||||
"value": "leftBottom"
|
||||
},
|
||||
{
|
||||
"label": "下",
|
||||
"value": "bottom"
|
||||
},
|
||||
{
|
||||
"label": "右下",
|
||||
"value": "rightBottom"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "accuracy",
|
||||
@ -87855,7 +87969,13 @@
|
||||
"placeholder": "",
|
||||
"description": "",
|
||||
"advanced": false,
|
||||
"valueTypes": []
|
||||
"valueTypes": [],
|
||||
"options": [
|
||||
{
|
||||
"label": "本地图片",
|
||||
"value": "0"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "uiSelector",
|
||||
|
||||
@ -1,3 +1,5 @@
|
||||
import path from "node:path";
|
||||
import { promises as fs } from "node:fs";
|
||||
import { register } from "../engine/registry";
|
||||
import { RuntimeError } from "../engine/types";
|
||||
|
||||
@ -15,7 +17,8 @@ async function getOcr(): Promise<{ detect: (image: string) => Promise<{ texts: R
|
||||
}
|
||||
|
||||
function requireImage(args: Record<string, unknown>, nodeId: string): string {
|
||||
for (const key of ["image_path", "imagePath", "file_path", "filePath", "path", "image", "pictureSource"]) {
|
||||
// uiSelector 是 OcrOffline_GetOcrTextNew 目录里唯一的图片字段（P0-1 修复）
|
||||
for (const key of ["image_path", "imagePath", "file_path", "filePath", "path", "image", "pictureSource", "uiSelector"]) {
|
||||
const v = String(args[key] ?? "").trim();
|
||||
if (v) return v;
|
||||
}
|
||||
@ -54,16 +57,61 @@ register("OcrOffline_GetOcrTextPosition", async (ctx, node, args) => {
|
||||
return words;
|
||||
});
|
||||
|
||||
/** 按垂直中心聚类成行（行高 60% 为阈值），行内按 x 排序，返回二维文本表格 */
|
||||
function groupOcrRows(words: RapidOcrLine[]): string[][] {
|
||||
const prepared = words
|
||||
.filter((word) => String(word.text ?? "").trim())
|
||||
.map((word) => {
|
||||
const box = word.box ?? [];
|
||||
const xs = box.map((point) => point[0]);
|
||||
const ys = box.map((point) => point[1]);
|
||||
const x = xs.length ? Math.min(...xs) : 0;
|
||||
const y = ys.length ? Math.min(...ys) : 0;
|
||||
const height = ys.length ? Math.max(...ys) - y : 0;
|
||||
return { text: word.text, x, cy: y + height / 2, h: height };
|
||||
})
|
||||
.sort((a, b) => a.cy - b.cy || a.x - b.x);
|
||||
const rows: Array<{ cy: number; h: number; cells: typeof prepared }> = [];
|
||||
let current: (typeof rows)[number] | null = null;
|
||||
for (const item of prepared) {
|
||||
if (!current || Math.abs(item.cy - current.cy) > Math.max(current.h, item.h, 8) * 0.6) {
|
||||
current = { cy: item.cy, h: item.h, cells: [] };
|
||||
rows.push(current);
|
||||
}
|
||||
current.cells.push(item);
|
||||
}
|
||||
return rows.map((row) => row.cells.sort((a, b) => a.x - b.x).map((cell) => cell.text.trim()));
|
||||
}
|
||||
|
||||
register("OcrOffline_GetOcrTable", async (ctx, node, args) => {
|
||||
const imagePath = requireImage(args, node.id);
|
||||
const ocr = await getOcr();
|
||||
const result = await ocr.detect(imagePath);
|
||||
const sorted = [...result.texts].sort((a, b) => {
|
||||
const ay = a.box ? Math.min(...a.box.map(p => p[1])) : 0;
|
||||
const by = b.box ? Math.min(...b.box.map(p => p[1])) : 0;
|
||||
return ay - by;
|
||||
});
|
||||
const rows = sorted.map(t => t.text);
|
||||
const rows = groupOcrRows(result.texts);
|
||||
ctx.log("OCR 表格识别完成：" + rows.length + " 行", "info", node.id);
|
||||
return rows;
|
||||
|
||||
// showTable=1：写入流程数据表（与 Bivartable 系列组件共享同一份数据簿）
|
||||
if (String(args.showTable ?? "1") === "1") {
|
||||
ctx.vars["__rpaDatabook"] = rows;
|
||||
ctx.log("已写入流程数据表", "info", node.id);
|
||||
}
|
||||
|
||||
// isExcel=1：另存为 Excel 文件
|
||||
let excelFile = "";
|
||||
if (String(args.isExcel ?? "0") === "1") {
|
||||
const ExcelJS = (await import("exceljs")).default;
|
||||
const workbook = new ExcelJS.Workbook();
|
||||
const sheet = workbook.addWorksheet("Sheet1");
|
||||
sheet.addRows(rows);
|
||||
const rawFolder = String(args.excelPath ?? "").trim();
|
||||
// ${res} 流程资源目录占位与空值统一落到运行证据目录
|
||||
const folder = rawFolder && rawFolder !== "${res}" ? rawFolder : ctx.evidenceDir;
|
||||
const fileName = String(args.excelName ?? "表格数据.xlsx").replace(/\$\{randomString\}/g, Math.random().toString(36).slice(2, 8));
|
||||
excelFile = path.resolve(path.isAbsolute(fileName) ? fileName : path.join(folder, fileName));
|
||||
await fs.mkdir(path.dirname(excelFile), { recursive: true });
|
||||
await workbook.xlsx.writeFile(excelFile);
|
||||
ctx.log("已保存 Excel：" + excelFile, "info", node.id);
|
||||
}
|
||||
|
||||
return excelFile ? { rows, excelFile } : rows;
|
||||
});
|
||||
81
src/main/capabilities/vision-helpers.test.ts
Normal file
81
src/main/capabilities/vision-helpers.test.ts
Normal file
@ -0,0 +1,81 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { PNG } from "pngjs";
|
||||
import { clickPoint, clampRegion, cropPng, parseRegion } from "./vision";
|
||||
|
||||
function makeImage(width = 100, height = 80): PNG {
|
||||
const png = new PNG({ width, height });
|
||||
for (let i = 0; i < png.data.length; i += 4) {
|
||||
png.data[i] = 200;
|
||||
png.data[i + 1] = 100;
|
||||
png.data[i + 2] = 50;
|
||||
png.data[i + 3] = 255;
|
||||
}
|
||||
return png;
|
||||
}
|
||||
|
||||
describe("parseRegion", () => {
|
||||
it("accepts object x/y/width/height", () => {
|
||||
const image = makeImage();
|
||||
expect(parseRegion('{"x":10,"y":20,"width":30,"height":40}', image)).toEqual({ x: 10, y: 20, width: 30, height: 40 });
|
||||
});
|
||||
|
||||
it("accepts array [x,y,width,height]", () => {
|
||||
const image = makeImage();
|
||||
expect(parseRegion("[1,2,3,4]", image)).toEqual({ x: 1, y: 2, width: 3, height: 4 });
|
||||
});
|
||||
|
||||
it("accepts left/top aliases and corner points", () => {
|
||||
const image = makeImage();
|
||||
expect(parseRegion('{"left":5,"top":6,"width":7,"height":8}', image)).toEqual({ x: 5, y: 6, width: 7, height: 8 });
|
||||
expect(parseRegion('{"x":5,"y":6,"x2":15,"y2":16}', image)).toEqual({ x: 5, y: 6, width: 10, height: 10 });
|
||||
});
|
||||
|
||||
it("rejects malformed JSON and unknown shapes", () => {
|
||||
const image = makeImage();
|
||||
expect(() => parseRegion("not-json", image)).toThrow(/无法解析/);
|
||||
expect(() => parseRegion('{"foo":1}', image)).toThrow(/不符合要求/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("clampRegion", () => {
|
||||
it("clamps region into image bounds", () => {
|
||||
const image = makeImage(100, 80);
|
||||
expect(clampRegion({ x: -5, y: -5, width: 200, height: 200 }, image)).toEqual({ x: 0, y: 0, width: 100, height: 80 });
|
||||
expect(clampRegion({ x: 90, y: 70, width: 50, height: 50 }, image)).toEqual({ x: 90, y: 70, width: 10, height: 10 });
|
||||
});
|
||||
});
|
||||
|
||||
describe("cropPng", () => {
|
||||
it("crops sub-image with expected size and pixel origin", () => {
|
||||
const image = makeImage();
|
||||
const cropped = cropPng(image, { x: 10, y: 20, width: 30, height: 40 });
|
||||
expect(cropped.width).toBe(30);
|
||||
expect(cropped.height).toBe(40);
|
||||
const index = (0 * 30 + 0) * 4;
|
||||
expect([cropped.data[index], cropped.data[index + 1], cropped.data[index + 2]]).toEqual([200, 100, 50]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("clickPoint", () => {
|
||||
const region = { x: 100, y: 200, width: 60, height: 40, score: 1 };
|
||||
|
||||
it("defaults to center", () => {
|
||||
expect(clickPoint(region, "")).toEqual({ x: 130, y: 220 });
|
||||
expect(clickPoint(region, "center")).toEqual({ x: 130, y: 220 });
|
||||
});
|
||||
|
||||
it("supports nine-grid anchors", () => {
|
||||
expect(clickPoint(region, "leftTop")).toEqual({ x: 100, y: 200 });
|
||||
expect(clickPoint(region, "rightBottom")).toEqual({ x: 160, y: 240 });
|
||||
expect(clickPoint(region, "top")).toEqual({ x: 130, y: 200 });
|
||||
expect(clickPoint(region, "left")).toEqual({ x: 100, y: 220 });
|
||||
});
|
||||
|
||||
it("supports custom offset JSON", () => {
|
||||
expect(clickPoint(region, '{"x":5,"y":6}')).toEqual({ x: 105, y: 206 });
|
||||
});
|
||||
|
||||
it("falls back to center for unknown values", () => {
|
||||
expect(clickPoint(region, "bogus")).toEqual({ x: 130, y: 220 });
|
||||
});
|
||||
});
|
||||
@ -23,7 +23,7 @@ async function capturePrimaryScreenPng(): Promise<PNG> {
|
||||
}
|
||||
|
||||
/** 解析区域 JSON：对象 {x,y,width,height}（兼容 left/top、x1y1x2y2 角点）或数组 [x,y,width,height]，并夹取到图片边界内 */
|
||||
function parseRegion(raw: string, image: PNG): { x: number; y: number; width: number; height: number } {
|
||||
export function parseRegion(raw: string, image: PNG): { x: number; y: number; width: number; height: number } {
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(raw);
|
||||
@ -61,7 +61,7 @@ function parseRegion(raw: string, image: PNG): { x: number; y: number; width: nu
|
||||
});
|
||||
}
|
||||
|
||||
function clampRegion(region: { x: number; y: number; width: number; height: number }, image: PNG): { x: number; y: number; width: number; height: number } {
|
||||
export function clampRegion(region: { x: number; y: number; width: number; height: number }, image: PNG): { x: number; y: number; width: number; height: number } {
|
||||
const x = Math.max(0, Math.min(Math.round(region.x), image.width - 1));
|
||||
const y = Math.max(0, Math.min(Math.round(region.y), image.height - 1));
|
||||
const width = Math.max(1, Math.min(Math.round(region.width), image.width - x));
|
||||
@ -69,13 +69,13 @@ function clampRegion(region: { x: number; y: number; width: number; height: numb
|
||||
return { x, y, width, height };
|
||||
}
|
||||
|
||||
function cropPng(source: PNG, region: { x: number; y: number; width: number; height: number }): PNG {
|
||||
export function cropPng(source: PNG, region: { x: number; y: number; width: number; height: number }): PNG {
|
||||
const cropped = new PNG({ width: region.width, height: region.height });
|
||||
PNG.bitblt(source, cropped, region.x, region.y, region.width, region.height, 0, 0);
|
||||
return cropped;
|
||||
}
|
||||
|
||||
register("Image_Screenshot", async (_ctx, node, args) => {
|
||||
register("Image_Screenshot", async (ctx, node, args) => {
|
||||
const folder = text(args, ["outImagePath", "folder"]);
|
||||
const name = text(args, ["outImageName", "name"], `screenshot_${Date.now()}.png`);
|
||||
const imageArg = text(args, ["image_path"]);
|
||||
@ -95,9 +95,9 @@ register("Image_Screenshot", async (_ctx, node, args) => {
|
||||
image = PNG.sync.read(await fs.readFile(sourceFile));
|
||||
output = folder ? path.resolve(path.join(folder, name)) : sourceFile;
|
||||
} else {
|
||||
// 当前屏幕来源：image_path 即保存路径，缺省用 输出目录+文件名
|
||||
// 当前屏幕来源：image_path 即保存路径，缺省用 输出目录+文件名；再缺省落流程证据目录（路径可预期）
|
||||
image = await capturePrimaryScreenPng();
|
||||
output = path.resolve(imageArg || (folder ? path.join(folder, name) : name));
|
||||
output = path.resolve(imageArg || (folder ? path.join(folder, name) : path.join(ctx.evidenceDir, name)));
|
||||
}
|
||||
|
||||
if (range === "2") {
|
||||
@ -119,10 +119,46 @@ interface MatchResult { x: number; y: number; width: number; height: number; sco
|
||||
|
||||
async function captureScreen(target: string): Promise<string> {
|
||||
await fs.mkdir(path.dirname(target), { recursive: true });
|
||||
await powershell(`Add-Type -AssemblyName System.Drawing; Add-Type -AssemblyName System.Windows.Forms; $bounds = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds; $bitmap = New-Object System.Drawing.Bitmap $bounds.Width,$bounds.Height; $graphics = [System.Drawing.Graphics]::FromImage($bitmap); $graphics.CopyFromScreen($bounds.Location,[System.Drawing.Point]::Empty,$bounds.Size); $bitmap.Save(${psString(target)},[System.Drawing.Imaging.ImageFormat]::Png); $graphics.Dispose(); $bitmap.Dispose()`);
|
||||
await fs.writeFile(target, PNG.sync.write(await capturePrimaryScreenPng()));
|
||||
return target;
|
||||
}
|
||||
|
||||
interface WindowRect { x: number; y: number; width: number; height: number }
|
||||
|
||||
/** 按标题（精确→包含）或进程名查找窗口，前置并返回窗口矩形（屏幕坐标） */
|
||||
async function focusWindowRect(windowObj: string): Promise<WindowRect> {
|
||||
const quoted = psString(windowObj);
|
||||
const stdout = await powershell(`Add-Type @'
|
||||
using System; using System.Runtime.InteropServices;
|
||||
public static class WinRect {
|
||||
[DllImport("user32.dll")] public static extern IntPtr FindWindow(string cls, string title);
|
||||
[DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr h);
|
||||
[DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr h, int cmd);
|
||||
[DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr h, out RECT r);
|
||||
public struct RECT { public int Left; public int Top; public int Right; public int Bottom; }
|
||||
}
|
||||
'@;
|
||||
$h = [WinRect]::FindWindow($null, ${quoted});
|
||||
if ($h -eq [IntPtr]::Zero) { $p = Get-Process | Where-Object { $_.MainWindowTitle -and ($_.MainWindowTitle -eq ${quoted} -or $_.MainWindowTitle.Contains(${quoted})) } | Select-Object -First 1; if ($p) { $h = $p.MainWindowHandle } }
|
||||
if ($h -eq [IntPtr]::Zero) { Write-Output 'NOT_FOUND'; exit }
|
||||
[WinRect]::ShowWindow($h, 9) | Out-Null;
|
||||
[WinRect]::SetForegroundWindow($h) | Out-Null;
|
||||
Start-Sleep -Milliseconds 300;
|
||||
$r = New-Object WinRect+RECT;
|
||||
[WinRect]::GetWindowRect($h, [ref]$r) | Out-Null;
|
||||
Write-Output ("{0},{1},{2},{3}" -f $r.Left, $r.Top, $r.Right, $r.Bottom)`);
|
||||
const line = stdout.trim().split(/\r?\n/).pop() ?? "";
|
||||
if (line === "NOT_FOUND") {
|
||||
throw new RuntimeError("not_found", `未找到窗口「${windowObj}」`, { recommendation: "确认窗口已打开，窗口对象可填完整标题、标题包含关键字或进程名" });
|
||||
}
|
||||
const parts = line.split(",").map(Number);
|
||||
if (parts.length !== 4 || parts.some((v) => !Number.isFinite(v))) {
|
||||
throw new RuntimeError("desktop_error", "窗口矩形获取失败", { detail: line.slice(0, 120) });
|
||||
}
|
||||
const [left, top, right, bottom] = parts;
|
||||
return { x: left, y: top, width: Math.max(1, right - left), height: Math.max(1, bottom - top) };
|
||||
}
|
||||
|
||||
function pixelDifference(screen: PNG, template: PNG, offsetX: number, offsetY: number, sampleStep: number, cutoff: number): number {
|
||||
let difference = 0;
|
||||
let samples = 0;
|
||||
@ -176,18 +212,104 @@ async function locate(args: Record<string, unknown>, evidenceDir: string, nodeId
|
||||
const rawTemplate = text(args, ["image", "image_path", "template"]);
|
||||
if (!rawTemplate.trim()) throw new RuntimeError("validation_error", "模板图片路径为空", { nodeId });
|
||||
const template = path.resolve(rawTemplate);
|
||||
|
||||
// 查找范围：1 整个屏幕（默认）/ 2 指定窗口 / 3 指定区域；窗口/区域模式下结果坐标会平移回屏幕坐标系
|
||||
const rangeMode = text(args, ["find_range"], "1");
|
||||
const screenshot = path.join(evidenceDir, `${nodeId}_screen.png`);
|
||||
await captureScreen(screenshot);
|
||||
let offsetX = 0;
|
||||
let offsetY = 0;
|
||||
if (rangeMode === "2") {
|
||||
const windowObj = text(args, ["window_obj"]);
|
||||
if (!windowObj.trim()) throw new RuntimeError("validation_error", "查找范围为「指定窗口」时，窗口对象不能为空", { nodeId, recommendation: "填写窗口标题（或标题关键字、进程名）" });
|
||||
const rect = await focusWindowRect(windowObj);
|
||||
const screen = await capturePrimaryScreenPng();
|
||||
await fs.writeFile(screenshot, PNG.sync.write(cropPng(screen, clampRegion(rect, screen))));
|
||||
offsetX = rect.x;
|
||||
offsetY = rect.y;
|
||||
} else if (rangeMode === "3") {
|
||||
const regionRaw = text(args, ["shot_position", "action", "position"]);
|
||||
if (!regionRaw.trim()) throw new RuntimeError("validation_error", "查找范围为「指定区域」时，需要填写区域 JSON", { nodeId, recommendation: '在 shot_position 填写 {"x":0,"y":0,"width":100,"height":80} 或 [x, y, width, height]' });
|
||||
const screen = await capturePrimaryScreenPng();
|
||||
const region = clampRegion(parseRegion(regionRaw, screen), screen);
|
||||
await fs.writeFile(screenshot, PNG.sync.write(cropPng(screen, region)));
|
||||
offsetX = region.x;
|
||||
offsetY = region.y;
|
||||
} else {
|
||||
await captureScreen(screenshot);
|
||||
}
|
||||
|
||||
const result = await findImage(template, screenshot, Number(text(args, ["accuracy"], "0.2")));
|
||||
if (!result) throw new RuntimeError("not_found", "屏幕中未找到目标图像", { nodeId, recommendation: "重新截取模板，或调低匹配精度" });
|
||||
return result;
|
||||
return { ...result, x: result.x + offsetX, y: result.y + offsetY };
|
||||
}
|
||||
|
||||
register(["Image_Find", "Image_Find_V1"], async (ctx, node, args) => locate(args, ctx.evidenceDir, node.id));
|
||||
/** 点击位置锚点（相对匹配区域的比例坐标），支持九宫格取值 */
|
||||
const POSITION_ANCHORS: Record<string, [number, number]> = {
|
||||
center: [0.5, 0.5],
|
||||
leftTop: [0, 0], top: [0.5, 0], centerTop: [0.5, 0], rightTop: [1, 0],
|
||||
left: [0, 0.5], centerLeft: [0, 0.5], right: [1, 0.5], centerRight: [1, 0.5],
|
||||
leftBottom: [0, 1], bottom: [0.5, 1], centerBottom: [0.5, 1], rightBottom: [1, 1]
|
||||
};
|
||||
|
||||
export function clickPoint(result: MatchResult, position: string): { x: number; y: number } {
|
||||
const raw = position.trim();
|
||||
if (raw && raw.toLowerCase() !== "center") {
|
||||
if (raw.startsWith("{")) {
|
||||
try {
|
||||
const offset = JSON.parse(raw) as { x?: unknown; y?: unknown };
|
||||
const dx = Number(offset.x);
|
||||
const dy = Number(offset.y);
|
||||
if (Number.isFinite(dx) && Number.isFinite(dy)) {
|
||||
return { x: Math.round(result.x + dx), y: Math.round(result.y + dy) };
|
||||
}
|
||||
} catch {
|
||||
// 非法 JSON 落回中心
|
||||
}
|
||||
}
|
||||
const anchor = POSITION_ANCHORS[raw] ?? POSITION_ANCHORS[raw.toLowerCase()];
|
||||
if (anchor) {
|
||||
return { x: Math.round(result.x + result.width * anchor[0]), y: Math.round(result.y + result.height * anchor[1]) };
|
||||
}
|
||||
}
|
||||
return { x: Math.round(result.x + result.width / 2), y: Math.round(result.y + result.height / 2) };
|
||||
}
|
||||
|
||||
const CLICK_ACTIONS: Record<string, string> = {
|
||||
left: "[Mouse]::mouse_event(2,0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 30; [Mouse]::mouse_event(4,0,0,0,[UIntPtr]::Zero)",
|
||||
right: "[Mouse]::mouse_event(8,0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 30; [Mouse]::mouse_event(16,0,0,0,[UIntPtr]::Zero)",
|
||||
middle: "[Mouse]::mouse_event(32,0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 30; [Mouse]::mouse_event(64,0,0,0,[UIntPtr]::Zero)",
|
||||
double: "[Mouse]::mouse_event(2,0,0,0,[UIntPtr]::Zero); [Mouse]::mouse_event(4,0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 60; [Mouse]::mouse_event(2,0,0,0,[UIntPtr]::Zero); [Mouse]::mouse_event(4,0,0,0,[UIntPtr]::Zero)"
|
||||
};
|
||||
|
||||
async function injectClick(x: number, y: number, clickKey: string, showTrack: boolean): Promise<void> {
|
||||
const action = CLICK_ACTIONS[clickKey] ?? CLICK_ACTIONS.left;
|
||||
// show_track=1 时从当前光标位置平滑移动过去（20 步插值），否则直接跳转
|
||||
const move = showTrack
|
||||
? `for ($i = 1; $i -le 20; $i++) { [Mouse]::SetCursorPos([int]($p.X + (${x} - $p.X) * $i / 20), [int]($p.Y + (${y} - $p.Y) * $i / 20)); Start-Sleep -Milliseconds 12 }`
|
||||
: `[Mouse]::SetCursorPos(${x}, ${y})`;
|
||||
await powershell(`Add-Type @'
|
||||
using System; using System.Runtime.InteropServices;
|
||||
public static class Mouse {
|
||||
[DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y);
|
||||
[DllImport("user32.dll")] public static extern bool GetCursorPos(out POINT p);
|
||||
[DllImport("user32.dll")] public static extern void mouse_event(uint f, uint x, uint y, uint d, UIntPtr e);
|
||||
public struct POINT { public int X; public int Y; }
|
||||
}
|
||||
'@;
|
||||
$p = New-Object Mouse+POINT;
|
||||
[Mouse]::GetCursorPos([ref]$p) | Out-Null;
|
||||
${move};
|
||||
Start-Sleep -Milliseconds 60;
|
||||
${action}`);
|
||||
}
|
||||
|
||||
register(["Image_Click", "Image_Click_V1", "Image_Click_V2", "Image_Click_V3"], async (ctx, node, args) => {
|
||||
const result = await locate(args, ctx.evidenceDir, node.id);
|
||||
const x = Math.round(result.x + result.width / 2);
|
||||
const y = Math.round(result.y + result.height / 2);
|
||||
await powershell(`Add-Type @'\nusing System; using System.Runtime.InteropServices; public static class Clicker { [DllImport("user32.dll")] public static extern bool SetCursorPos(int X,int Y); [DllImport("user32.dll")] public static extern void mouse_event(uint f,uint x,uint y,uint d,UIntPtr e); }\n'@; [Clicker]::SetCursorPos(${x},${y}); [Clicker]::mouse_event(2,0,0,0,[UIntPtr]::Zero); [Clicker]::mouse_event(4,0,0,0,[UIntPtr]::Zero)`);
|
||||
return { ...result, click: { x, y } };
|
||||
const point = clickPoint(result, text(args, ["position"], "center"));
|
||||
const clickKey = text(args, ["click_key"], "left");
|
||||
const showTrack = String(args.show_track ?? "0") === "1" || args.show_track === true;
|
||||
await injectClick(point.x, point.y, clickKey, showTrack);
|
||||
ctx.log(`已${clickKey === "right" ? "右键" : clickKey === "double" ? "双击" : "点击"}图像 (${point.x}, ${point.y})，匹配得分 ${result.score.toFixed(3)}`, "info", node.id);
|
||||
return { ...result, click: point };
|
||||
});
|
||||
|
||||
Loading…
Reference in New Issue
Block a user