Merge branch 'main' of http://43.140.215.244:7040/wangyukun/PRA-only-demo
This commit is contained in:
yu
commit
bf70cf92e5
@ -7,6 +7,10 @@
|
||||
1. 静态核对——组件目录（catalog.json）字段定义 vs 实现代码（vision.ts / ocr-caps.ts）逐键比对
|
||||
2. 真实运行——GDI 生成含中英文的测试图、真实截屏、模板匹配直测、OCR 引擎实识别（飞桨 @gutenye/ocr-node）
|
||||
- **说明**：本次仅检测未修改任何代码，问题留给同事修复。
|
||||
- **修复状态（2026-11-02 更新）**：报告中的全部问题已由刘爱家修复完成——
|
||||
- 提交 `342145c`：P2-2（屏幕截图图像来源/截图范围，含区域裁剪与下拉选项）
|
||||
- 提交 `ba17d6d`：P0-1（GetOcrTextNew 传图键）、P1-1（点击位置九宫格/偏移）、P1-2（鼠标按键左/右/中/双击）、P1-3（GetOcrTable 行列聚类 + 写数据表 + 存 Excel）、P2-1（查找范围指定窗口/指定区域）、P3-2（鼠标移动轨迹动画）、P3-3（截图空路径落证据目录；表格行内按 x 排序）；P3-1（accuracy2）无特征匹配层实现，已在目录中隐藏并注明原因
|
||||
- 回归：typecheck 通过；vitest 256 过（4 失败为与本次无关的桌面/PDF 环境遗留项）；新增 vision-helpers 单测 12 项覆盖区域解析/裁剪/九宫格
|
||||
|
||||
---
|
||||
|
||||
|
||||
@ -61583,7 +61583,45 @@
|
||||
"placeholder": "",
|
||||
"description": "",
|
||||
"advanced": false,
|
||||
"valueTypes": []
|
||||
"valueTypes": [],
|
||||
"options": [
|
||||
{
|
||||
"label": "中心",
|
||||
"value": "center"
|
||||
},
|
||||
{
|
||||
"label": "左上",
|
||||
"value": "leftTop"
|
||||
},
|
||||
{
|
||||
"label": "上",
|
||||
"value": "top"
|
||||
},
|
||||
{
|
||||
"label": "右上",
|
||||
"value": "rightTop"
|
||||
},
|
||||
{
|
||||
"label": "左",
|
||||
"value": "left"
|
||||
},
|
||||
{
|
||||
"label": "右",
|
||||
"value": "right"
|
||||
},
|
||||
{
|
||||
"label": "左下",
|
||||
"value": "leftBottom"
|
||||
},
|
||||
{
|
||||
"label": "下",
|
||||
"value": "bottom"
|
||||
},
|
||||
{
|
||||
"label": "右下",
|
||||
"value": "rightBottom"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "accuracy",
|
||||
@ -61603,13 +61641,13 @@
|
||||
{
|
||||
"key": "accuracy2",
|
||||
"label": "特征相似度",
|
||||
"widget": "text",
|
||||
"widget": "hidden",
|
||||
"type": "string",
|
||||
"displayType": "text-expression",
|
||||
"required": true,
|
||||
"defaultValue": "0.2",
|
||||
"placeholder": "请输入相似度阈值",
|
||||
"description": "即需要查找的图片与模板图片的针对局部图像特征相似程度的阈值指标，与模板相似度共同决定组件运行结果。用户可自定义配置一个0~1区间的相似度取值（如0.55即55%相似度），0为完全不相似、1为完全一致，越大则命中阈值越高。（如无特殊需求，建议默认0.2）",
|
||||
"description": "【暂未启用】当前实现为纯像素模板匹配，无特征匹配层；字段保留以兼容旧流程",
|
||||
"advanced": false,
|
||||
"valueTypes": [
|
||||
"rpa_num"
|
||||
@ -61857,15 +61895,25 @@
|
||||
{
|
||||
"key": "pictureSource",
|
||||
"label": "图像来源",
|
||||
"widget": "radio",
|
||||
"widget": "hidden",
|
||||
"type": "string",
|
||||
"displayType": "buttonGroup",
|
||||
"required": false,
|
||||
"defaultValue": "1",
|
||||
"placeholder": "",
|
||||
"description": "选择要截图的图像来源，可以是当前屏幕，也可以是指定的图片文件。",
|
||||
"description": "【界面已隐藏】实现默认截取当前屏幕；历史流程如存有 2(指定图片文件) 仍按原逻辑生效",
|
||||
"advanced": false,
|
||||
"valueTypes": []
|
||||
"valueTypes": [],
|
||||
"options": [
|
||||
{
|
||||
"label": "当前屏幕",
|
||||
"value": "1"
|
||||
},
|
||||
{
|
||||
"label": "指定图片文件",
|
||||
"value": "2"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "shotRange",
|
||||
@ -61878,7 +61926,17 @@
|
||||
"placeholder": "",
|
||||
"description": "选择要截图的范围，可以使截取整个屏幕，或手动选择截图区域",
|
||||
"advanced": false,
|
||||
"valueTypes": []
|
||||
"valueTypes": [],
|
||||
"options": [
|
||||
{
|
||||
"label": "整个屏幕",
|
||||
"value": "1"
|
||||
},
|
||||
{
|
||||
"label": "框选区域",
|
||||
"value": "2"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "image_path",
|
||||
@ -62771,7 +62829,21 @@
|
||||
"placeholder": "",
|
||||
"description": "请选择查找目标图像的范围",
|
||||
"advanced": false,
|
||||
"valueTypes": []
|
||||
"valueTypes": [],
|
||||
"options": [
|
||||
{
|
||||
"label": "整个屏幕",
|
||||
"value": "1"
|
||||
},
|
||||
{
|
||||
"label": "指定窗口",
|
||||
"value": "2"
|
||||
},
|
||||
{
|
||||
"label": "指定区域",
|
||||
"value": "3"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "window_obj",
|
||||
@ -63277,7 +63349,21 @@
|
||||
"placeholder": "",
|
||||
"description": "请选择查找目标图像的范围",
|
||||
"advanced": false,
|
||||
"valueTypes": []
|
||||
"valueTypes": [],
|
||||
"options": [
|
||||
{
|
||||
"label": "整个屏幕",
|
||||
"value": "1"
|
||||
},
|
||||
{
|
||||
"label": "指定窗口",
|
||||
"value": "2"
|
||||
},
|
||||
{
|
||||
"label": "指定区域",
|
||||
"value": "3"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "window_obj",
|
||||
@ -63346,7 +63432,17 @@
|
||||
"placeholder": "",
|
||||
"description": "",
|
||||
"advanced": false,
|
||||
"valueTypes": []
|
||||
"valueTypes": [],
|
||||
"options": [
|
||||
{
|
||||
"label": "左键",
|
||||
"value": "left"
|
||||
},
|
||||
{
|
||||
"label": "右键",
|
||||
"value": "right"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "position",
|
||||
@ -63359,7 +63455,45 @@
|
||||
"placeholder": "",
|
||||
"description": "",
|
||||
"advanced": false,
|
||||
"valueTypes": []
|
||||
"valueTypes": [],
|
||||
"options": [
|
||||
{
|
||||
"label": "中心",
|
||||
"value": "center"
|
||||
},
|
||||
{
|
||||
"label": "左上",
|
||||
"value": "leftTop"
|
||||
},
|
||||
{
|
||||
"label": "上",
|
||||
"value": "top"
|
||||
},
|
||||
{
|
||||
"label": "右上",
|
||||
"value": "rightTop"
|
||||
},
|
||||
{
|
||||
"label": "左",
|
||||
"value": "left"
|
||||
},
|
||||
{
|
||||
"label": "右",
|
||||
"value": "right"
|
||||
},
|
||||
{
|
||||
"label": "左下",
|
||||
"value": "leftBottom"
|
||||
},
|
||||
{
|
||||
"label": "下",
|
||||
"value": "bottom"
|
||||
},
|
||||
{
|
||||
"label": "右下",
|
||||
"value": "rightBottom"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "accuracy",
|
||||
@ -87835,7 +87969,13 @@
|
||||
"placeholder": "",
|
||||
"description": "",
|
||||
"advanced": false,
|
||||
"valueTypes": []
|
||||
"valueTypes": [],
|
||||
"options": [
|
||||
{
|
||||
"label": "本地图片",
|
||||
"value": "0"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "uiSelector",
|
||||
|
||||
@ -178,21 +178,4 @@ register("WebBrowser_Screenshot", async (ctx, node, args) => {
|
||||
return file;
|
||||
});
|
||||
|
||||
register("Image_Screenshot", async (ctx, node, args) => {
|
||||
const { desktopCapturer, screen } = await import("electron");
|
||||
const folder = String(args.outImagePath ?? args.image_path ?? ctx.evidenceDir);
|
||||
const name = String(args.outImageName ?? `${node.id}.png`);
|
||||
const { promises: fs } = await import("node:fs");
|
||||
const path = await import("node:path");
|
||||
const target = path.resolve(folder.endsWith(".png") ? folder : path.join(folder, name));
|
||||
const primary = screen.getPrimaryDisplay();
|
||||
const sources = await desktopCapturer.getSources({
|
||||
types: ["screen"],
|
||||
thumbnailSize: { width: primary.size.width, height: primary.size.height }
|
||||
});
|
||||
const source = sources[0];
|
||||
if (!source) throw new RuntimeError("desktop_error", "无法获取屏幕截图");
|
||||
await fs.mkdir(path.dirname(target), { recursive: true });
|
||||
await fs.writeFile(target, source.thumbnail.toPNG());
|
||||
return target;
|
||||
});
|
||||
// Image_Screenshot 统一在 vision.ts 实现（屏幕/文件来源 + 区域裁剪），此处不再重复注册
|
||||
|
||||
@ -1,3 +1,5 @@
|
||||
import path from "node:path";
|
||||
import { promises as fs } from "node:fs";
|
||||
import { register } from "../engine/registry";
|
||||
import { RuntimeError } from "../engine/types";
|
||||
|
||||
@ -15,7 +17,8 @@ async function getOcr(): Promise<{ detect: (image: string) => Promise<{ texts: R
|
||||
}
|
||||
|
||||
function requireImage(args: Record<string, unknown>, nodeId: string): string {
|
||||
for (const key of ["image_path", "imagePath", "file_path", "filePath", "path", "image", "pictureSource"]) {
|
||||
// uiSelector 是 OcrOffline_GetOcrTextNew 目录里唯一的图片字段（P0-1 修复）
|
||||
for (const key of ["image_path", "imagePath", "file_path", "filePath", "path", "image", "pictureSource", "uiSelector"]) {
|
||||
const v = String(args[key] ?? "").trim();
|
||||
if (v) return v;
|
||||
}
|
||||
@ -54,16 +57,61 @@ register("OcrOffline_GetOcrTextPosition", async (ctx, node, args) => {
|
||||
return words;
|
||||
});
|
||||
|
||||
/** 按垂直中心聚类成行（行高 60% 为阈值），行内按 x 排序，返回二维文本表格 */
|
||||
function groupOcrRows(words: RapidOcrLine[]): string[][] {
|
||||
const prepared = words
|
||||
.filter((word) => String(word.text ?? "").trim())
|
||||
.map((word) => {
|
||||
const box = word.box ?? [];
|
||||
const xs = box.map((point) => point[0]);
|
||||
const ys = box.map((point) => point[1]);
|
||||
const x = xs.length ? Math.min(...xs) : 0;
|
||||
const y = ys.length ? Math.min(...ys) : 0;
|
||||
const height = ys.length ? Math.max(...ys) - y : 0;
|
||||
return { text: word.text, x, cy: y + height / 2, h: height };
|
||||
})
|
||||
.sort((a, b) => a.cy - b.cy || a.x - b.x);
|
||||
const rows: Array<{ cy: number; h: number; cells: typeof prepared }> = [];
|
||||
let current: (typeof rows)[number] | null = null;
|
||||
for (const item of prepared) {
|
||||
if (!current || Math.abs(item.cy - current.cy) > Math.max(current.h, item.h, 8) * 0.6) {
|
||||
current = { cy: item.cy, h: item.h, cells: [] };
|
||||
rows.push(current);
|
||||
}
|
||||
current.cells.push(item);
|
||||
}
|
||||
return rows.map((row) => row.cells.sort((a, b) => a.x - b.x).map((cell) => cell.text.trim()));
|
||||
}
|
||||
|
||||
register("OcrOffline_GetOcrTable", async (ctx, node, args) => {
|
||||
const imagePath = requireImage(args, node.id);
|
||||
const ocr = await getOcr();
|
||||
const result = await ocr.detect(imagePath);
|
||||
const sorted = [...result.texts].sort((a, b) => {
|
||||
const ay = a.box ? Math.min(...a.box.map(p => p[1])) : 0;
|
||||
const by = b.box ? Math.min(...b.box.map(p => p[1])) : 0;
|
||||
return ay - by;
|
||||
});
|
||||
const rows = sorted.map(t => t.text);
|
||||
const rows = groupOcrRows(result.texts);
|
||||
ctx.log("OCR 表格识别完成：" + rows.length + " 行", "info", node.id);
|
||||
return rows;
|
||||
|
||||
// showTable=1：写入流程数据表（与 Bivartable 系列组件共享同一份数据簿）
|
||||
if (String(args.showTable ?? "1") === "1") {
|
||||
ctx.vars["__rpaDatabook"] = rows;
|
||||
ctx.log("已写入流程数据表", "info", node.id);
|
||||
}
|
||||
|
||||
// isExcel=1：另存为 Excel 文件
|
||||
let excelFile = "";
|
||||
if (String(args.isExcel ?? "0") === "1") {
|
||||
const ExcelJS = (await import("exceljs")).default;
|
||||
const workbook = new ExcelJS.Workbook();
|
||||
const sheet = workbook.addWorksheet("Sheet1");
|
||||
sheet.addRows(rows);
|
||||
const rawFolder = String(args.excelPath ?? "").trim();
|
||||
// ${res} 流程资源目录占位与空值统一落到运行证据目录
|
||||
const folder = rawFolder && rawFolder !== "${res}" ? rawFolder : ctx.evidenceDir;
|
||||
const fileName = String(args.excelName ?? "表格数据.xlsx").replace(/\$\{randomString\}/g, Math.random().toString(36).slice(2, 8));
|
||||
excelFile = path.resolve(path.isAbsolute(fileName) ? fileName : path.join(folder, fileName));
|
||||
await fs.mkdir(path.dirname(excelFile), { recursive: true });
|
||||
await workbook.xlsx.writeFile(excelFile);
|
||||
ctx.log("已保存 Excel：" + excelFile, "info", node.id);
|
||||
}
|
||||
|
||||
return excelFile ? { rows, excelFile } : rows;
|
||||
});
|
||||
81
src/main/capabilities/vision-helpers.test.ts
Normal file
81
src/main/capabilities/vision-helpers.test.ts
Normal file
@ -0,0 +1,81 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { PNG } from "pngjs";
|
||||
import { clickPoint, clampRegion, cropPng, parseRegion } from "./vision";
|
||||
|
||||
function makeImage(width = 100, height = 80): PNG {
|
||||
const png = new PNG({ width, height });
|
||||
for (let i = 0; i < png.data.length; i += 4) {
|
||||
png.data[i] = 200;
|
||||
png.data[i + 1] = 100;
|
||||
png.data[i + 2] = 50;
|
||||
png.data[i + 3] = 255;
|
||||
}
|
||||
return png;
|
||||
}
|
||||
|
||||
describe("parseRegion", () => {
|
||||
it("accepts object x/y/width/height", () => {
|
||||
const image = makeImage();
|
||||
expect(parseRegion('{"x":10,"y":20,"width":30,"height":40}', image)).toEqual({ x: 10, y: 20, width: 30, height: 40 });
|
||||
});
|
||||
|
||||
it("accepts array [x,y,width,height]", () => {
|
||||
const image = makeImage();
|
||||
expect(parseRegion("[1,2,3,4]", image)).toEqual({ x: 1, y: 2, width: 3, height: 4 });
|
||||
});
|
||||
|
||||
it("accepts left/top aliases and corner points", () => {
|
||||
const image = makeImage();
|
||||
expect(parseRegion('{"left":5,"top":6,"width":7,"height":8}', image)).toEqual({ x: 5, y: 6, width: 7, height: 8 });
|
||||
expect(parseRegion('{"x":5,"y":6,"x2":15,"y2":16}', image)).toEqual({ x: 5, y: 6, width: 10, height: 10 });
|
||||
});
|
||||
|
||||
it("rejects malformed JSON and unknown shapes", () => {
|
||||
const image = makeImage();
|
||||
expect(() => parseRegion("not-json", image)).toThrow(/无法解析/);
|
||||
expect(() => parseRegion('{"foo":1}', image)).toThrow(/不符合要求/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("clampRegion", () => {
|
||||
it("clamps region into image bounds", () => {
|
||||
const image = makeImage(100, 80);
|
||||
expect(clampRegion({ x: -5, y: -5, width: 200, height: 200 }, image)).toEqual({ x: 0, y: 0, width: 100, height: 80 });
|
||||
expect(clampRegion({ x: 90, y: 70, width: 50, height: 50 }, image)).toEqual({ x: 90, y: 70, width: 10, height: 10 });
|
||||
});
|
||||
});
|
||||
|
||||
describe("cropPng", () => {
|
||||
it("crops sub-image with expected size and pixel origin", () => {
|
||||
const image = makeImage();
|
||||
const cropped = cropPng(image, { x: 10, y: 20, width: 30, height: 40 });
|
||||
expect(cropped.width).toBe(30);
|
||||
expect(cropped.height).toBe(40);
|
||||
const index = (0 * 30 + 0) * 4;
|
||||
expect([cropped.data[index], cropped.data[index + 1], cropped.data[index + 2]]).toEqual([200, 100, 50]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("clickPoint", () => {
|
||||
const region = { x: 100, y: 200, width: 60, height: 40, score: 1 };
|
||||
|
||||
it("defaults to center", () => {
|
||||
expect(clickPoint(region, "")).toEqual({ x: 130, y: 220 });
|
||||
expect(clickPoint(region, "center")).toEqual({ x: 130, y: 220 });
|
||||
});
|
||||
|
||||
it("supports nine-grid anchors", () => {
|
||||
expect(clickPoint(region, "leftTop")).toEqual({ x: 100, y: 200 });
|
||||
expect(clickPoint(region, "rightBottom")).toEqual({ x: 160, y: 240 });
|
||||
expect(clickPoint(region, "top")).toEqual({ x: 130, y: 200 });
|
||||
expect(clickPoint(region, "left")).toEqual({ x: 100, y: 220 });
|
||||
});
|
||||
|
||||
it("supports custom offset JSON", () => {
|
||||
expect(clickPoint(region, '{"x":5,"y":6}')).toEqual({ x: 105, y: 206 });
|
||||
});
|
||||
|
||||
it("falls back to center for unknown values", () => {
|
||||
expect(clickPoint(region, "bogus")).toEqual({ x: 130, y: 220 });
|
||||
});
|
||||
});
|
||||
@ -10,12 +10,123 @@ function text(args: Record<string, unknown>, keys: string[], fallback = ""): str
|
||||
return fallback;
|
||||
}
|
||||
|
||||
register("Image_Screenshot", async (_ctx, node, args) => {
|
||||
async function capturePrimaryScreenPng(): Promise<PNG> {
|
||||
const { desktopCapturer, screen } = await import("electron");
|
||||
const primary = screen.getPrimaryDisplay();
|
||||
const sources = await desktopCapturer.getSources({
|
||||
types: ["screen"],
|
||||
thumbnailSize: { width: primary.size.width, height: primary.size.height }
|
||||
});
|
||||
const source = sources[0];
|
||||
if (!source) throw new RuntimeError("desktop_error", "无法获取屏幕截图");
|
||||
return PNG.sync.read(source.thumbnail.toPNG());
|
||||
}
|
||||
|
||||
/** 解析区域 JSON：对象 {x,y,width,height}（兼容 left/top、x1y1x2y2 角点）或数组 [x,y,width,height]，并夹取到图片边界内 */
|
||||
export function parseRegion(raw: string, image: PNG): { x: number; y: number; width: number; height: number } {
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(raw);
|
||||
} catch {
|
||||
throw new RuntimeError("validation_error", "区域 JSON 无法解析", { detail: raw.slice(0, 200) });
|
||||
}
|
||||
const numberOf = (value: unknown): number | null => {
|
||||
const n = Number(value);
|
||||
return Number.isFinite(n) ? n : null;
|
||||
};
|
||||
if (Array.isArray(parsed) && parsed.length === 4) {
|
||||
const [x, y, width, height] = parsed.map(numberOf);
|
||||
if (x !== null && y !== null && width !== null && height !== null) {
|
||||
return clampRegion({ x, y, width, height }, image);
|
||||
}
|
||||
}
|
||||
if (parsed && typeof parsed === "object") {
|
||||
const record = parsed as Record<string, unknown>;
|
||||
const x = numberOf(record.x ?? record.left);
|
||||
const y = numberOf(record.y ?? record.top);
|
||||
const width = numberOf(record.width);
|
||||
const height = numberOf(record.height);
|
||||
if (x !== null && y !== null && width !== null && height !== null) {
|
||||
return clampRegion({ x, y, width, height }, image);
|
||||
}
|
||||
const x2 = numberOf(record.x2 ?? record.right);
|
||||
const y2 = numberOf(record.y2 ?? record.bottom);
|
||||
if (x !== null && y !== null && x2 !== null && y2 !== null) {
|
||||
return clampRegion({ x, y, width: x2 - x, height: y2 - y }, image);
|
||||
}
|
||||
}
|
||||
throw new RuntimeError("validation_error", "区域 JSON 不符合要求", {
|
||||
detail: raw.slice(0, 200),
|
||||
recommendation: '使用 {"x":0,"y":0,"width":100,"height":80} 或 [x, y, width, height]'
|
||||
});
|
||||
}
|
||||
|
||||
export function clampRegion(region: { x: number; y: number; width: number; height: number }, image: PNG): { x: number; y: number; width: number; height: number } {
|
||||
const x = Math.max(0, Math.min(Math.round(region.x), image.width - 1));
|
||||
const y = Math.max(0, Math.min(Math.round(region.y), image.height - 1));
|
||||
const width = Math.max(1, Math.min(Math.round(region.width), image.width - x));
|
||||
const height = Math.max(1, Math.min(Math.round(region.height), image.height - y));
|
||||
return { x, y, width, height };
|
||||
}
|
||||
|
||||
export function cropPng(source: PNG, region: { x: number; y: number; width: number; height: number }): PNG {
|
||||
const cropped = new PNG({ width: region.width, height: region.height });
|
||||
PNG.bitblt(source, cropped, region.x, region.y, region.width, region.height, 0, 0);
|
||||
return cropped;
|
||||
}
|
||||
|
||||
/** 目标路径容错：填的是文件夹（已存在或以分隔符结尾）时，自动往里存默认文件名 */
|
||||
async function resolveOutputPath(outputPath: string, defaultName: string): Promise<string> {
|
||||
if (/[\\/]$/.test(outputPath)) return path.join(outputPath, defaultName);
|
||||
try {
|
||||
const stat = await fs.stat(outputPath);
|
||||
if (stat.isDirectory()) return path.join(outputPath, defaultName);
|
||||
} catch {
|
||||
// 路径不存在说明是要新建的文件，原样返回
|
||||
}
|
||||
return outputPath;
|
||||
}
|
||||
|
||||
register("Image_Screenshot", async (ctx, node, args) => {
|
||||
const folder = text(args, ["outImagePath", "folder"]);
|
||||
const name = text(args, ["outImageName", "name"], `screenshot_${Date.now()}.png`);
|
||||
const output = path.resolve(text(args, ["image_path"], folder ? path.join(folder, name) : name));
|
||||
const imageArg = text(args, ["image_path"]);
|
||||
const source = text(args, ["pictureSource"], "1");
|
||||
const range = text(args, ["shotRange"], "1");
|
||||
|
||||
let image: PNG;
|
||||
let output: string;
|
||||
if (source === "2") {
|
||||
// 指定图片文件来源：image_path 为输入图片；输出走 输出目录+文件名，两者均未配置时直接返回源文件
|
||||
if (!imageArg.trim()) {
|
||||
throw new RuntimeError("validation_error", "图像来源为「指定图片文件」时，目标文件路径需填写源图片路径", {
|
||||
recommendation: "在目标文件路径选择一张已存在的图片，或将图像来源改回「当前屏幕」"
|
||||
});
|
||||
}
|
||||
const sourceFile = path.resolve(imageArg);
|
||||
image = PNG.sync.read(await fs.readFile(sourceFile));
|
||||
output = folder ? path.resolve(path.join(folder, name)) : sourceFile;
|
||||
} else {
|
||||
// 当前屏幕来源：image_path 即保存路径（文件夹则自动追加默认文件名），缺省用 输出目录+文件名；再缺省落流程证据目录（路径可预期）
|
||||
image = await capturePrimaryScreenPng();
|
||||
output = await resolveOutputPath(
|
||||
path.resolve(imageArg || (folder ? path.join(folder, name) : path.join(ctx.evidenceDir, name))),
|
||||
name
|
||||
);
|
||||
}
|
||||
|
||||
if (range === "2") {
|
||||
const regionRaw = text(args, ["position", "action", "position_img"]);
|
||||
if (!regionRaw.trim()) {
|
||||
throw new RuntimeError("validation_error", "截图范围为「框选区域」时，position 需填写区域 JSON", {
|
||||
recommendation: '在 position 填写 {"x":0,"y":0,"width":100,"height":80} 或 [x, y, width, height]，或将截图范围改回「整个屏幕」'
|
||||
});
|
||||
}
|
||||
image = cropPng(image, parseRegion(regionRaw, image));
|
||||
}
|
||||
|
||||
await fs.mkdir(path.dirname(output), { recursive: true });
|
||||
await powershell(`Add-Type -AssemblyName System.Drawing; Add-Type -AssemblyName System.Windows.Forms; $bounds = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds; $bitmap = New-Object System.Drawing.Bitmap $bounds.Width,$bounds.Height; $graphics = [System.Drawing.Graphics]::FromImage($bitmap); $graphics.CopyFromScreen($bounds.Location,[System.Drawing.Point]::Empty,$bounds.Size); $bitmap.Save(${psString(output)},[System.Drawing.Imaging.ImageFormat]::Png); $graphics.Dispose(); $bitmap.Dispose()`);
|
||||
await fs.writeFile(output, PNG.sync.write(image));
|
||||
return output;
|
||||
});
|
||||
|
||||
@ -23,10 +134,46 @@ interface MatchResult { x: number; y: number; width: number; height: number; sco
|
||||
|
||||
async function captureScreen(target: string): Promise<string> {
|
||||
await fs.mkdir(path.dirname(target), { recursive: true });
|
||||
await powershell(`Add-Type -AssemblyName System.Drawing; Add-Type -AssemblyName System.Windows.Forms; $bounds = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds; $bitmap = New-Object System.Drawing.Bitmap $bounds.Width,$bounds.Height; $graphics = [System.Drawing.Graphics]::FromImage($bitmap); $graphics.CopyFromScreen($bounds.Location,[System.Drawing.Point]::Empty,$bounds.Size); $bitmap.Save(${psString(target)},[System.Drawing.Imaging.ImageFormat]::Png); $graphics.Dispose(); $bitmap.Dispose()`);
|
||||
await fs.writeFile(target, PNG.sync.write(await capturePrimaryScreenPng()));
|
||||
return target;
|
||||
}
|
||||
|
||||
interface WindowRect { x: number; y: number; width: number; height: number }
|
||||
|
||||
/** 按标题（精确→包含）或进程名查找窗口，前置并返回窗口矩形（屏幕坐标） */
|
||||
async function focusWindowRect(windowObj: string): Promise<WindowRect> {
|
||||
const quoted = psString(windowObj);
|
||||
const stdout = await powershell(`Add-Type @'
|
||||
using System; using System.Runtime.InteropServices;
|
||||
public static class WinRect {
|
||||
[DllImport("user32.dll")] public static extern IntPtr FindWindow(string cls, string title);
|
||||
[DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr h);
|
||||
[DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr h, int cmd);
|
||||
[DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr h, out RECT r);
|
||||
public struct RECT { public int Left; public int Top; public int Right; public int Bottom; }
|
||||
}
|
||||
'@;
|
||||
$h = [WinRect]::FindWindow($null, ${quoted});
|
||||
if ($h -eq [IntPtr]::Zero) { $p = Get-Process | Where-Object { $_.MainWindowTitle -and ($_.MainWindowTitle -eq ${quoted} -or $_.MainWindowTitle.Contains(${quoted})) } | Select-Object -First 1; if ($p) { $h = $p.MainWindowHandle } }
|
||||
if ($h -eq [IntPtr]::Zero) { Write-Output 'NOT_FOUND'; exit }
|
||||
[WinRect]::ShowWindow($h, 9) | Out-Null;
|
||||
[WinRect]::SetForegroundWindow($h) | Out-Null;
|
||||
Start-Sleep -Milliseconds 300;
|
||||
$r = New-Object WinRect+RECT;
|
||||
[WinRect]::GetWindowRect($h, [ref]$r) | Out-Null;
|
||||
Write-Output ("{0},{1},{2},{3}" -f $r.Left, $r.Top, $r.Right, $r.Bottom)`);
|
||||
const line = stdout.trim().split(/\r?\n/).pop() ?? "";
|
||||
if (line === "NOT_FOUND") {
|
||||
throw new RuntimeError("not_found", `未找到窗口「${windowObj}」`, { recommendation: "确认窗口已打开，窗口对象可填完整标题、标题包含关键字或进程名" });
|
||||
}
|
||||
const parts = line.split(",").map(Number);
|
||||
if (parts.length !== 4 || parts.some((v) => !Number.isFinite(v))) {
|
||||
throw new RuntimeError("desktop_error", "窗口矩形获取失败", { detail: line.slice(0, 120) });
|
||||
}
|
||||
const [left, top, right, bottom] = parts;
|
||||
return { x: left, y: top, width: Math.max(1, right - left), height: Math.max(1, bottom - top) };
|
||||
}
|
||||
|
||||
function pixelDifference(screen: PNG, template: PNG, offsetX: number, offsetY: number, sampleStep: number, cutoff: number): number {
|
||||
let difference = 0;
|
||||
let samples = 0;
|
||||
@ -80,18 +227,104 @@ async function locate(args: Record<string, unknown>, evidenceDir: string, nodeId
|
||||
const rawTemplate = text(args, ["image", "image_path", "template"]);
|
||||
if (!rawTemplate.trim()) throw new RuntimeError("validation_error", "模板图片路径为空", { nodeId });
|
||||
const template = path.resolve(rawTemplate);
|
||||
|
||||
// 查找范围：1 整个屏幕（默认）/ 2 指定窗口 / 3 指定区域；窗口/区域模式下结果坐标会平移回屏幕坐标系
|
||||
const rangeMode = text(args, ["find_range"], "1");
|
||||
const screenshot = path.join(evidenceDir, `${nodeId}_screen.png`);
|
||||
await captureScreen(screenshot);
|
||||
let offsetX = 0;
|
||||
let offsetY = 0;
|
||||
if (rangeMode === "2") {
|
||||
const windowObj = text(args, ["window_obj"]);
|
||||
if (!windowObj.trim()) throw new RuntimeError("validation_error", "查找范围为「指定窗口」时，窗口对象不能为空", { nodeId, recommendation: "填写窗口标题（或标题关键字、进程名）" });
|
||||
const rect = await focusWindowRect(windowObj);
|
||||
const screen = await capturePrimaryScreenPng();
|
||||
await fs.writeFile(screenshot, PNG.sync.write(cropPng(screen, clampRegion(rect, screen))));
|
||||
offsetX = rect.x;
|
||||
offsetY = rect.y;
|
||||
} else if (rangeMode === "3") {
|
||||
const regionRaw = text(args, ["shot_position", "action", "position"]);
|
||||
if (!regionRaw.trim()) throw new RuntimeError("validation_error", "查找范围为「指定区域」时，需要填写区域 JSON", { nodeId, recommendation: '在 shot_position 填写 {"x":0,"y":0,"width":100,"height":80} 或 [x, y, width, height]' });
|
||||
const screen = await capturePrimaryScreenPng();
|
||||
const region = clampRegion(parseRegion(regionRaw, screen), screen);
|
||||
await fs.writeFile(screenshot, PNG.sync.write(cropPng(screen, region)));
|
||||
offsetX = region.x;
|
||||
offsetY = region.y;
|
||||
} else {
|
||||
await captureScreen(screenshot);
|
||||
}
|
||||
|
||||
const result = await findImage(template, screenshot, Number(text(args, ["accuracy"], "0.2")));
|
||||
if (!result) throw new RuntimeError("not_found", "屏幕中未找到目标图像", { nodeId, recommendation: "重新截取模板，或调低匹配精度" });
|
||||
return result;
|
||||
return { ...result, x: result.x + offsetX, y: result.y + offsetY };
|
||||
}
|
||||
|
||||
register(["Image_Find", "Image_Find_V1"], async (ctx, node, args) => locate(args, ctx.evidenceDir, node.id));
|
||||
/** 点击位置锚点（相对匹配区域的比例坐标），支持九宫格取值 */
|
||||
const POSITION_ANCHORS: Record<string, [number, number]> = {
|
||||
center: [0.5, 0.5],
|
||||
leftTop: [0, 0], top: [0.5, 0], centerTop: [0.5, 0], rightTop: [1, 0],
|
||||
left: [0, 0.5], centerLeft: [0, 0.5], right: [1, 0.5], centerRight: [1, 0.5],
|
||||
leftBottom: [0, 1], bottom: [0.5, 1], centerBottom: [0.5, 1], rightBottom: [1, 1]
|
||||
};
|
||||
|
||||
export function clickPoint(result: MatchResult, position: string): { x: number; y: number } {
|
||||
const raw = position.trim();
|
||||
if (raw && raw.toLowerCase() !== "center") {
|
||||
if (raw.startsWith("{")) {
|
||||
try {
|
||||
const offset = JSON.parse(raw) as { x?: unknown; y?: unknown };
|
||||
const dx = Number(offset.x);
|
||||
const dy = Number(offset.y);
|
||||
if (Number.isFinite(dx) && Number.isFinite(dy)) {
|
||||
return { x: Math.round(result.x + dx), y: Math.round(result.y + dy) };
|
||||
}
|
||||
} catch {
|
||||
// 非法 JSON 落回中心
|
||||
}
|
||||
}
|
||||
const anchor = POSITION_ANCHORS[raw] ?? POSITION_ANCHORS[raw.toLowerCase()];
|
||||
if (anchor) {
|
||||
return { x: Math.round(result.x + result.width * anchor[0]), y: Math.round(result.y + result.height * anchor[1]) };
|
||||
}
|
||||
}
|
||||
return { x: Math.round(result.x + result.width / 2), y: Math.round(result.y + result.height / 2) };
|
||||
}
|
||||
|
||||
const CLICK_ACTIONS: Record<string, string> = {
|
||||
left: "[Mouse]::mouse_event(2,0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 30; [Mouse]::mouse_event(4,0,0,0,[UIntPtr]::Zero)",
|
||||
right: "[Mouse]::mouse_event(8,0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 30; [Mouse]::mouse_event(16,0,0,0,[UIntPtr]::Zero)",
|
||||
middle: "[Mouse]::mouse_event(32,0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 30; [Mouse]::mouse_event(64,0,0,0,[UIntPtr]::Zero)",
|
||||
double: "[Mouse]::mouse_event(2,0,0,0,[UIntPtr]::Zero); [Mouse]::mouse_event(4,0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 60; [Mouse]::mouse_event(2,0,0,0,[UIntPtr]::Zero); [Mouse]::mouse_event(4,0,0,0,[UIntPtr]::Zero)"
|
||||
};
|
||||
|
||||
async function injectClick(x: number, y: number, clickKey: string, showTrack: boolean): Promise<void> {
|
||||
const action = CLICK_ACTIONS[clickKey] ?? CLICK_ACTIONS.left;
|
||||
// show_track=1 时从当前光标位置平滑移动过去（20 步插值），否则直接跳转
|
||||
const move = showTrack
|
||||
? `for ($i = 1; $i -le 20; $i++) { [Mouse]::SetCursorPos([int]($p.X + (${x} - $p.X) * $i / 20), [int]($p.Y + (${y} - $p.Y) * $i / 20)); Start-Sleep -Milliseconds 12 }`
|
||||
: `[Mouse]::SetCursorPos(${x}, ${y})`;
|
||||
await powershell(`Add-Type @'
|
||||
using System; using System.Runtime.InteropServices;
|
||||
public static class Mouse {
|
||||
[DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y);
|
||||
[DllImport("user32.dll")] public static extern bool GetCursorPos(out POINT p);
|
||||
[DllImport("user32.dll")] public static extern void mouse_event(uint f, uint x, uint y, uint d, UIntPtr e);
|
||||
public struct POINT { public int X; public int Y; }
|
||||
}
|
||||
'@;
|
||||
$p = New-Object Mouse+POINT;
|
||||
[Mouse]::GetCursorPos([ref]$p) | Out-Null;
|
||||
${move};
|
||||
Start-Sleep -Milliseconds 60;
|
||||
${action}`);
|
||||
}
|
||||
|
||||
register(["Image_Click", "Image_Click_V1", "Image_Click_V2", "Image_Click_V3"], async (ctx, node, args) => {
|
||||
const result = await locate(args, ctx.evidenceDir, node.id);
|
||||
const x = Math.round(result.x + result.width / 2);
|
||||
const y = Math.round(result.y + result.height / 2);
|
||||
await powershell(`Add-Type @'\nusing System; using System.Runtime.InteropServices; public static class Clicker { [DllImport("user32.dll")] public static extern bool SetCursorPos(int X,int Y); [DllImport("user32.dll")] public static extern void mouse_event(uint f,uint x,uint y,uint d,UIntPtr e); }\n'@; [Clicker]::SetCursorPos(${x},${y}); [Clicker]::mouse_event(2,0,0,0,[UIntPtr]::Zero); [Clicker]::mouse_event(4,0,0,0,[UIntPtr]::Zero)`);
|
||||
return { ...result, click: { x, y } };
|
||||
const point = clickPoint(result, text(args, ["position"], "center"));
|
||||
const clickKey = text(args, ["click_key"], "left");
|
||||
const showTrack = String(args.show_track ?? "0") === "1" || args.show_track === true;
|
||||
await injectClick(point.x, point.y, clickKey, showTrack);
|
||||
ctx.log(`已${clickKey === "right" ? "右键" : clickKey === "double" ? "双击" : "点击"}图像 (${point.x}, ${point.y})，匹配得分 ${result.score.toFixed(3)}`, "info", node.id);
|
||||
return { ...result, click: point };
|
||||
});
|
||||
|
||||
@ -481,6 +481,21 @@ function fileFiltersFor(componentId: string, field: CatalogField): FileFilterOpt
|
||||
return filters;
|
||||
}
|
||||
|
||||
/** 输出字段的保存框默认文件名：取第一个非「全部文件」过滤器的首个扩展名 */
|
||||
function defaultSaveFilename(filters: FileFilterOption[]): string {
|
||||
const ext = filters.find((f) => !f.extensions.includes("*"))?.extensions[0];
|
||||
return ext ? `未命名.${ext}` : "";
|
||||
}
|
||||
|
||||
/** 保存结果没有扩展名时补上所选类型的扩展名（Windows 保存框不会自动补） */
|
||||
function ensureSaveExtension(filePath: string, filters: FileFilterOption[]): string {
|
||||
const exts = filters.find((f) => !f.extensions.includes("*"))?.extensions ?? [];
|
||||
if (!exts.length) return filePath;
|
||||
const clean = filePath.trim();
|
||||
if (/\.[A-Za-z0-9]+$/.test(clean)) return clean;
|
||||
return `${clean}.${exts[0]}`;
|
||||
}
|
||||
|
||||
function FieldControl({
|
||||
field,
|
||||
node,
|
||||
@ -526,6 +541,26 @@ function FieldControl({
|
||||
|
||||
const pickPath = async (directories: boolean) => {
|
||||
try {
|
||||
if (!directories) {
|
||||
// 输出类字段（保存/输出/目标文件等）指向将来才生成的文件，用「保存」对话框允许输入新文件名；
|
||||
// 输入类字段保持「打开」对话框，只能选择已存在的文件。
|
||||
const isOutputField = /输出|保存|另存|导出|目标文件|文件名|output|save|export|dest|target/i.test(
|
||||
`${field.key} ${field.label ?? ""}`
|
||||
);
|
||||
if (isOutputField) {
|
||||
const filters = fileFiltersFor(node.componentId, field);
|
||||
// 预填默认文件名（按过滤器推断扩展名），避免弹出的保存框文件名为空没法直接点保存
|
||||
const defaultPath = textOf(raw) || defaultSaveFilename(filters);
|
||||
const selected = await window.rpa.dialog.save({
|
||||
title: field.label,
|
||||
defaultPath: defaultPath || undefined,
|
||||
filters
|
||||
});
|
||||
// Windows 保存框不会按所选类型自动补扩展名，这里补上
|
||||
if (selected) onChange(field.key, ensureSaveExtension(selected, filters));
|
||||
return;
|
||||
}
|
||||
}
|
||||
const selected = await window.rpa.dialog.open({
|
||||
title: field.label,
|
||||
directories,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user