From c11e1d15271f865dccff970d60fbc0872d312c87 Mon Sep 17 00:00:00 2001 From: yu <191972707@qq.com> Date: Mon, 2 Nov 2026 14:25:54 +0800 Subject: [PATCH 1/8] =?UTF-8?q?fix:=20=E8=BE=93=E5=87=BA=E7=B1=BB=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E5=AD=97=E6=AE=B5=E6=94=B9=E7=94=A8=E4=BF=9D=E5=AD=98?= =?UTF-8?q?=E5=AF=B9=E8=AF=9D=E6=A1=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 目标文件路径/输出目录这类字段指向将来才生成的文件, 原先的打开文件对话框只能选已存在文件, 导致截屏保存位置无法选择 - 字段键/标签命中 输出|保存|另存|导出|目标文件|output/save/export 等关键词时改用保存对话框, 可直接输入新文件名 - 输入类字段保持打开对话框不变 --- src/renderer/src/panels/Properties.tsx | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/renderer/src/panels/Properties.tsx b/src/renderer/src/panels/Properties.tsx index 967fb96..cf8deb5 100644 --- a/src/renderer/src/panels/Properties.tsx +++ b/src/renderer/src/panels/Properties.tsx @@ -526,6 +526,22 @@ function FieldControl({ const pickPath = async (directories: boolean) => { try { + if (!directories) { + // 输出类字段（保存/输出/目标文件等）指向将来才生成的文件，用「保存」对话框允许输入新文件名； + // 输入类字段保持「打开」对话框，只能选择已存在的文件。 + const isOutputField = /输出|保存|另存|导出|目标文件|文件名|output|save|export|dest|target/i.test( + `${field.key} ${field.label ?? ""}` + ); + if (isOutputField) { + const selected = await window.rpa.dialog.save({ + title: field.label, + defaultPath: textOf(raw) || undefined, + filters: fileFiltersFor(node.componentId, field) + }); + if (selected) onChange(field.key, selected); + return; + } + } const selected = await window.rpa.dialog.open({ title: field.label, directories, From 342145cc467ad43854a596ebd8090855cda1623b Mon Sep 17 00:00:00 2001 From: yu <191972707@qq.com> Date: Mon, 2 Nov 2026 14:36:14 +0800 Subject: [PATCH 2/8] =?UTF-8?q?feat:=20=E5=B1=8F=E5=B9=95=E6=88=AA?= =?UTF-8?q?=E5=9B=BE=E7=BB=84=E4=BB=B6=E5=AE=9E=E7=8E=B0=E5=9B=BE=E5=83=8F?= =?UTF-8?q?=E6=9D=A5=E6=BA=90=E4=B8=8E=E6=88=AA=E5=9B=BE=E8=8C=83=E5=9B=B4?= =?UTF-8?q?=EF=BC=88=E5=8E=9F=20P2=20=E9=97=AE=E9=A2=98=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - vision.ts 重写 Image_Screenshot：pictureSource=1 当前屏幕(desktopCapturer)/2 指定图片文件(image_path 作输入, 输出走输出目录+文件名, 未配置则原样返回)；shotRange=2 框选区域按 position/action 区域 JSON 裁剪(pngjs bitblt)，区域支持 {x,y,width,height}/[x,y,w,h]/left,top/角点写法并夹取边界 - 移除 browser.ts 重复注册，实现单点维护 - catalog: pictureSource/shotRange 补下拉选项(当前屏幕/指定图片文件, 整个屏幕/框选区域) - typecheck 通过；browser/cv-vision/capability-audit/app-system 回归 50 项全过 --- resources/catalog.json | 24 +++++++- src/main/capabilities/browser.ts | 19 +----- src/main/capabilities/vision.ts | 100 ++++++++++++++++++++++++++++++- 3 files changed, 121 insertions(+), 22 deletions(-) diff --git a/resources/catalog.json b/resources/catalog.json index c87626f..1670f59 100644 --- a/resources/catalog.json +++ b/resources/catalog.json @@ -61865,7 +61865,17 @@ "placeholder": "", "description": "选择要截图的图像来源，可以是当前屏幕，也可以是指定的图片文件。", "advanced": false, - "valueTypes": [] + "valueTypes": [], + "options": [ + { + "label": "当前屏幕", + "value": "1" + }, + { + "label": "指定图片文件", + "value": "2" + } + ] }, { "key": "shotRange", @@ -61878,7 +61888,17 @@ "placeholder": "", "description": "选择要截图的范围，可以使截取整个屏幕，或手动选择截图区域", "advanced": false, - "valueTypes": [] + "valueTypes": [], + "options": [ + { + "label": "整个屏幕", + "value": "1" + }, + { + "label": "框选区域", + "value": "2" + } + ] }, { "key": "image_path", diff --git a/src/main/capabilities/browser.ts b/src/main/capabilities/browser.ts index a210376..173ccc3 100644 --- a/src/main/capabilities/browser.ts +++ b/src/main/capabilities/browser.ts @@ -178,21 +178,4 @@ register("WebBrowser_Screenshot", async (ctx, node, args) => { return file; }); -register("Image_Screenshot", async (ctx, node, args) => { - const { desktopCapturer, screen } = await import("electron"); - const folder = String(args.outImagePath ?? args.image_path ?? ctx.evidenceDir); - const name = String(args.outImageName ?? `${node.id}.png`); - const { promises: fs } = await import("node:fs"); - const path = await import("node:path"); - const target = path.resolve(folder.endsWith(".png") ? folder : path.join(folder, name)); - const primary = screen.getPrimaryDisplay(); - const sources = await desktopCapturer.getSources({ - types: ["screen"], - thumbnailSize: { width: primary.size.width, height: primary.size.height } - }); - const source = sources[0]; - if (!source) throw new RuntimeError("desktop_error", "无法获取屏幕截图"); - await fs.mkdir(path.dirname(target), { recursive: true }); - await fs.writeFile(target, source.thumbnail.toPNG()); - return target; -}); +// Image_Screenshot 统一在 vision.ts 实现（屏幕/文件来源 + 区域裁剪），此处不再重复注册 diff --git a/src/main/capabilities/vision.ts b/src/main/capabilities/vision.ts index 15fb7c5..68b5df5 100644 --- a/src/main/capabilities/vision.ts +++ b/src/main/capabilities/vision.ts @@ -10,12 +10,108 @@ function text(args: Record, keys: string[], fallback = ""): str return fallback; } +async function capturePrimaryScreenPng(): Promise { + const { desktopCapturer, screen } = await import("electron"); + const primary = screen.getPrimaryDisplay(); + const sources = await desktopCapturer.getSources({ + types: ["screen"], + thumbnailSize: { width: primary.size.width, height: primary.size.height } + }); + const source = sources[0]; + if (!source) throw new RuntimeError("desktop_error", "无法获取屏幕截图"); + return PNG.sync.read(source.thumbnail.toPNG()); +} + +/** 解析区域 JSON：对象 {x,y,width,height}（兼容 left/top、x1y1x2y2 角点）或数组 [x,y,width,height]，并夹取到图片边界内 */ +function parseRegion(raw: string, image: PNG): { x: number; y: number; width: number; height: number } { + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + throw new RuntimeError("validation_error", "区域 JSON 无法解析", { detail: raw.slice(0, 200) }); + } + const numberOf = (value: unknown): number | null => { + const n = Number(value); + return Number.isFinite(n) ? n : null; + }; + if (Array.isArray(parsed) && parsed.length === 4) { + const [x, y, width, height] = parsed.map(numberOf); + if (x !== null && y !== null && width !== null && height !== null) { + return clampRegion({ x, y, width, height }, image); + } + } + if (parsed && typeof parsed === "object") { + const record = parsed as Record; + const x = numberOf(record.x ?? record.left); + const y = numberOf(record.y ?? record.top); + const width = numberOf(record.width); + const height = numberOf(record.height); + if (x !== null && y !== null && width !== null && height !== null) { + return clampRegion({ x, y, width, height }, image); + } + const x2 = numberOf(record.x2 ?? record.right); + const y2 = numberOf(record.y2 ?? record.bottom); + if (x !== null && y !== null && x2 !== null && y2 !== null) { + return clampRegion({ x, y, width: x2 - x, height: y2 - y }, image); + } + } + throw new RuntimeError("validation_error", "区域 JSON 不符合要求", { + detail: raw.slice(0, 200), + recommendation: '使用 {"x":0,"y":0,"width":100,"height":80} 或 [x, y, width, height]' + }); +} + +function clampRegion(region: { x: number; y: number; width: number; height: number }, image: PNG): { x: number; y: number; width: number; height: number } { + const x = Math.max(0, Math.min(Math.round(region.x), image.width - 1)); + const y = Math.max(0, Math.min(Math.round(region.y), image.height - 1)); + const width = Math.max(1, Math.min(Math.round(region.width), image.width - x)); + const height = Math.max(1, Math.min(Math.round(region.height), image.height - y)); + return { x, y, width, height }; +} + +function cropPng(source: PNG, region: { x: number; y: number; width: number; height: number }): PNG { + const cropped = new PNG({ width: region.width, height: region.height }); + PNG.bitblt(source, cropped, region.x, region.y, region.width, region.height, 0, 0); + return cropped; +} + register("Image_Screenshot", async (_ctx, node, args) => { const folder = text(args, ["outImagePath", "folder"]); const name = text(args, ["outImageName", "name"], `screenshot_${Date.now()}.png`); - const output = path.resolve(text(args, ["image_path"], folder ? path.join(folder, name) : name)); + const imageArg = text(args, ["image_path"]); + const source = text(args, ["pictureSource"], "1"); + const range = text(args, ["shotRange"], "1"); + + let image: PNG; + let output: string; + if (source === "2") { + // 指定图片文件来源：image_path 为输入图片；输出走 输出目录+文件名，两者均未配置时直接返回源文件 + if (!imageArg.trim()) { + throw new RuntimeError("validation_error", "图像来源为「指定图片文件」时，目标文件路径需填写源图片路径", { + recommendation: "在目标文件路径选择一张已存在的图片，或将图像来源改回「当前屏幕」" + }); + } + const sourceFile = path.resolve(imageArg); + image = PNG.sync.read(await fs.readFile(sourceFile)); + output = folder ? path.resolve(path.join(folder, name)) : sourceFile; + } else { + // 当前屏幕来源：image_path 即保存路径，缺省用 输出目录+文件名 + image = await capturePrimaryScreenPng(); + output = path.resolve(imageArg || (folder ? path.join(folder, name) : name)); + } + + if (range === "2") { + const regionRaw = text(args, ["position", "action", "position_img"]); + if (!regionRaw.trim()) { + throw new RuntimeError("validation_error", "截图范围为「框选区域」时，position 需填写区域 JSON", { + recommendation: '在 position 填写 {"x":0,"y":0,"width":100,"height":80} 或 [x, y, width, height]，或将截图范围改回「整个屏幕」' + }); + } + image = cropPng(image, parseRegion(regionRaw, image)); + } + await fs.mkdir(path.dirname(output), { recursive: true }); - await powershell(`Add-Type -AssemblyName System.Drawing; Add-Type -AssemblyName System.Windows.Forms; $bounds = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds; $bitmap = New-Object System.Drawing.Bitmap $bounds.Width,$bounds.Height; $graphics = [System.Drawing.Graphics]::FromImage($bitmap); $graphics.CopyFromScreen($bounds.Location,[System.Drawing.Point]::Empty,$bounds.Size); $bitmap.Save(${psString(output)},[System.Drawing.Imaging.ImageFormat]::Png); $graphics.Dispose(); $bitmap.Dispose()`); + await fs.writeFile(output, PNG.sync.write(image)); return output; }); From ba17d6dc2a5ced8adf1ea91a4cf26a2542229ed6 Mon Sep 17 00:00:00 2001 From: yu <191972707@qq.com> Date: Mon, 2 Nov 2026 14:53:34 +0800 Subject: [PATCH 3/8] =?UTF-8?q?fix:=20=E5=9B=BE=E5=83=8F=E4=B8=8EOCR=20?= =?UTF-8?q?=E6=A3=80=E6=B5=8B=E6=8A=A5=E5=91=8A=E9=97=AE=E9=A2=98=E5=85=A8?= =?UTF-8?q?=E9=87=8F=E4=BF=AE=E5=A4=8D=EF=BC=88P0/P1/P2/P3=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - P0-1: GetOcrTextNew 传图键补 uiSelector, 组件恢复可用 - P1-1: Image_Click 全版本 position 生效(九宫格锚点+自定义偏移 JSON), catalog 补下拉选项 - P1-2: Click_V3 click_key 生效(左/右/中键/双击), catalog 补选项 - P1-3: GetOcrTable 按垂直中心聚合成行列(行内按 x 排序), showTable 写流程数据表(与 Bivartable 共享 __rpaDatabook), isExcel 用 exceljs 另存并处理 res/randomString 占位 - P2-1: Find_V1/Click_V3 find_range 生效(整个屏幕/指定窗口/指定区域), 窗口模式前置+取矩形裁剪, 坐标平移回屏幕系 - P2-2(补): Image_Screenshot 空路径兜底到流程证据目录 - P3-1: accuracy2 无特征匹配层实现, catalog 隐藏并注明 - P3-2: show_track=1 时鼠标 20 步插值平滑移动后点击 - P3-3: 截屏/Find 换 desktopCapturer(去掉逐次 Add-Type); 新增 vision-helpers.test.ts 12 项单测(区域解析/边界夹取/裁剪/九宫格) - 回归: typecheck 通过; vitest 256 过(4 失败为桌面/PDF 环境遗留); 报告文档已标注修复状态 --- docs/图像与OCR问题检测报告.md | 4 + resources/catalog.json | 136 ++++++++++++++++- src/main/capabilities/ocr-caps.ts | 64 +++++++- src/main/capabilities/vision-helpers.test.ts | 81 ++++++++++ src/main/capabilities/vision.ts | 148 +++++++++++++++++-- 5 files changed, 404 insertions(+), 29 deletions(-) create mode 100644 src/main/capabilities/vision-helpers.test.ts diff --git a/docs/图像与OCR问题检测报告.md b/docs/图像与OCR问题检测报告.md index 244af9b..114c12f 100644 --- a/docs/图像与OCR问题检测报告.md +++ b/docs/图像与OCR问题检测报告.md @@ -7,6 +7,10 @@ 1. 静态核对——组件目录（catalog.json）字段定义 vs 实现代码（vision.ts / ocr-caps.ts）逐键比对 2. 真实运行——GDI 生成含中英文的测试图、真实截屏、模板匹配直测、OCR 引擎实识别（飞桨 @gutenye/ocr-node） - **说明**：本次仅检测未修改任何代码，问题留给同事修复。 +- **修复状态（2026-11-02 更新）**：报告中的全部问题已由刘爱家修复完成—— + - 提交 `342145c`：P2-2（屏幕截图图像来源/截图范围，含区域裁剪与下拉选项） + - 提交 `<本次>`：P0-1（GetOcrTextNew 传图键）、P1-1（点击位置九宫格/偏移）、P1-2（鼠标按键左/右/中/双击）、P1-3（GetOcrTable 行列聚类 + 写数据表 + 存 Excel）、P2-1（查找范围指定窗口/指定区域）、P3-2（鼠标移动轨迹动画）、P3-3（截图空路径落证据目录；表格行内按 x 排序）；P3-1（accuracy2）无特征匹配层实现，已在目录中隐藏并注明原因 + - 回归：typecheck 通过；vitest 256 过（4 失败为与本次无关的桌面/PDF 环境遗留项）；新增 vision-helpers 单测 12 项覆盖区域解析/裁剪/九宫格 --- diff --git a/resources/catalog.json b/resources/catalog.json index 1670f59..af5e8ab 100644 --- a/resources/catalog.json +++ b/resources/catalog.json @@ -61583,7 +61583,45 @@ "placeholder": "", "description": "", "advanced": false, - "valueTypes": [] + "valueTypes": [], + "options": [ + { + "label": "中心", + "value": "center" + }, + { + "label": "左上", + "value": "leftTop" + }, + { + "label": "上", + "value": "top" + }, + { + "label": "右上", + "value": "rightTop" + }, + { + "label": "左", + "value": "left" + }, + { + "label": "右", + "value": "right" + }, + { + "label": "左下", + "value": "leftBottom" + }, + { + "label": "下", + "value": "bottom" + }, + { + "label": "右下", + "value": "rightBottom" + } + ] }, { "key": "accuracy", @@ -61603,13 +61641,13 @@ { "key": "accuracy2", "label": "特征相似度", - "widget": "text", + "widget": "hidden", "type": "string", "displayType": "text-expression", "required": true, "defaultValue": "0.2", "placeholder": "请输入相似度阈值", - "description": "即需要查找的图片与模板图片的针对局部图像特征相似程度的阈值指标，与模板相似度共同决定组件运行结果。用户可自定义配置一个0~1区间的相似度取值（如0.55即55%相似度），0为完全不相似、1为完全一致，越大则命中阈值越高。（如无特殊需求，建议默认0.2）", + "description": "【暂未启用】当前实现为纯像素模板匹配，无特征匹配层；字段保留以兼容旧流程", "advanced": false, "valueTypes": [ "rpa_num" @@ -62791,7 +62829,21 @@ "placeholder": "", "description": "请选择查找目标图像的范围", "advanced": false, - "valueTypes": [] + "valueTypes": [], + "options": [ + { + "label": "整个屏幕", + "value": "1" + }, + { + "label": "指定窗口", + "value": "2" + }, + { + "label": "指定区域", + "value": "3" + } + ] }, { "key": "window_obj", @@ -63297,7 +63349,21 @@ "placeholder": "", "description": "请选择查找目标图像的范围", "advanced": false, - "valueTypes": [] + "valueTypes": [], + "options": [ + { + "label": "整个屏幕", + "value": "1" + }, + { + "label": "指定窗口", + "value": "2" + }, + { + "label": "指定区域", + "value": "3" + } + ] }, { "key": "window_obj", @@ -63366,7 +63432,17 @@ "placeholder": "", "description": "", "advanced": false, - "valueTypes": [] + "valueTypes": [], + "options": [ + { + "label": "左键", + "value": "left" + }, + { + "label": "右键", + "value": "right" + } + ] }, { "key": "position", @@ -63379,7 +63455,45 @@ "placeholder": "", "description": "", "advanced": false, - "valueTypes": [] + "valueTypes": [], + "options": [ + { + "label": "中心", + "value": "center" + }, + { + "label": "左上", + "value": "leftTop" + }, + { + "label": "上", + "value": "top" + }, + { + "label": "右上", + "value": "rightTop" + }, + { + "label": "左", + "value": "left" + }, + { + "label": "右", + "value": "right" + }, + { + "label": "左下", + "value": "leftBottom" + }, + { + "label": "下", + "value": "bottom" + }, + { + "label": "右下", + "value": "rightBottom" + } + ] }, { "key": "accuracy", @@ -87855,7 +87969,13 @@ "placeholder": "", "description": "", "advanced": false, - "valueTypes": [] + "valueTypes": [], + "options": [ + { + "label": "本地图片", + "value": "0" + } + ] }, { "key": "uiSelector", diff --git a/src/main/capabilities/ocr-caps.ts b/src/main/capabilities/ocr-caps.ts index 932a17f..e2bef41 100644 --- a/src/main/capabilities/ocr-caps.ts +++ b/src/main/capabilities/ocr-caps.ts @@ -1,3 +1,5 @@ +import path from "node:path"; +import { promises as fs } from "node:fs"; import { register } from "../engine/registry"; import { RuntimeError } from "../engine/types"; @@ -15,7 +17,8 @@ async function getOcr(): Promise<{ detect: (image: string) => Promise<{ texts: R } function requireImage(args: Record, nodeId: string): string { - for (const key of ["image_path", "imagePath", "file_path", "filePath", "path", "image", "pictureSource"]) { + // uiSelector 是 OcrOffline_GetOcrTextNew 目录里唯一的图片字段（P0-1 修复） + for (const key of ["image_path", "imagePath", "file_path", "filePath", "path", "image", "pictureSource", "uiSelector"]) { const v = String(args[key] ?? "").trim(); if (v) return v; } @@ -54,16 +57,61 @@ register("OcrOffline_GetOcrTextPosition", async (ctx, node, args) => { return words; }); +/** 按垂直中心聚类成行（行高 60% 为阈值），行内按 x 排序，返回二维文本表格 */ +function groupOcrRows(words: RapidOcrLine[]): string[][] { + const prepared = words + .filter((word) => String(word.text ?? "").trim()) + .map((word) => { + const box = word.box ?? []; + const xs = box.map((point) => point[0]); + const ys = box.map((point) => point[1]); + const x = xs.length ? Math.min(...xs) : 0; + const y = ys.length ? Math.min(...ys) : 0; + const height = ys.length ? Math.max(...ys) - y : 0; + return { text: word.text, x, cy: y + height / 2, h: height }; + }) + .sort((a, b) => a.cy - b.cy || a.x - b.x); + const rows: Array<{ cy: number; h: number; cells: typeof prepared }> = []; + let current: (typeof rows)[number] | null = null; + for (const item of prepared) { + if (!current || Math.abs(item.cy - current.cy) > Math.max(current.h, item.h, 8) * 0.6) { + current = { cy: item.cy, h: item.h, cells: [] }; + rows.push(current); + } + current.cells.push(item); + } + return rows.map((row) => row.cells.sort((a, b) => a.x - b.x).map((cell) => cell.text.trim())); +} + register("OcrOffline_GetOcrTable", async (ctx, node, args) => { const imagePath = requireImage(args, node.id); const ocr = await getOcr(); const result = await ocr.detect(imagePath); - const sorted = [...result.texts].sort((a, b) => { - const ay = a.box ? Math.min(...a.box.map(p => p[1])) : 0; - const by = b.box ? Math.min(...b.box.map(p => p[1])) : 0; - return ay - by; - }); - const rows = sorted.map(t => t.text); + const rows = groupOcrRows(result.texts); ctx.log("OCR 表格识别完成：" + rows.length + " 行", "info", node.id); - return rows; + + // showTable=1：写入流程数据表（与 Bivartable 系列组件共享同一份数据簿） + if (String(args.showTable ?? "1") === "1") { + ctx.vars["__rpaDatabook"] = rows; + ctx.log("已写入流程数据表", "info", node.id); + } + + // isExcel=1：另存为 Excel 文件 + let excelFile = ""; + if (String(args.isExcel ?? "0") === "1") { + const ExcelJS = (await import("exceljs")).default; + const workbook = new ExcelJS.Workbook(); + const sheet = workbook.addWorksheet("Sheet1"); + sheet.addRows(rows); + const rawFolder = String(args.excelPath ?? "").trim(); + // ${res} 流程资源目录占位与空值统一落到运行证据目录 + const folder = rawFolder && rawFolder !== "${res}" ? rawFolder : ctx.evidenceDir; + const fileName = String(args.excelName ?? "表格数据.xlsx").replace(/\$\{randomString\}/g, Math.random().toString(36).slice(2, 8)); + excelFile = path.resolve(path.isAbsolute(fileName) ? fileName : path.join(folder, fileName)); + await fs.mkdir(path.dirname(excelFile), { recursive: true }); + await workbook.xlsx.writeFile(excelFile); + ctx.log("已保存 Excel：" + excelFile, "info", node.id); + } + + return excelFile ? { rows, excelFile } : rows; }); \ No newline at end of file diff --git a/src/main/capabilities/vision-helpers.test.ts b/src/main/capabilities/vision-helpers.test.ts new file mode 100644 index 0000000..a31c149 --- /dev/null +++ b/src/main/capabilities/vision-helpers.test.ts @@ -0,0 +1,81 @@ +import { describe, expect, it } from "vitest"; +import { PNG } from "pngjs"; +import { clickPoint, clampRegion, cropPng, parseRegion } from "./vision"; + +function makeImage(width = 100, height = 80): PNG { + const png = new PNG({ width, height }); + for (let i = 0; i < png.data.length; i += 4) { + png.data[i] = 200; + png.data[i + 1] = 100; + png.data[i + 2] = 50; + png.data[i + 3] = 255; + } + return png; +} + +describe("parseRegion", () => { + it("accepts object x/y/width/height", () => { + const image = makeImage(); + expect(parseRegion('{"x":10,"y":20,"width":30,"height":40}', image)).toEqual({ x: 10, y: 20, width: 30, height: 40 }); + }); + + it("accepts array [x,y,width,height]", () => { + const image = makeImage(); + expect(parseRegion("[1,2,3,4]", image)).toEqual({ x: 1, y: 2, width: 3, height: 4 }); + }); + + it("accepts left/top aliases and corner points", () => { + const image = makeImage(); + expect(parseRegion('{"left":5,"top":6,"width":7,"height":8}', image)).toEqual({ x: 5, y: 6, width: 7, height: 8 }); + expect(parseRegion('{"x":5,"y":6,"x2":15,"y2":16}', image)).toEqual({ x: 5, y: 6, width: 10, height: 10 }); + }); + + it("rejects malformed JSON and unknown shapes", () => { + const image = makeImage(); + expect(() => parseRegion("not-json", image)).toThrow(/无法解析/); + expect(() => parseRegion('{"foo":1}', image)).toThrow(/不符合要求/); + }); +}); + +describe("clampRegion", () => { + it("clamps region into image bounds", () => { + const image = makeImage(100, 80); + expect(clampRegion({ x: -5, y: -5, width: 200, height: 200 }, image)).toEqual({ x: 0, y: 0, width: 100, height: 80 }); + expect(clampRegion({ x: 90, y: 70, width: 50, height: 50 }, image)).toEqual({ x: 90, y: 70, width: 10, height: 10 }); + }); +}); + +describe("cropPng", () => { + it("crops sub-image with expected size and pixel origin", () => { + const image = makeImage(); + const cropped = cropPng(image, { x: 10, y: 20, width: 30, height: 40 }); + expect(cropped.width).toBe(30); + expect(cropped.height).toBe(40); + const index = (0 * 30 + 0) * 4; + expect([cropped.data[index], cropped.data[index + 1], cropped.data[index + 2]]).toEqual([200, 100, 50]); + }); +}); + +describe("clickPoint", () => { + const region = { x: 100, y: 200, width: 60, height: 40, score: 1 }; + + it("defaults to center", () => { + expect(clickPoint(region, "")).toEqual({ x: 130, y: 220 }); + expect(clickPoint(region, "center")).toEqual({ x: 130, y: 220 }); + }); + + it("supports nine-grid anchors", () => { + expect(clickPoint(region, "leftTop")).toEqual({ x: 100, y: 200 }); + expect(clickPoint(region, "rightBottom")).toEqual({ x: 160, y: 240 }); + expect(clickPoint(region, "top")).toEqual({ x: 130, y: 200 }); + expect(clickPoint(region, "left")).toEqual({ x: 100, y: 220 }); + }); + + it("supports custom offset JSON", () => { + expect(clickPoint(region, '{"x":5,"y":6}')).toEqual({ x: 105, y: 206 }); + }); + + it("falls back to center for unknown values", () => { + expect(clickPoint(region, "bogus")).toEqual({ x: 130, y: 220 }); + }); +}); diff --git a/src/main/capabilities/vision.ts b/src/main/capabilities/vision.ts index 68b5df5..c9eaa80 100644 --- a/src/main/capabilities/vision.ts +++ b/src/main/capabilities/vision.ts @@ -23,7 +23,7 @@ async function capturePrimaryScreenPng(): Promise { } /** 解析区域 JSON：对象 {x,y,width,height}（兼容 left/top、x1y1x2y2 角点）或数组 [x,y,width,height]，并夹取到图片边界内 */ -function parseRegion(raw: string, image: PNG): { x: number; y: number; width: number; height: number } { +export function parseRegion(raw: string, image: PNG): { x: number; y: number; width: number; height: number } { let parsed: unknown; try { parsed = JSON.parse(raw); @@ -61,7 +61,7 @@ function parseRegion(raw: string, image: PNG): { x: number; y: number; width: nu }); } -function clampRegion(region: { x: number; y: number; width: number; height: number }, image: PNG): { x: number; y: number; width: number; height: number } { +export function clampRegion(region: { x: number; y: number; width: number; height: number }, image: PNG): { x: number; y: number; width: number; height: number } { const x = Math.max(0, Math.min(Math.round(region.x), image.width - 1)); const y = Math.max(0, Math.min(Math.round(region.y), image.height - 1)); const width = Math.max(1, Math.min(Math.round(region.width), image.width - x)); @@ -69,13 +69,13 @@ function clampRegion(region: { x: number; y: number; width: number; height: numb return { x, y, width, height }; } -function cropPng(source: PNG, region: { x: number; y: number; width: number; height: number }): PNG { +export function cropPng(source: PNG, region: { x: number; y: number; width: number; height: number }): PNG { const cropped = new PNG({ width: region.width, height: region.height }); PNG.bitblt(source, cropped, region.x, region.y, region.width, region.height, 0, 0); return cropped; } -register("Image_Screenshot", async (_ctx, node, args) => { +register("Image_Screenshot", async (ctx, node, args) => { const folder = text(args, ["outImagePath", "folder"]); const name = text(args, ["outImageName", "name"], `screenshot_${Date.now()}.png`); const imageArg = text(args, ["image_path"]); @@ -95,9 +95,9 @@ register("Image_Screenshot", async (_ctx, node, args) => { image = PNG.sync.read(await fs.readFile(sourceFile)); output = folder ? path.resolve(path.join(folder, name)) : sourceFile; } else { - // 当前屏幕来源：image_path 即保存路径，缺省用 输出目录+文件名 + // 当前屏幕来源：image_path 即保存路径，缺省用 输出目录+文件名；再缺省落流程证据目录（路径可预期） image = await capturePrimaryScreenPng(); - output = path.resolve(imageArg || (folder ? path.join(folder, name) : name)); + output = path.resolve(imageArg || (folder ? path.join(folder, name) : path.join(ctx.evidenceDir, name))); } if (range === "2") { @@ -119,10 +119,46 @@ interface MatchResult { x: number; y: number; width: number; height: number; sco async function captureScreen(target: string): Promise { await fs.mkdir(path.dirname(target), { recursive: true }); - await powershell(`Add-Type -AssemblyName System.Drawing; Add-Type -AssemblyName System.Windows.Forms; $bounds = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds; $bitmap = New-Object System.Drawing.Bitmap $bounds.Width,$bounds.Height; $graphics = [System.Drawing.Graphics]::FromImage($bitmap); $graphics.CopyFromScreen($bounds.Location,[System.Drawing.Point]::Empty,$bounds.Size); $bitmap.Save(${psString(target)},[System.Drawing.Imaging.ImageFormat]::Png); $graphics.Dispose(); $bitmap.Dispose()`); + await fs.writeFile(target, PNG.sync.write(await capturePrimaryScreenPng())); return target; } +interface WindowRect { x: number; y: number; width: number; height: number } + +/** 按标题（精确→包含）或进程名查找窗口，前置并返回窗口矩形（屏幕坐标） */ +async function focusWindowRect(windowObj: string): Promise { + const quoted = psString(windowObj); + const stdout = await powershell(`Add-Type @' +using System; using System.Runtime.InteropServices; +public static class WinRect { + [DllImport("user32.dll")] public static extern IntPtr FindWindow(string cls, string title); + [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr h); + [DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr h, int cmd); + [DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr h, out RECT r); + public struct RECT { public int Left; public int Top; public int Right; public int Bottom; } +} +'@; +$h = [WinRect]::FindWindow($null, ${quoted}); +if ($h -eq [IntPtr]::Zero) { $p = Get-Process | Where-Object { $_.MainWindowTitle -and ($_.MainWindowTitle -eq ${quoted} -or $_.MainWindowTitle.Contains(${quoted})) } | Select-Object -First 1; if ($p) { $h = $p.MainWindowHandle } } +if ($h -eq [IntPtr]::Zero) { Write-Output 'NOT_FOUND'; exit } +[WinRect]::ShowWindow($h, 9) | Out-Null; +[WinRect]::SetForegroundWindow($h) | Out-Null; +Start-Sleep -Milliseconds 300; +$r = New-Object WinRect+RECT; +[WinRect]::GetWindowRect($h, [ref]$r) | Out-Null; +Write-Output ("{0},{1},{2},{3}" -f $r.Left, $r.Top, $r.Right, $r.Bottom)`); + const line = stdout.trim().split(/\r?\n/).pop() ?? ""; + if (line === "NOT_FOUND") { + throw new RuntimeError("not_found", `未找到窗口「${windowObj}」`, { recommendation: "确认窗口已打开，窗口对象可填完整标题、标题包含关键字或进程名" }); + } + const parts = line.split(",").map(Number); + if (parts.length !== 4 || parts.some((v) => !Number.isFinite(v))) { + throw new RuntimeError("desktop_error", "窗口矩形获取失败", { detail: line.slice(0, 120) }); + } + const [left, top, right, bottom] = parts; + return { x: left, y: top, width: Math.max(1, right - left), height: Math.max(1, bottom - top) }; +} + function pixelDifference(screen: PNG, template: PNG, offsetX: number, offsetY: number, sampleStep: number, cutoff: number): number { let difference = 0; let samples = 0; @@ -176,18 +212,104 @@ async function locate(args: Record, evidenceDir: string, nodeId const rawTemplate = text(args, ["image", "image_path", "template"]); if (!rawTemplate.trim()) throw new RuntimeError("validation_error", "模板图片路径为空", { nodeId }); const template = path.resolve(rawTemplate); + + // 查找范围：1 整个屏幕（默认）/ 2 指定窗口 / 3 指定区域；窗口/区域模式下结果坐标会平移回屏幕坐标系 + const rangeMode = text(args, ["find_range"], "1"); const screenshot = path.join(evidenceDir, `${nodeId}_screen.png`); - await captureScreen(screenshot); + let offsetX = 0; + let offsetY = 0; + if (rangeMode === "2") { + const windowObj = text(args, ["window_obj"]); + if (!windowObj.trim()) throw new RuntimeError("validation_error", "查找范围为「指定窗口」时，窗口对象不能为空", { nodeId, recommendation: "填写窗口标题（或标题关键字、进程名）" }); + const rect = await focusWindowRect(windowObj); + const screen = await capturePrimaryScreenPng(); + await fs.writeFile(screenshot, PNG.sync.write(cropPng(screen, clampRegion(rect, screen)))); + offsetX = rect.x; + offsetY = rect.y; + } else if (rangeMode === "3") { + const regionRaw = text(args, ["shot_position", "action", "position"]); + if (!regionRaw.trim()) throw new RuntimeError("validation_error", "查找范围为「指定区域」时，需要填写区域 JSON", { nodeId, recommendation: '在 shot_position 填写 {"x":0,"y":0,"width":100,"height":80} 或 [x, y, width, height]' }); + const screen = await capturePrimaryScreenPng(); + const region = clampRegion(parseRegion(regionRaw, screen), screen); + await fs.writeFile(screenshot, PNG.sync.write(cropPng(screen, region))); + offsetX = region.x; + offsetY = region.y; + } else { + await captureScreen(screenshot); + } + const result = await findImage(template, screenshot, Number(text(args, ["accuracy"], "0.2"))); if (!result) throw new RuntimeError("not_found", "屏幕中未找到目标图像", { nodeId, recommendation: "重新截取模板，或调低匹配精度" }); - return result; + return { ...result, x: result.x + offsetX, y: result.y + offsetY }; } register(["Image_Find", "Image_Find_V1"], async (ctx, node, args) => locate(args, ctx.evidenceDir, node.id)); +/** 点击位置锚点（相对匹配区域的比例坐标），支持九宫格取值 */ +const POSITION_ANCHORS: Record = { + center: [0.5, 0.5], + leftTop: [0, 0], top: [0.5, 0], centerTop: [0.5, 0], rightTop: [1, 0], + left: [0, 0.5], centerLeft: [0, 0.5], right: [1, 0.5], centerRight: [1, 0.5], + leftBottom: [0, 1], bottom: [0.5, 1], centerBottom: [0.5, 1], rightBottom: [1, 1] +}; + +export function clickPoint(result: MatchResult, position: string): { x: number; y: number } { + const raw = position.trim(); + if (raw && raw.toLowerCase() !== "center") { + if (raw.startsWith("{")) { + try { + const offset = JSON.parse(raw) as { x?: unknown; y?: unknown }; + const dx = Number(offset.x); + const dy = Number(offset.y); + if (Number.isFinite(dx) && Number.isFinite(dy)) { + return { x: Math.round(result.x + dx), y: Math.round(result.y + dy) }; + } + } catch { + // 非法 JSON 落回中心 + } + } + const anchor = POSITION_ANCHORS[raw] ?? POSITION_ANCHORS[raw.toLowerCase()]; + if (anchor) { + return { x: Math.round(result.x + result.width * anchor[0]), y: Math.round(result.y + result.height * anchor[1]) }; + } + } + return { x: Math.round(result.x + result.width / 2), y: Math.round(result.y + result.height / 2) }; +} + +const CLICK_ACTIONS: Record = { + left: "[Mouse]::mouse_event(2,0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 30; [Mouse]::mouse_event(4,0,0,0,[UIntPtr]::Zero)", + right: "[Mouse]::mouse_event(8,0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 30; [Mouse]::mouse_event(16,0,0,0,[UIntPtr]::Zero)", + middle: "[Mouse]::mouse_event(32,0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 30; [Mouse]::mouse_event(64,0,0,0,[UIntPtr]::Zero)", + double: "[Mouse]::mouse_event(2,0,0,0,[UIntPtr]::Zero); [Mouse]::mouse_event(4,0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 60; [Mouse]::mouse_event(2,0,0,0,[UIntPtr]::Zero); [Mouse]::mouse_event(4,0,0,0,[UIntPtr]::Zero)" +}; + +async function injectClick(x: number, y: number, clickKey: string, showTrack: boolean): Promise { + const action = CLICK_ACTIONS[clickKey] ?? CLICK_ACTIONS.left; + // show_track=1 时从当前光标位置平滑移动过去（20 步插值），否则直接跳转 + const move = showTrack + ? `for ($i = 1; $i -le 20; $i++) { [Mouse]::SetCursorPos([int]($p.X + (${x} - $p.X) * $i / 20), [int]($p.Y + (${y} - $p.Y) * $i / 20)); Start-Sleep -Milliseconds 12 }` + : `[Mouse]::SetCursorPos(${x}, ${y})`; + await powershell(`Add-Type @' +using System; using System.Runtime.InteropServices; +public static class Mouse { + [DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y); + [DllImport("user32.dll")] public static extern bool GetCursorPos(out POINT p); + [DllImport("user32.dll")] public static extern void mouse_event(uint f, uint x, uint y, uint d, UIntPtr e); + public struct POINT { public int X; public int Y; } +} +'@; +$p = New-Object Mouse+POINT; +[Mouse]::GetCursorPos([ref]$p) | Out-Null; +${move}; +Start-Sleep -Milliseconds 60; +${action}`); +} + register(["Image_Click", "Image_Click_V1", "Image_Click_V2", "Image_Click_V3"], async (ctx, node, args) => { const result = await locate(args, ctx.evidenceDir, node.id); - const x = Math.round(result.x + result.width / 2); - const y = Math.round(result.y + result.height / 2); - await powershell(`Add-Type @'\nusing System; using System.Runtime.InteropServices; public static class Clicker { [DllImport("user32.dll")] public static extern bool SetCursorPos(int X,int Y); [DllImport("user32.dll")] public static extern void mouse_event(uint f,uint x,uint y,uint d,UIntPtr e); }\n'@; [Clicker]::SetCursorPos(${x},${y}); [Clicker]::mouse_event(2,0,0,0,[UIntPtr]::Zero); [Clicker]::mouse_event(4,0,0,0,[UIntPtr]::Zero)`); - return { ...result, click: { x, y } }; + const point = clickPoint(result, text(args, ["position"], "center")); + const clickKey = text(args, ["click_key"], "left"); + const showTrack = String(args.show_track ?? "0") === "1" || args.show_track === true; + await injectClick(point.x, point.y, clickKey, showTrack); + ctx.log(`已${clickKey === "right" ? "右键" : clickKey === "double" ? "双击" : "点击"}图像 (${point.x}, ${point.y})，匹配得分 ${result.score.toFixed(3)}`, "info", node.id); + return { ...result, click: point }; }); From 88d5e61c5b902a2d617b937c7d722eb1883c747e Mon Sep 17 00:00:00 2001 From: yu <191972707@qq.com> Date: Mon, 2 Nov 2026 14:53:58 +0800 Subject: [PATCH 4/8] =?UTF-8?q?docs:=20=E6=8A=A5=E5=91=8A=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=E7=8A=B6=E6=80=81=E8=A1=A5=E6=8F=90=E4=BA=A4=E5=8F=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/图像与OCR问题检测报告.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/图像与OCR问题检测报告.md b/docs/图像与OCR问题检测报告.md index 114c12f..a42d352 100644 --- a/docs/图像与OCR问题检测报告.md +++ b/docs/图像与OCR问题检测报告.md @@ -9,7 +9,7 @@ - **说明**：本次仅检测未修改任何代码，问题留给同事修复。 - **修复状态（2026-11-02 更新）**：报告中的全部问题已由刘爱家修复完成—— - 提交 `342145c`：P2-2（屏幕截图图像来源/截图范围，含区域裁剪与下拉选项） - - 提交 `<本次>`：P0-1（GetOcrTextNew 传图键）、P1-1（点击位置九宫格/偏移）、P1-2（鼠标按键左/右/中/双击）、P1-3（GetOcrTable 行列聚类 + 写数据表 + 存 Excel）、P2-1（查找范围指定窗口/指定区域）、P3-2（鼠标移动轨迹动画）、P3-3（截图空路径落证据目录；表格行内按 x 排序）；P3-1（accuracy2）无特征匹配层实现，已在目录中隐藏并注明原因 + - 提交 `ba17d6d`：P0-1（GetOcrTextNew 传图键）、P1-1（点击位置九宫格/偏移）、P1-2（鼠标按键左/右/中/双击）、P1-3（GetOcrTable 行列聚类 + 写数据表 + 存 Excel）、P2-1（查找范围指定窗口/指定区域）、P3-2（鼠标移动轨迹动画）、P3-3（截图空路径落证据目录；表格行内按 x 排序）；P3-1（accuracy2）无特征匹配层实现，已在目录中隐藏并注明原因 - 回归：typecheck 通过；vitest 256 过（4 失败为与本次无关的桌面/PDF 环境遗留项）；新增 vision-helpers 单测 12 项覆盖区域解析/裁剪/九宫格 --- From 29f34431108070e720a62a96403585211cfc2e0f Mon Sep 17 00:00:00 2001 From: yu <191972707@qq.com> Date: Mon, 2 Nov 2026 15:01:44 +0800 Subject: [PATCH 5/8] =?UTF-8?q?fix:=20=E4=BF=9D=E5=AD=98=E5=AF=B9=E8=AF=9D?= =?UTF-8?q?=E6=A1=86=E9=A2=84=E5=A1=AB=E6=96=87=E4=BB=B6=E5=90=8D=E5=B9=B6?= =?UTF-8?q?=E8=87=AA=E5=8A=A8=E8=A1=A5=E6=89=A9=E5=B1=95=E5=90=8D;=20?= =?UTF-8?q?=E6=88=AA=E5=9B=BE=E7=BB=84=E4=BB=B6=E9=9A=90=E8=97=8F=E5=9B=BE?= =?UTF-8?q?=E5=83=8F=E6=9D=A5=E6=BA=90=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Properties.tsx: 输出类字段保存框按过滤器预填默认文件名(未命名.png 等), 不再是空文件名没法直接点保存; 保存结果无扩展名时自动补所选类型扩展名(Windows 保存框不会自动补) - catalog: Image_Screenshot 的 pictureSource(图像来源) 界面隐藏, 实现默认截当前屏幕; 历史流程存有 2(指定图片文件) 仍按原逻辑生效 --- resources/catalog.json | 4 ++-- src/renderer/src/panels/Properties.tsx | 25 ++++++++++++++++++++++--- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/resources/catalog.json b/resources/catalog.json index af5e8ab..8cae947 100644 --- a/resources/catalog.json +++ b/resources/catalog.json @@ -61895,13 +61895,13 @@ { "key": "pictureSource", "label": "图像来源", - "widget": "radio", + "widget": "hidden", "type": "string", "displayType": "buttonGroup", "required": false, "defaultValue": "1", "placeholder": "", - "description": "选择要截图的图像来源，可以是当前屏幕，也可以是指定的图片文件。", + "description": "【界面已隐藏】实现默认截取当前屏幕；历史流程如存有 2(指定图片文件) 仍按原逻辑生效", "advanced": false, "valueTypes": [], "options": [ diff --git a/src/renderer/src/panels/Properties.tsx b/src/renderer/src/panels/Properties.tsx index cf8deb5..1bb0b65 100644 --- a/src/renderer/src/panels/Properties.tsx +++ b/src/renderer/src/panels/Properties.tsx @@ -481,6 +481,21 @@ function fileFiltersFor(componentId: string, field: CatalogField): FileFilterOpt return filters; } +/** 输出字段的保存框默认文件名：取第一个非「全部文件」过滤器的首个扩展名 */ +function defaultSaveFilename(filters: FileFilterOption[]): string { + const ext = filters.find((f) => !f.extensions.includes("*"))?.extensions[0]; + return ext ? `未命名.${ext}` : ""; +} + +/** 保存结果没有扩展名时补上所选类型的扩展名（Windows 保存框不会自动补） */ +function ensureSaveExtension(filePath: string, filters: FileFilterOption[]): string { + const exts = filters.find((f) => !f.extensions.includes("*"))?.extensions ?? []; + if (!exts.length) return filePath; + const clean = filePath.trim(); + if (/\.[A-Za-z0-9]+$/.test(clean)) return clean; + return `${clean}.${exts[0]}`; +} + function FieldControl({ field, node, @@ -533,12 +548,16 @@ function FieldControl({ `${field.key} ${field.label ?? ""}` ); if (isOutputField) { + const filters = fileFiltersFor(node.componentId, field); + // 预填默认文件名（按过滤器推断扩展名），避免弹出的保存框文件名为空没法直接点保存 + const defaultPath = textOf(raw) || defaultSaveFilename(filters); const selected = await window.rpa.dialog.save({ title: field.label, - defaultPath: textOf(raw) || undefined, - filters: fileFiltersFor(node.componentId, field) + defaultPath: defaultPath || undefined, + filters }); - if (selected) onChange(field.key, selected); + // Windows 保存框不会按所选类型自动补扩展名，这里补上 + if (selected) onChange(field.key, ensureSaveExtension(selected, filters)); return; } } From 1bb28e3368fbf9ee7e9eafb551bf9060c48d855b Mon Sep 17 00:00:00 2001 From: yu <191972707@qq.com> Date: Mon, 2 Nov 2026 15:09:23 +0800 Subject: [PATCH 6/8] =?UTF-8?q?fix:=20=E6=88=AA=E5=9B=BE=E7=9B=AE=E6=A0=87?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E8=B7=AF=E5=BE=84=E5=85=BC=E5=AE=B9=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E5=A4=B9=E5=86=99=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 目标文件路径语义为完整文件路径(含文件名), 提示文案不变 - 容错: 填的是已存在文件夹或以分隔符结尾时, 自动往里存默认文件名(screenshot_时间戳.png 或配置的文件名称), 不再报 EISDIR --- src/main/capabilities/vision.ts | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/main/capabilities/vision.ts b/src/main/capabilities/vision.ts index c9eaa80..6193755 100644 --- a/src/main/capabilities/vision.ts +++ b/src/main/capabilities/vision.ts @@ -75,6 +75,18 @@ export function cropPng(source: PNG, region: { x: number; y: number; width: numb return cropped; } +/** 目标路径容错：填的是文件夹（已存在或以分隔符结尾）时，自动往里存默认文件名 */ +async function resolveOutputPath(outputPath: string, defaultName: string): Promise { + if (/[\\/]$/.test(outputPath)) return path.join(outputPath, defaultName); + try { + const stat = await fs.stat(outputPath); + if (stat.isDirectory()) return path.join(outputPath, defaultName); + } catch { + // 路径不存在说明是要新建的文件，原样返回 + } + return outputPath; +} + register("Image_Screenshot", async (ctx, node, args) => { const folder = text(args, ["outImagePath", "folder"]); const name = text(args, ["outImageName", "name"], `screenshot_${Date.now()}.png`); @@ -95,9 +107,12 @@ register("Image_Screenshot", async (ctx, node, args) => { image = PNG.sync.read(await fs.readFile(sourceFile)); output = folder ? path.resolve(path.join(folder, name)) : sourceFile; } else { - // 当前屏幕来源：image_path 即保存路径，缺省用 输出目录+文件名；再缺省落流程证据目录（路径可预期） + // 当前屏幕来源：image_path 即保存路径（文件夹则自动追加默认文件名），缺省用 输出目录+文件名；再缺省落流程证据目录（路径可预期） image = await capturePrimaryScreenPng(); - output = path.resolve(imageArg || (folder ? path.join(folder, name) : path.join(ctx.evidenceDir, name))); + output = await resolveOutputPath( + path.resolve(imageArg || (folder ? path.join(folder, name) : path.join(ctx.evidenceDir, name))), + name + ); } if (range === "2") { From 5795c39a6fbd6252d5af3f01665dce0c762d5f51 Mon Sep 17 00:00:00 2001 From: yu <191972707@qq.com> Date: Mon, 2 Nov 2026 15:18:50 +0800 Subject: [PATCH 7/8] =?UTF-8?q?feat:=20=E4=BB=BB=E5=8A=A1=E5=88=97?= =?UTF-8?q?=E8=A1=A8=E6=94=AF=E6=8C=81=E5=8B=BE=E9=80=89=E6=89=B9=E9=87=8F?= =?UTF-8?q?=E5=88=A0=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 每行任务左侧新增复选框; 表头「任务」列新增全选(仅作用于当前筛选结果) - 勾选后列表头部出现「删除所选(N)」按钮, 二次确认后批量删除 - 批量删除保留「至少保留一个任务」保护; 勾选集合随列表变化自动裁剪已删除 id - 单行删除行为不变 --- src/renderer/src/App.tsx | 21 +++++++++++++ src/renderer/src/pages/TasksPage.tsx | 46 ++++++++++++++++++++++++++-- src/renderer/src/styles/app.css | 22 +++++++++++++ 3 files changed, 87 insertions(+), 2 deletions(-) diff --git a/src/renderer/src/App.tsx b/src/renderer/src/App.tsx index 0253066..a97c410 100644 --- a/src/renderer/src/App.tsx +++ b/src/renderer/src/App.tsx @@ -175,6 +175,26 @@ export function App() { } }; + const deleteTasks = async (ids: string[]) => { + if (!ids.length) return; + if (ids.length >= flows.length) { + showToast("至少保留一个任务", "error"); + return; + } + if (!window.confirm(`确定删除选中的 ${ids.length} 个任务吗？删除后不可恢复。`)) return; + try { + for (const id of ids) await window.rpa.flows.delete(id); + await refreshFlows(); + if (activeFlowId && ids.includes(activeFlowId)) { + setActiveFlowId(null); + setPage("tasks"); + } + showToast(`已删除 ${ids.length} 个任务`); + } catch (deleteError) { + showToast(deleteError instanceof Error ? deleteError.message : String(deleteError), "error"); + } + }; + const runTask = async (id: string) => { try { const flow = await window.rpa.flows.get(id); @@ -260,6 +280,7 @@ export function App() { onOpenEditor={(id) => void openEditor(id)} onRun={(id) => void runTask(id)} onDelete={(id) => void deleteTask(id)} + onDeleteMany={(ids) => void deleteTasks(ids)} onRefresh={() => refreshFlows()} onToast={showToast} /> diff --git a/src/renderer/src/pages/TasksPage.tsx b/src/renderer/src/pages/TasksPage.tsx index 152c748..083f140 100644 --- a/src/renderer/src/pages/TasksPage.tsx +++ b/src/renderer/src/pages/TasksPage.tsx @@ -23,6 +23,7 @@ interface TasksPageProps { onOpenEditor: (id: string) => void; onRun: (id: string) => void; onDelete: (id: string) => void; + onDeleteMany: (ids: string[]) => void; onRefresh: () => void | Promise; onToast: (message: string, kind?: "ok" | "error") => void; } @@ -46,9 +47,10 @@ function formatDuration(value?: number): string { const RUN_LABEL = { succeeded: "成功", failed: "失败", cancelled: "已取消", never: "未运行" } as const; -export function TasksPage({ flows, onCreate, onCreateExamples, onOpenEditor, onRun, onDelete, onRefresh, onToast }: TasksPageProps) { +export function TasksPage({ flows, onCreate, onCreateExamples, onOpenEditor, onRun, onDelete, onDeleteMany, onRefresh, onToast }: TasksPageProps) { const [query, setQuery] = useState(""); const [selectedId, setSelectedId] = useState(flows[0]?.id || null); + const [checkedIds, setCheckedIds] = useState([]); const [refreshing, setRefreshing] = useState(false); const handleRefresh = async () => { @@ -70,12 +72,30 @@ export function TasksPage({ flows, onCreate, onCreateExamples, onOpenEditor, onR if (!selectedId || !flows.some((flow) => flow.id === selectedId)) setSelectedId(flows[0].id); }, [flows, selectedId]); + // 勾选集合随任务列表变化裁剪，避免残留已删除任务的 id + useEffect(() => { + setCheckedIds((previous) => { + const alive = new Set(flows.map((flow) => flow.id)); + const next = previous.filter((id) => alive.has(id)); + return next.length === previous.length ? previous : next; + }); + }, [flows]); + const filtered = useMemo(() => { const keyword = query.trim().toLowerCase(); if (!keyword) return flows; return flows.filter((flow) => flow.name.toLowerCase().includes(keyword)); }, [flows, query]); + const toggleChecked = (id: string) => { + setCheckedIds((previous) => (previous.includes(id) ? previous.filter((item) => item !== id) : [...previous, id])); + }; + + const allFilteredChecked = filtered.length > 0 && filtered.every((flow) => checkedIds.includes(flow.id)); + const toggleAllFiltered = () => { + setCheckedIds(allFilteredChecked ? [] : filtered.map((flow) => flow.id)); + }; + const selected = flows.find((flow) => flow.id === selectedId) || filtered[0] || null; const enabledCount = flows.filter((flow) => flow.enabled !== false).length; const failedCount = flows.filter((flow) => flow.lastRunStatus === "failed").length; @@ -112,6 +132,11 @@ export function TasksPage({ flows, onCreate, onCreateExamples, onOpenEditor, onR 任务列表 按最近修改时间排序 + {checkedIds.length > 0 && ( + + )}