diff --git a/resources/catalog.json b/resources/catalog.json index 1c887cb..f330214 100644 --- a/resources/catalog.json +++ b/resources/catalog.json @@ -68766,7 +68766,7 @@ "image" ], "categorySignature": "image", - "description": "获取截图在整体图片中的位置坐标。", + "description": "在一张较大的图(如整屏/整窗截图)中查找指定小图,返回其相对位置坐标。", "messageTemplate": "获取图像 $1 中,$2 的相对位置坐标, 然后把结果保存到 $0", "baseType": "Image_RelativePosition", "functionName": "Image.RelativePosition", @@ -68794,14 +68794,14 @@ "fields": [ { "key": "source", - "label": "模板图像", + "label": "被搜索图像", "widget": "text", "type": "string", "displayType": "imageSelector", "required": true, "defaultValue": "", "placeholder": "截取或选择图像", - "description": "填写参数值,支持 ${变量} 引用上游输出", + "description": "被搜索的较大图片路径(如整屏截图),支持 ${变量} 引用上游输出", "advanced": false, "valueTypes": [ "rpa_str" @@ -68809,14 +68809,14 @@ }, { "key": "template", - "label": "目标图像", + "label": "查找目标(小图)", "widget": "text", "type": "string", "displayType": "imageSelector", "required": true, "defaultValue": "", "placeholder": "截取或选择图像", - "description": "图片文件的完整路径,用于匹配或识别;点右侧按钮选择文件", + "description": "要寻找的小图案完整路径(按钮/图标/文字块),可点右侧按钮选择文件", "advanced": false, "valueTypes": [ "rpa_str" diff --git a/src/main/capabilities/ocr-smoke.test.ts b/src/main/capabilities/ocr-smoke.test.ts index aead65d..45c584a 100644 --- a/src/main/capabilities/ocr-smoke.test.ts +++ b/src/main/capabilities/ocr-smoke.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { getHandler } from "../engine/registry"; -import type { CatalogComponent, RunContext } from "../engine/types"; +import type { CatalogComponent, FlowNode } from "../../shared/types"; +import type { RunContext } from "../engine/types"; import "./ocr-caps"; import "./remaining-core"; @@ -11,7 +12,7 @@ const ctx = { evidenceDir: "D:/office-test2/ocr-test" } as unknown as RunContext; -const node = { id: "smoke", componentId: "smoke" }; +const node = { id: "smoke", componentId: "smoke", position: { x: 0, y: 0 }, values: {}, options: {} }; async function runOcr(id: string, args: Record) { const handler = getHandler(id); diff --git a/src/main/capabilities/relativeposition-smoke.test.ts b/src/main/capabilities/relativeposition-smoke.test.ts new file mode 100644 index 0000000..edd37f4 --- /dev/null +++ b/src/main/capabilities/relativeposition-smoke.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from "vitest"; +import { getHandler } from "../engine/registry"; +import type { CatalogComponent, FlowNode } from "../../shared/types"; +import type { RunContext } from "../engine/types"; +import "./vision"; +import "./remaining-core"; + +const ctx = { + signal: new AbortController().signal, + log: () => {}, + vars: {}, + evidenceDir: "D:/office-test2/ocr-test" +} as unknown as RunContext; + +const node = { id: "smoke-rp", componentId: "Image_RelativePosition", position: { x: 0, y: 0 }, values: {}, options: {} }; + +describe("对比图像相对位置冒烟", () => { + it("小图在大图中命中并返回坐标", async () => { + const handler = getHandler("Image_RelativePosition"); + expect(handler).toBeTruthy(); + const out = await handler!( + ctx, + node, + { + source: "D:/office-test2/ocr-test/general.png", + template: "D:/office-test2/ocr-test/order-template.png" + }, + {} as CatalogComponent + ); + console.log("[RelativePosition 正向]", JSON.stringify(out)); + const hit = out as { x: number; y: number; width: number; height: number; score: number }; + expect(hit.x).toBeGreaterThanOrEqual(0); + expect(hit.y).toBeGreaterThan(0); + expect(hit.score).toBeGreaterThan(0.9); + }, 120000); + + it("填反(大图填进查找目标)自动互换后仍命中", async () => { + const logs: string[] = []; + const ctxWithLog = { ...ctx, log: (m: string) => logs.push(m) } as unknown as RunContext; + const handler = getHandler("Image_RelativePosition")!; + const out = await handler!( + ctxWithLog, + node, + { + source: "D:/office-test2/ocr-test/order-template.png", + template: "D:/office-test2/ocr-test/general.png" + }, + {} as CatalogComponent + ); + console.log("[RelativePosition 互换]", JSON.stringify(out), "logs:", logs.join(" | ")); + expect(out).toBeTruthy(); + }, 120000); +}); diff --git a/src/main/capabilities/remaining-core.ts b/src/main/capabilities/remaining-core.ts index f78f178..023cc45 100644 --- a/src/main/capabilities/remaining-core.ts +++ b/src/main/capabilities/remaining-core.ts @@ -203,9 +203,33 @@ register("Image_CropImg", async (_ctx, node, args) => { await powershell(`Add-Type -AssemblyName System.Drawing;$image=[System.Drawing.Image]::FromFile(${psString(input)});$rect=[System.Drawing.Rectangle]::new(${x},${y},${width},${height});$bitmap=[System.Drawing.Bitmap]::new($rect.Width,$rect.Height);$graphics=[System.Drawing.Graphics]::FromImage($bitmap);$graphics.DrawImage($image,[System.Drawing.Rectangle]::new(0,0,$rect.Width,$rect.Height),$rect,[System.Drawing.GraphicsUnit]::Pixel);$bitmap.Save(${psString(output)});$graphics.Dispose();$bitmap.Dispose();$image.Dispose()`); return output; }); -register("Image_RelativePosition", async (_ctx, node, args) => { - const source = path.resolve(text(args, ["source", "image"])); const template = path.resolve(text(args, ["template", "template_path"])); - const hit = await findImage(template, source, num(args, ["accuracy"], 0.2)); if (!hit) throw new RuntimeError("not_found", "模板未出现在源图中"); return hit; +register("Image_RelativePosition", async (ctx, node, args) => { + const source = path.resolve(text(args, ["source", "image"])); + const template = path.resolve(text(args, ["template", "template_path"])); + if (!source.trim() || !template.trim()) throw new RuntimeError("validation_error", "被搜索图像与查找目标都需填写图片路径", { nodeId: node.id, componentId: node.componentId }); + if (!fs.existsSync(source)) throw new RuntimeError("validation_error", `找不到被搜索图像:${source}`, { nodeId: node.id, componentId: node.componentId }); + if (!fs.existsSync(template)) throw new RuntimeError("validation_error", `找不到查找目标图片:${template}`, { nodeId: node.id, componentId: node.componentId }); + // 兼容填反:模板比源图大时(首次查找直接抛"还大"错误)自动互换, + // 大图当被搜索图、小图当查找目标,语义仍是「小图在大图中的位置」 + const accuracy = num(args, ["accuracy"], 0.2); + let hit: Awaited> = null; + let firstError: unknown = null; + try { + hit = await findImage(template, source, accuracy); + } catch (error) { + firstError = error; + try { + hit = await findImage(source, template, accuracy); + if (hit) ctx.log("两张图疑似填反,已自动按「小图在大图中」查找并命中", "warn", node.id); + } catch { + // 互换也失败则回落到原始方向的错误信息 + } + } + if (!hit) { + if (firstError) throw firstError; + throw new RuntimeError("not_found", "查找目标未出现在被搜索图像中", { nodeId: node.id, componentId: node.componentId, recommendation: "「查找目标」应是被搜索图里出现的小图案(按钮/图标/文字块);两张图互不包含时无法匹配" }); + } + return hit; }); register("logic_control_if_ai_vision", async (ctx, node, args) => {