fix: 对比图像相对位置字段正名(被搜索图像/查找目标小图)+填反自动互换兜底, 补OCR与相对位置冒烟回归
This commit is contained in:
yu
parent
9ec301b081
commit
c973209418
@ -68766,7 +68766,7 @@
|
||||
"image"
|
||||
],
|
||||
"categorySignature": "image",
|
||||
"description": "获取截图在整体图片中的位置坐标。",
|
||||
"description": "在一张较大的图（如整屏/整窗截图）中查找指定小图，返回其相对位置坐标。",
|
||||
"messageTemplate": "获取图像 $1 中，$2 的相对位置坐标， 然后把结果保存到 $0",
|
||||
"baseType": "Image_RelativePosition",
|
||||
"functionName": "Image.RelativePosition",
|
||||
@ -68794,14 +68794,14 @@
|
||||
"fields": [
|
||||
{
|
||||
"key": "source",
|
||||
"label": "模板图像",
|
||||
"label": "被搜索图像",
|
||||
"widget": "text",
|
||||
"type": "string",
|
||||
"displayType": "imageSelector",
|
||||
"required": true,
|
||||
"defaultValue": "",
|
||||
"placeholder": "截取或选择图像",
|
||||
"description": "填写参数值，支持 ${变量} 引用上游输出",
|
||||
"description": "被搜索的较大图片路径（如整屏截图），支持 ${变量} 引用上游输出",
|
||||
"advanced": false,
|
||||
"valueTypes": [
|
||||
"rpa_str"
|
||||
@ -68809,14 +68809,14 @@
|
||||
},
|
||||
{
|
||||
"key": "template",
|
||||
"label": "目标图像",
|
||||
"label": "查找目标（小图）",
|
||||
"widget": "text",
|
||||
"type": "string",
|
||||
"displayType": "imageSelector",
|
||||
"required": true,
|
||||
"defaultValue": "",
|
||||
"placeholder": "截取或选择图像",
|
||||
"description": "图片文件的完整路径，用于匹配或识别；点右侧按钮选择文件",
|
||||
"description": "要寻找的小图案完整路径（按钮/图标/文字块），可点右侧按钮选择文件",
|
||||
"advanced": false,
|
||||
"valueTypes": [
|
||||
"rpa_str"
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { getHandler } from "../engine/registry";
|
||||
import type { CatalogComponent, RunContext } from "../engine/types";
|
||||
import type { CatalogComponent, FlowNode } from "../../shared/types";
|
||||
import type { RunContext } from "../engine/types";
|
||||
import "./ocr-caps";
|
||||
import "./remaining-core";
|
||||
|
||||
@ -11,7 +12,7 @@ const ctx = {
|
||||
evidenceDir: "D:/office-test2/ocr-test"
|
||||
} as unknown as RunContext;
|
||||
|
||||
const node = { id: "smoke", componentId: "smoke" };
|
||||
const node = { id: "smoke", componentId: "smoke", position: { x: 0, y: 0 }, values: {}, options: {} };
|
||||
|
||||
async function runOcr(id: string, args: Record<string, unknown>) {
|
||||
const handler = getHandler(id);
|
||||
|
||||
53
src/main/capabilities/relativeposition-smoke.test.ts
Normal file
53
src/main/capabilities/relativeposition-smoke.test.ts
Normal file
@ -0,0 +1,53 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { getHandler } from "../engine/registry";
|
||||
import type { CatalogComponent, FlowNode } from "../../shared/types";
|
||||
import type { RunContext } from "../engine/types";
|
||||
import "./vision";
|
||||
import "./remaining-core";
|
||||
|
||||
const ctx = {
|
||||
signal: new AbortController().signal,
|
||||
log: () => {},
|
||||
vars: {},
|
||||
evidenceDir: "D:/office-test2/ocr-test"
|
||||
} as unknown as RunContext;
|
||||
|
||||
const node = { id: "smoke-rp", componentId: "Image_RelativePosition", position: { x: 0, y: 0 }, values: {}, options: {} };
|
||||
|
||||
describe("对比图像相对位置冒烟", () => {
|
||||
it("小图在大图中命中并返回坐标", async () => {
|
||||
const handler = getHandler("Image_RelativePosition");
|
||||
expect(handler).toBeTruthy();
|
||||
const out = await handler!(
|
||||
ctx,
|
||||
node,
|
||||
{
|
||||
source: "D:/office-test2/ocr-test/general.png",
|
||||
template: "D:/office-test2/ocr-test/order-template.png"
|
||||
},
|
||||
{} as CatalogComponent
|
||||
);
|
||||
console.log("[RelativePosition 正向]", JSON.stringify(out));
|
||||
const hit = out as { x: number; y: number; width: number; height: number; score: number };
|
||||
expect(hit.x).toBeGreaterThanOrEqual(0);
|
||||
expect(hit.y).toBeGreaterThan(0);
|
||||
expect(hit.score).toBeGreaterThan(0.9);
|
||||
}, 120000);
|
||||
|
||||
it("填反（大图填进查找目标）自动互换后仍命中", async () => {
|
||||
const logs: string[] = [];
|
||||
const ctxWithLog = { ...ctx, log: (m: string) => logs.push(m) } as unknown as RunContext;
|
||||
const handler = getHandler("Image_RelativePosition")!;
|
||||
const out = await handler!(
|
||||
ctxWithLog,
|
||||
node,
|
||||
{
|
||||
source: "D:/office-test2/ocr-test/order-template.png",
|
||||
template: "D:/office-test2/ocr-test/general.png"
|
||||
},
|
||||
{} as CatalogComponent
|
||||
);
|
||||
console.log("[RelativePosition 互换]", JSON.stringify(out), "logs:", logs.join(" | "));
|
||||
expect(out).toBeTruthy();
|
||||
}, 120000);
|
||||
});
|
||||
@ -203,9 +203,33 @@ register("Image_CropImg", async (_ctx, node, args) => {
|
||||
await powershell(`Add-Type -AssemblyName System.Drawing;$image=[System.Drawing.Image]::FromFile(${psString(input)});$rect=[System.Drawing.Rectangle]::new(${x},${y},${width},${height});$bitmap=[System.Drawing.Bitmap]::new($rect.Width,$rect.Height);$graphics=[System.Drawing.Graphics]::FromImage($bitmap);$graphics.DrawImage($image,[System.Drawing.Rectangle]::new(0,0,$rect.Width,$rect.Height),$rect,[System.Drawing.GraphicsUnit]::Pixel);$bitmap.Save(${psString(output)});$graphics.Dispose();$bitmap.Dispose();$image.Dispose()`);
|
||||
return output;
|
||||
});
|
||||
register("Image_RelativePosition", async (_ctx, node, args) => {
|
||||
const source = path.resolve(text(args, ["source", "image"])); const template = path.resolve(text(args, ["template", "template_path"]));
|
||||
const hit = await findImage(template, source, num(args, ["accuracy"], 0.2)); if (!hit) throw new RuntimeError("not_found", "模板未出现在源图中"); return hit;
|
||||
register("Image_RelativePosition", async (ctx, node, args) => {
|
||||
const source = path.resolve(text(args, ["source", "image"]));
|
||||
const template = path.resolve(text(args, ["template", "template_path"]));
|
||||
if (!source.trim() || !template.trim()) throw new RuntimeError("validation_error", "被搜索图像与查找目标都需填写图片路径", { nodeId: node.id, componentId: node.componentId });
|
||||
if (!fs.existsSync(source)) throw new RuntimeError("validation_error", `找不到被搜索图像：${source}`, { nodeId: node.id, componentId: node.componentId });
|
||||
if (!fs.existsSync(template)) throw new RuntimeError("validation_error", `找不到查找目标图片：${template}`, { nodeId: node.id, componentId: node.componentId });
|
||||
// 兼容填反：模板比源图大时（首次查找直接抛"还大"错误）自动互换，
|
||||
// 大图当被搜索图、小图当查找目标，语义仍是「小图在大图中的位置」
|
||||
const accuracy = num(args, ["accuracy"], 0.2);
|
||||
let hit: Awaited<ReturnType<typeof findImage>> = null;
|
||||
let firstError: unknown = null;
|
||||
try {
|
||||
hit = await findImage(template, source, accuracy);
|
||||
} catch (error) {
|
||||
firstError = error;
|
||||
try {
|
||||
hit = await findImage(source, template, accuracy);
|
||||
if (hit) ctx.log("两张图疑似填反，已自动按「小图在大图中」查找并命中", "warn", node.id);
|
||||
} catch {
|
||||
// 互换也失败则回落到原始方向的错误信息
|
||||
}
|
||||
}
|
||||
if (!hit) {
|
||||
if (firstError) throw firstError;
|
||||
throw new RuntimeError("not_found", "查找目标未出现在被搜索图像中", { nodeId: node.id, componentId: node.componentId, recommendation: "「查找目标」应是被搜索图里出现的小图案（按钮/图标/文字块）；两张图互不包含时无法匹配" });
|
||||
}
|
||||
return hit;
|
||||
});
|
||||
|
||||
register("logic_control_if_ai_vision", async (ctx, node, args) => {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user