feat: CV 组件搜索区域 UI 接入完成
- catalog 为 CV_MouseClick/GetText/SetText/MouseHover 新增可选「搜索区域」字段(region widget)
- 属性面板 region 字段渲染「屏幕框选」按钮, 框选后存 {x,y,width,height,imagePath}
- 运行时框选截图内 OCR 定位 + 偏移换算屏幕绝对坐标; 纯坐标范围则全屏截屏后过滤区域内命中
- 新增 2 个区域过滤测试用例(区域内命中/区域外拒绝)
This commit is contained in:
yu
parent
5aa113ea0e
commit
d98185d1cf
@ -10,7 +10,7 @@
|
||||
"vendor": 997,
|
||||
"native": 16,
|
||||
"triggers": 3,
|
||||
"fields": 4293
|
||||
"fields": 4297
|
||||
},
|
||||
"groups": [
|
||||
{
|
||||
@ -11482,6 +11482,19 @@
|
||||
"description": "屏幕上可见的目标文本，运行时截屏后通过 OCR 定位（填按钮或界面上的文字，如「登录」）",
|
||||
"advanced": false,
|
||||
"valueTypes": []
|
||||
},
|
||||
{
|
||||
"key": "searchRegion",
|
||||
"label": "搜索区域",
|
||||
"widget": "region",
|
||||
"type": "object",
|
||||
"displayType": "region",
|
||||
"required": false,
|
||||
"defaultValue": "",
|
||||
"placeholder": "",
|
||||
"description": "可选。点「屏幕框选」框住目标所在区域，运行时只在该区域内查找目标文本，屏幕上有相同文字时可避免点错；留空则全屏查找",
|
||||
"advanced": false,
|
||||
"valueTypes": []
|
||||
}
|
||||
],
|
||||
"output": {
|
||||
@ -11542,6 +11555,19 @@
|
||||
"advanced": false,
|
||||
"valueTypes": []
|
||||
},
|
||||
{
|
||||
"key": "searchRegion",
|
||||
"label": "搜索区域",
|
||||
"widget": "region",
|
||||
"type": "object",
|
||||
"displayType": "region",
|
||||
"required": false,
|
||||
"defaultValue": "",
|
||||
"placeholder": "",
|
||||
"description": "可选。点「屏幕框选」框住目标所在区域，运行时只在该区域内查找目标文本，屏幕上有相同文字时可避免点错；留空则全屏查找",
|
||||
"advanced": false,
|
||||
"valueTypes": []
|
||||
},
|
||||
{
|
||||
"key": "text",
|
||||
"label": "文本",
|
||||
@ -11608,6 +11634,19 @@
|
||||
"advanced": false,
|
||||
"valueTypes": []
|
||||
},
|
||||
{
|
||||
"key": "searchRegion",
|
||||
"label": "搜索区域",
|
||||
"widget": "region",
|
||||
"type": "object",
|
||||
"displayType": "region",
|
||||
"required": false,
|
||||
"defaultValue": "",
|
||||
"placeholder": "",
|
||||
"description": "可选。点「屏幕框选」框住目标所在区域，运行时只在该区域内查找目标文本，屏幕上有相同文字时可避免点错；留空则全屏查找",
|
||||
"advanced": false,
|
||||
"valueTypes": []
|
||||
},
|
||||
{
|
||||
"key": "offsetX",
|
||||
"label": "x轴偏移量",
|
||||
@ -11688,6 +11727,19 @@
|
||||
"advanced": false,
|
||||
"valueTypes": []
|
||||
},
|
||||
{
|
||||
"key": "searchRegion",
|
||||
"label": "搜索区域",
|
||||
"widget": "region",
|
||||
"type": "object",
|
||||
"displayType": "region",
|
||||
"required": false,
|
||||
"defaultValue": "",
|
||||
"placeholder": "",
|
||||
"description": "可选。点「屏幕框选」框住目标所在区域，运行时只在该区域内查找目标文本，屏幕上有相同文字时可避免点错；留空则全屏查找",
|
||||
"advanced": false,
|
||||
"valueTypes": []
|
||||
},
|
||||
{
|
||||
"key": "clickway",
|
||||
"label": "点击类型",
|
||||
|
||||
@ -2,7 +2,10 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const powershellMock = vi.hoisted(() => vi.fn(async () => '{"ok":true}'));
|
||||
const detectMock = vi.hoisted(() => vi.fn(async () => ({
|
||||
texts: [{ text: "确认保存", box: [[0, 0], [20, 0], [20, 10], [0, 10]] }]
|
||||
texts: [
|
||||
{ text: "确认保存", box: [[0, 0], [20, 0], [20, 10], [0, 10]] },
|
||||
{ text: "确认", box: [[100, 200], [130, 200], [130, 210], [100, 210]] }
|
||||
]
|
||||
})));
|
||||
|
||||
vi.mock("./desktop", () => ({ powershell: powershellMock }));
|
||||
@ -47,4 +50,21 @@ describe("CV vision capabilities", () => {
|
||||
}, {} as any)).resolves.toBe("确认保存");
|
||||
expect(powershellMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("restricts matching to the search region when provided", async () => {
|
||||
// 屏幕上有两处「确认」: 区域(0,0,50,50)内只应命中 (5,5) 那处, 而不是第一个全屏命中 (115,205)
|
||||
await expect(getHandler("CV_MouseClick")!({ log: vi.fn() } as any, { id: "n3", componentId: "CV_MouseClick" } as any, {
|
||||
cvSelector: "确认",
|
||||
searchRegion: JSON.stringify({ x: 0, y: 0, width: 50, height: 50 })
|
||||
}, {} as any)).resolves.toBe(true);
|
||||
const script = String((powershellMock.mock.calls as unknown as Array<[string]>)[1]?.[0] ?? "");
|
||||
expect(script).toContain("SetCursorPos(10, 5)");
|
||||
});
|
||||
|
||||
it("rejects when target is outside the search region", async () => {
|
||||
await expect(getHandler("CV_MouseClick")!({ log: vi.fn() } as any, { id: "n4", componentId: "CV_MouseClick" } as any, {
|
||||
cvSelector: "确认",
|
||||
searchRegion: JSON.stringify({ x: 500, y: 500, width: 50, height: 50 })
|
||||
}, {} as any)).rejects.toThrow("未找到目标文本");
|
||||
});
|
||||
});
|
||||
|
||||
@ -772,12 +772,25 @@ function FieldControl({
|
||||
placeholder={field.placeholder || ""}
|
||||
onChange={(event) => onChange(field.key, event.target.value)}
|
||||
/>
|
||||
) : field.widget === "region" ? (
|
||||
<div className="field-row">
|
||||
<input
|
||||
className="text-input"
|
||||
value={textOf(raw)}
|
||||
placeholder={field.placeholder || "点右侧「屏幕框选」在屏幕上框选生成"}
|
||||
onChange={(event) => onChange(field.key, event.target.value)}
|
||||
/>
|
||||
<VarMenu variables={variables} onPick={insertVariable} />
|
||||
<button type="button" className="mini-btn" disabled={picking} onClick={() => void pickRegionToField()} title="在屏幕上框选一块区域">
|
||||
{picking ? "框选中" : "屏幕框选"}
|
||||
</button>
|
||||
</div>
|
||||
) : (
|
||||
<div className="field-row">
|
||||
<input
|
||||
className="text-input"
|
||||
value={textOf(raw)}
|
||||
placeholder={field.placeholder || (field.widget === "region" ? "区域 / 坐标 JSON" : "")}
|
||||
placeholder={field.placeholder || ""}
|
||||
onChange={(event) => onChange(field.key, event.target.value)}
|
||||
/>
|
||||
<VarMenu variables={variables} onPick={insertVariable} />
|
||||
|
||||
Loading…
Reference in New Issue
Block a user