147 lines
6.8 KiB
TypeScript
147 lines
6.8 KiB
TypeScript
import { beforeEach, describe, expect, it, vi } from "vitest";
|
|
import fs from "node:fs";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
|
|
const powershellMock = vi.hoisted(() => vi.fn(async () => '{"ok":true}'));
|
|
const detectMock = vi.hoisted(() => vi.fn(async () => ({
|
|
texts: [
|
|
{ text: "确认保存", box: [[0, 0], [20, 0], [20, 10], [0, 10]] },
|
|
{ text: "确认", box: [[100, 200], [130, 200], [130, 210], [100, 210]] }
|
|
]
|
|
})));
|
|
// 屏幕服务模拟: 默认无画面(走 GDI 回退), 多屏用例按显示器 id/请求宽度提供对应画面
|
|
const electronState = vi.hoisted(() => ({ sources: [] as Array<{ display_id: string; thumbnail: any }> }));
|
|
|
|
vi.mock("../driver/windows/powershell", () => ({ powershell: powershellMock }));
|
|
vi.mock("@gutenye/ocr-node", () => ({
|
|
default: { create: vi.fn(async () => ({ detect: detectMock })) }
|
|
}));
|
|
|
|
import { getHandler } from "../../src/engine/registry";
|
|
import { setHostBridge, type HostDisplay } from "../../src/engine/host-bridge";
|
|
import "../../src/capabilities/cv-vision";
|
|
|
|
const displayOf = (id: string, x: number, width: number, height: number, scaleFactor: number): HostDisplay => ({
|
|
id,
|
|
bounds: { x, y: 0, width, height },
|
|
scaleFactor
|
|
});
|
|
const displays: HostDisplay[] = [
|
|
displayOf("1", 0, 1920, 1080, 1),
|
|
displayOf("2", 1920, 1280, 720, 2)
|
|
];
|
|
|
|
setHostBridge({
|
|
createFormWindow: () => { throw new Error("not used in cv-vision tests"); },
|
|
dipToScreenPoint: (point) => ({ x: point.x, y: point.y }),
|
|
screenToDipPoint: (point) => ({ x: point.x, y: point.y }),
|
|
loadAiConfig: () => ({ baseUrl: "", apiKey: "", model: "", visionModel: "" }),
|
|
screen: {
|
|
listDisplays: () => displays,
|
|
primaryDisplay: () => displays[0],
|
|
dipToScreenPoint: (point) => ({ x: point.x, y: point.y }),
|
|
displayMatching: () => displays[0],
|
|
captureDisplay: async (displayId: string, wantWidth: number, wantHeight: number) => {
|
|
// 模拟真实 captureDisplay 的对位规则，但不做 sources[0] 兜底：
|
|
// 保持"主屏无画面 → 跳过该屏"的旧多屏语义
|
|
const source = electronState.sources.find((item) => item.display_id === displayId && item.thumbnail.getSize().width === wantWidth)
|
|
|| electronState.sources.find((item) => item.thumbnail.getSize().width === wantWidth);
|
|
if (!source || source.thumbnail.isEmpty()) throw new Error(`无法获取显示器 ${displayId} 的屏幕截图`);
|
|
void wantHeight;
|
|
return source.thumbnail.toPNG();
|
|
}
|
|
}
|
|
});
|
|
|
|
describe("CV vision capabilities", () => {
|
|
beforeEach(() => {
|
|
powershellMock.mockClear();
|
|
detectMock.mockClear();
|
|
electronState.sources = [];
|
|
});
|
|
|
|
it.each(["CV_GetText", "CV_SetText", "CV_MouseHover", "CV_MouseClick"])("registers %s", (id) => {
|
|
expect(getHandler(id), id).toBeTruthy();
|
|
});
|
|
|
|
it("rejects an empty CV selector", async () => {
|
|
await expect(getHandler("CV_GetText")!({ log: vi.fn() } as any, { id: "n1", componentId: "CV_GetText" } as any, {}, {} as any))
|
|
.rejects.toThrow("CV 目标为空");
|
|
});
|
|
|
|
it("locates OCR text and clicks its center with offset", async () => {
|
|
const handler = getHandler("CV_MouseClick")!;
|
|
await expect(handler({ log: vi.fn() } as any, { id: "n1", componentId: "CV_MouseClick" } as any, {
|
|
cvSelector: "确认",
|
|
offsetX: 5,
|
|
offsetY: 2
|
|
}, {} as any)).resolves.toBe(true);
|
|
expect(detectMock).toHaveBeenCalledTimes(1);
|
|
const script = String((powershellMock.mock.calls as unknown as Array<[string]>)[1]?.[0] ?? "");
|
|
expect(script).toContain("SetCursorPos(15, 7)");
|
|
expect(script).toContain("mouse_event(0x0002");
|
|
});
|
|
|
|
it("returns matched OCR text", async () => {
|
|
await expect(getHandler("CV_GetText")!({ log: vi.fn() } as any, { id: "n2", componentId: "CV_GetText" } as any, {
|
|
cvSelector: "保存"
|
|
}, {} as any)).resolves.toBe("确认保存");
|
|
expect(powershellMock).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it("restricts matching to the search region when provided", async () => {
|
|
// 屏幕上有两处「确认」: 区域(0,0,50,50)内只应命中 (5,5) 那处, 而不是第一个全屏命中 (115,205)
|
|
await expect(getHandler("CV_MouseClick")!({ log: vi.fn() } as any, { id: "n3", componentId: "CV_MouseClick" } as any, {
|
|
cvSelector: "确认",
|
|
searchRegion: JSON.stringify({ x: 0, y: 0, width: 50, height: 50 })
|
|
}, {} as any)).resolves.toBe(true);
|
|
const script = String((powershellMock.mock.calls as unknown as Array<[string]>)[1]?.[0] ?? "");
|
|
expect(script).toContain("SetCursorPos(10, 5)");
|
|
});
|
|
|
|
it("rejects when target is outside the search region", async () => {
|
|
await expect(getHandler("CV_MouseClick")!({ log: vi.fn() } as any, { id: "n4", componentId: "CV_MouseClick" } as any, {
|
|
cvSelector: "确认",
|
|
searchRegion: JSON.stringify({ x: 500, y: 500, width: 50, height: 50 })
|
|
}, {} as any)).rejects.toThrow("未找到目标文本");
|
|
});
|
|
|
|
it("locates targets on a secondary display and maps DIP coordinates", async () => {
|
|
// 主屏无画面, 副屏(bounds 1920,0 1280x720 缩放2)物理中心(450,210) → DIP(2145,105)
|
|
const fakeThumb = (w: number, h: number): any => ({
|
|
isEmpty: () => false,
|
|
getSize: () => ({ width: w, height: h }),
|
|
toPNG: () => Buffer.from([137, 80, 78, 71])
|
|
});
|
|
electronState.sources = [{ display_id: "2", thumbnail: fakeThumb(2560, 1440) }];
|
|
detectMock.mockImplementationOnce(async () => ({
|
|
texts: [{ text: "副屏按钮", box: [[400, 200], [500, 200], [500, 220], [400, 220]] }]
|
|
}));
|
|
await expect(getHandler("CV_MouseClick")!({ log: vi.fn() } as any, { id: "n5", componentId: "CV_MouseClick" } as any, {
|
|
cvSelector: "副屏按钮"
|
|
}, {} as any)).resolves.toBe(true);
|
|
const script = String((powershellMock.mock.calls as unknown as Array<[string]>).at(-1)?.[0] ?? "");
|
|
expect(script).toContain("SetCursorPos(2145, 105)");
|
|
});
|
|
|
|
it("maps region-image hits through the display scale", async () => {
|
|
// 框选图来自缩放2的副屏: 图内物理中心(120,90) + 框选DIP原点(1920,0) → DIP(1980,45)
|
|
const regionShot = path.join(os.tmpdir(), "rpa-cv-region-test-" + Date.now() + ".png");
|
|
fs.writeFileSync(regionShot, Buffer.from([137, 80, 78, 71]));
|
|
try {
|
|
detectMock.mockImplementationOnce(async () => ({
|
|
texts: [{ text: "确认", box: [[100, 80], [140, 80], [140, 100], [100, 100]] }]
|
|
}));
|
|
await expect(getHandler("CV_MouseHover")!({ log: vi.fn() } as any, { id: "n6", componentId: "CV_MouseHover" } as any, {
|
|
cvSelector: "确认",
|
|
searchRegion: JSON.stringify({ x: 1920, y: 0, width: 640, height: 360, scale: 2, imagePath: regionShot })
|
|
}, {} as any)).resolves.toBe(true);
|
|
const script = String((powershellMock.mock.calls as unknown as Array<[string]>).at(-1)?.[0] ?? "");
|
|
expect(script).toContain("SetCursorPos(1980, 45)");
|
|
} finally {
|
|
try { fs.unlinkSync(regionShot); } catch {}
|
|
}
|
|
});
|
|
});
|