import { beforeEach, describe, expect, it, vi } from "vitest"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; const powershellMock = vi.hoisted(() => vi.fn(async () => '{"ok":true}')); const detectMock = vi.hoisted(() => vi.fn(async () => ({ texts: [ { text: "确认保存", box: [[0, 0], [20, 0], [20, 10], [0, 10]] }, { text: "确认", box: [[100, 200], [130, 200], [130, 210], [100, 210]] } ] }))); // desktopCapturer 模拟: 默认无画面(走 GDI 回退), 多屏用例按请求宽度提供对应显示器画面 const electronState = vi.hoisted(() => ({ sources: [] as Array<{ display_id: string; thumbnail: any }> })); vi.mock("./desktop", () => ({ powershell: powershellMock })); vi.mock("@gutenye/ocr-node", () => ({ default: { create: vi.fn(async () => ({ detect: detectMock })) } })); vi.mock("electron", () => ({ desktopCapturer: { getSources: vi.fn(async (opts: { thumbnailSize: { width: number } }) => electronState.sources.filter(s => s.thumbnail.getSize().width === opts.thumbnailSize.width)) }, screen: { getAllDisplays: vi.fn(() => [ { id: 1, bounds: { x: 0, y: 0, width: 1920, height: 1080 }, scaleFactor: 1 }, { id: 2, bounds: { x: 1920, y: 0, width: 1280, height: 720 }, scaleFactor: 2 } ]), dipToScreenPoint: vi.fn((p: { x: number; y: number }) => ({ x: p.x, y: p.y })) } })); import { getHandler } from "../engine/registry"; import "./cv-vision"; describe("CV vision capabilities", () => { beforeEach(() => { powershellMock.mockClear(); detectMock.mockClear(); electronState.sources = []; }); it.each(["CV_GetText", "CV_SetText", "CV_MouseHover", "CV_MouseClick"])("registers %s", (id) => { expect(getHandler(id), id).toBeTruthy(); }); it("rejects an empty CV selector", async () => { await expect(getHandler("CV_GetText")!({ log: vi.fn() } as any, { id: "n1", componentId: "CV_GetText" } as any, {}, {} as any)) .rejects.toThrow("CV 目标为空"); }); it("locates OCR text and clicks its center with offset", async () => { const handler = getHandler("CV_MouseClick")!; await expect(handler({ log: vi.fn() } as any, { id: "n1", componentId: "CV_MouseClick" } as any, { cvSelector: "确认", offsetX: 5, offsetY: 2 }, {} as any)).resolves.toBe(true); expect(detectMock).toHaveBeenCalledTimes(1); const script = String((powershellMock.mock.calls as unknown as Array<[string]>)[1]?.[0] ?? ""); expect(script).toContain("SetCursorPos(15, 7)"); expect(script).toContain("mouse_event(0x0002"); }); it("returns matched OCR text", async () => { await expect(getHandler("CV_GetText")!({ log: vi.fn() } as any, { id: "n2", componentId: "CV_GetText" } as any, { cvSelector: "保存" }, {} as any)).resolves.toBe("确认保存"); expect(powershellMock).toHaveBeenCalledTimes(1); }); it("restricts matching to the search region when provided", async () => { // 屏幕上有两处「确认」: 区域(0,0,50,50)内只应命中 (5,5) 那处, 而不是第一个全屏命中 (115,205) await expect(getHandler("CV_MouseClick")!({ log: vi.fn() } as any, { id: "n3", componentId: "CV_MouseClick" } as any, { cvSelector: "确认", searchRegion: JSON.stringify({ x: 0, y: 0, width: 50, height: 50 }) }, {} as any)).resolves.toBe(true); const script = String((powershellMock.mock.calls as unknown as Array<[string]>)[1]?.[0] ?? ""); expect(script).toContain("SetCursorPos(10, 5)"); }); it("rejects when target is outside the search region", async () => { await expect(getHandler("CV_MouseClick")!({ log: vi.fn() } as any, { id: "n4", componentId: "CV_MouseClick" } as any, { cvSelector: "确认", searchRegion: JSON.stringify({ x: 500, y: 500, width: 50, height: 50 }) }, {} as any)).rejects.toThrow("未找到目标文本"); }); it("locates targets on a secondary display and maps DIP coordinates", async () => { // 主屏无画面, 副屏(bounds 1920,0 1280x720 缩放2)物理中心(450,210) → DIP(2145,105) const fakeThumb = (w: number, h: number): any => ({ isEmpty: () => false, getSize: () => ({ width: w, height: h }), toPNG: () => Buffer.from([137, 80, 78, 71]) }); electronState.sources = [{ display_id: "2", thumbnail: fakeThumb(2560, 1440) }]; detectMock.mockImplementationOnce(async () => ({ texts: [{ text: "副屏按钮", box: [[400, 200], [500, 200], [500, 220], [400, 220]] }] })); await expect(getHandler("CV_MouseClick")!({ log: vi.fn() } as any, { id: "n5", componentId: "CV_MouseClick" } as any, { cvSelector: "副屏按钮" }, {} as any)).resolves.toBe(true); const script = String((powershellMock.mock.calls as unknown as Array<[string]>).at(-1)?.[0] ?? ""); expect(script).toContain("SetCursorPos(2145, 105)"); }); it("maps region-image hits through the display scale", async () => { // 框选图来自缩放2的副屏: 图内物理中心(120,90) + 框选DIP原点(1920,0) → DIP(1980,45) const regionShot = path.join(os.tmpdir(), "rpa-cv-region-test-" + Date.now() + ".png"); fs.writeFileSync(regionShot, Buffer.from([137, 80, 78, 71])); try { detectMock.mockImplementationOnce(async () => ({ texts: [{ text: "确认", box: [[100, 80], [140, 80], [140, 100], [100, 100]] }] })); await expect(getHandler("CV_MouseHover")!({ log: vi.fn() } as any, { id: "n6", componentId: "CV_MouseHover" } as any, { cvSelector: "确认", searchRegion: JSON.stringify({ x: 1920, y: 0, width: 640, height: 360, scale: 2, imagePath: regionShot }) }, {} as any)).resolves.toBe(true); const script = String((powershellMock.mock.calls as unknown as Array<[string]>).at(-1)?.[0] ?? ""); expect(script).toContain("SetCursorPos(1980, 45)"); } finally { try { fs.unlinkSync(regionShot); } catch {} } }); });