PRA-only-demo/src/main/capabilities/cv-vision.test.ts

import { beforeEach, describe, expect, it, vi } from "vitest";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";

const powershellMock = vi.hoisted(() => vi.fn(async () => '{"ok":true}'));
const detectMock = vi.hoisted(() => vi.fn(async () => ({
  texts: [
    { text: "确认保存", box: [[0, 0], [20, 0], [20, 10], [0, 10]] },
    { text: "确认", box: [[100, 200], [130, 200], [130, 210], [100, 210]] }
  ]
})));
// desktopCapturer 模拟: 默认无画面(走 GDI 回退), 多屏用例按请求宽度提供对应显示器画面
const electronState = vi.hoisted(() => ({ sources: [] as Array<{ display_id: string; thumbnail: any }> }));

vi.mock("./desktop", () => ({ powershell: powershellMock }));
vi.mock("@gutenye/ocr-node", () => ({
  default: { create: vi.fn(async () => ({ detect: detectMock })) }
}));
vi.mock("electron", () => ({
  desktopCapturer: {
    getSources: vi.fn(async (opts: { thumbnailSize: { width: number } }) =>
      electronState.sources.filter(s => s.thumbnail.getSize().width === opts.thumbnailSize.width))
  },
  screen: {
    getAllDisplays: vi.fn(() => [
      { id: 1, bounds: { x: 0, y: 0, width: 1920, height: 1080 }, scaleFactor: 1 },
      { id: 2, bounds: { x: 1920, y: 0, width: 1280, height: 720 }, scaleFactor: 2 }
    ]),
    dipToScreenPoint: vi.fn((p: { x: number; y: number }) => ({ x: p.x, y: p.y }))
  }
}));

import { getHandler } from "../engine/registry";
import "./cv-vision";

describe("CV vision capabilities", () => {
  beforeEach(() => {
    powershellMock.mockClear();
    detectMock.mockClear();
    electronState.sources = [];
  });

  it.each(["CV_GetText", "CV_SetText", "CV_MouseHover", "CV_MouseClick"])("registers %s", (id) => {
    expect(getHandler(id), id).toBeTruthy();
  });

  it("rejects an empty CV selector", async () => {
    await expect(getHandler("CV_GetText")!({ log: vi.fn() } as any, { id: "n1", componentId: "CV_GetText" } as any, {}, {} as any))
      .rejects.toThrow("CV 目标为空");
  });

  it("locates OCR text and clicks its center with offset", async () => {
    const handler = getHandler("CV_MouseClick")!;
    await expect(handler({ log: vi.fn() } as any, { id: "n1", componentId: "CV_MouseClick" } as any, {
      cvSelector: "确认",
      offsetX: 5,
      offsetY: 2
    }, {} as any)).resolves.toBe(true);
    expect(detectMock).toHaveBeenCalledTimes(1);
    const script = String((powershellMock.mock.calls as unknown as Array<[string]>)[1]?.[0] ?? "");
    expect(script).toContain("SetCursorPos(15, 7)");
    expect(script).toContain("mouse_event(0x0002");
  });

  it("returns matched OCR text", async () => {
    await expect(getHandler("CV_GetText")!({ log: vi.fn() } as any, { id: "n2", componentId: "CV_GetText" } as any, {
      cvSelector: "保存"
    }, {} as any)).resolves.toBe("确认保存");
    expect(powershellMock).toHaveBeenCalledTimes(1);
  });

  it("restricts matching to the search region when provided", async () => {
    // 屏幕上有两处「确认」: 区域(0,0,50,50)内只应命中 (5,5) 那处, 而不是第一个全屏命中 (115,205)
    await expect(getHandler("CV_MouseClick")!({ log: vi.fn() } as any, { id: "n3", componentId: "CV_MouseClick" } as any, {
      cvSelector: "确认",
      searchRegion: JSON.stringify({ x: 0, y: 0, width: 50, height: 50 })
    }, {} as any)).resolves.toBe(true);
    const script = String((powershellMock.mock.calls as unknown as Array<[string]>)[1]?.[0] ?? "");
    expect(script).toContain("SetCursorPos(10, 5)");
  });

  it("rejects when target is outside the search region", async () => {
    await expect(getHandler("CV_MouseClick")!({ log: vi.fn() } as any, { id: "n4", componentId: "CV_MouseClick" } as any, {
      cvSelector: "确认",
      searchRegion: JSON.stringify({ x: 500, y: 500, width: 50, height: 50 })
    }, {} as any)).rejects.toThrow("未找到目标文本");
  });

  it("locates targets on a secondary display and maps DIP coordinates", async () => {
    // 主屏无画面, 副屏(bounds 1920,0 1280x720 缩放2)物理中心(450,210) → DIP(2145,105)
    const fakeThumb = (w: number, h: number): any => ({
      isEmpty: () => false,
      getSize: () => ({ width: w, height: h }),
      toPNG: () => Buffer.from([137, 80, 78, 71])
    });
    electronState.sources = [{ display_id: "2", thumbnail: fakeThumb(2560, 1440) }];
    detectMock.mockImplementationOnce(async () => ({
      texts: [{ text: "副屏按钮", box: [[400, 200], [500, 200], [500, 220], [400, 220]] }]
    }));
    await expect(getHandler("CV_MouseClick")!({ log: vi.fn() } as any, { id: "n5", componentId: "CV_MouseClick" } as any, {
      cvSelector: "副屏按钮"
    }, {} as any)).resolves.toBe(true);
    const script = String((powershellMock.mock.calls as unknown as Array<[string]>).at(-1)?.[0] ?? "");
    expect(script).toContain("SetCursorPos(2145, 105)");
  });

  it("maps region-image hits through the display scale", async () => {
    // 框选图来自缩放2的副屏: 图内物理中心(120,90) + 框选DIP原点(1920,0) → DIP(1980,45)
    const regionShot = path.join(os.tmpdir(), "rpa-cv-region-test-" + Date.now() + ".png");
    fs.writeFileSync(regionShot, Buffer.from([137, 80, 78, 71]));
    try {
      detectMock.mockImplementationOnce(async () => ({
        texts: [{ text: "确认", box: [[100, 80], [140, 80], [140, 100], [100, 100]] }]
      }));
      await expect(getHandler("CV_MouseHover")!({ log: vi.fn() } as any, { id: "n6", componentId: "CV_MouseHover" } as any, {
        cvSelector: "确认",
        searchRegion: JSON.stringify({ x: 1920, y: 0, width: 640, height: 360, scale: 2, imagePath: regionShot })
      }, {} as any)).resolves.toBe(true);
      const script = String((powershellMock.mock.calls as unknown as Array<[string]>).at(-1)?.[0] ?? "");
      expect(script).toContain("SetCursorPos(1980, 45)");
    } finally {
      try { fs.unlinkSync(regionShot); } catch {}
    }
  });
});