feat: 视觉能力去 Electron 化,经 HostBridge.screen 注入(迁移阶段1-A9)
- HostBridge 新增可选 screen 服务(listDisplays/primaryDisplay/dipToScreenPoint/ displayMatching/captureDisplay),vision 3 处、cv-vision 2 处动态 import(electron) 全部改为 requireScreenService();未注入时抛稳定的 host_unavailable - demo 宿主 src/main/index.ts 用 desktopCapturer/screen 实现 screen 服务 - check-deps.mjs 升级:新增动态 import()/require() 扫描(旧正则实测 0 命中的盲区), 合成用例验证命中,升级后真实源码 0 违规 - 包依赖闭包:补声明 @cantoo/pdf-lib、@napi-rs/canvas、pdfjs-dist - cv-vision 测试从 vi.mock(electron) 改为宿主桥注入,11/11 通过
This commit is contained in:
1 parent
1da00827ac
commit
1a20dd72d1
7 files changed
+161
-62
No files matched your search
@@ -10,30 +10,50 @@ const detectMock = vi.hoisted(() => vi.fn(async () => ({
|
||||
{ text: "确认", box: [[100, 200], [130, 200], [130, 210], [100, 210]] }
|
||||
]
|
||||
})));
|
||||
// desktopCapturer 模拟: 默认无画面(走 GDI 回退), 多屏用例按请求宽度提供对应显示器画面
|
||||
// 屏幕服务模拟: 默认无画面(走 GDI 回退), 多屏用例按显示器 id/请求宽度提供对应画面
|
||||
const electronState = vi.hoisted(() => ({ sources: [] as Array<{ display_id: string; thumbnail: any }> }));
|
||||
|
||||
vi.mock("../driver/windows/powershell", () => ({ powershell: powershellMock }));
|
||||
vi.mock("@gutenye/ocr-node", () => ({
|
||||
default: { create: vi.fn(async () => ({ detect: detectMock })) }
|
||||
}));
|
||||
vi.mock("electron", () => ({
|
||||
desktopCapturer: {
|
||||
getSources: vi.fn(async (opts: { thumbnailSize: { width: number } }) =>
|
||||
electronState.sources.filter(s => s.thumbnail.getSize().width === opts.thumbnailSize.width))
|
||||
},
|
||||
screen: {
|
||||
getAllDisplays: vi.fn(() => [
|
||||
{ id: 1, bounds: { x: 0, y: 0, width: 1920, height: 1080 }, scaleFactor: 1 },
|
||||
{ id: 2, bounds: { x: 1920, y: 0, width: 1280, height: 720 }, scaleFactor: 2 }
|
||||
]),
|
||||
dipToScreenPoint: vi.fn((p: { x: number; y: number }) => ({ x: p.x, y: p.y }))
|
||||
}
|
||||
}));
|
||||
|
||||
import { getHandler } from "../engine/registry";
|
||||
import { setHostBridge, type HostDisplay } from "../engine/host-bridge";
|
||||
import "./cv-vision";
|
||||
|
||||
const displayOf = (id: string, x: number, width: number, height: number, scaleFactor: number): HostDisplay => ({
|
||||
id,
|
||||
bounds: { x, y: 0, width, height },
|
||||
scaleFactor
|
||||
});
|
||||
const displays: HostDisplay[] = [
|
||||
displayOf("1", 0, 1920, 1080, 1),
|
||||
displayOf("2", 1920, 1280, 720, 2)
|
||||
];
|
||||
|
||||
setHostBridge({
|
||||
createFormWindow: () => { throw new Error("not used in cv-vision tests"); },
|
||||
dipToScreenPoint: (point) => ({ x: point.x, y: point.y }),
|
||||
screenToDipPoint: (point) => ({ x: point.x, y: point.y }),
|
||||
loadAiConfig: () => ({ baseUrl: "", apiKey: "", model: "", visionModel: "" }),
|
||||
screen: {
|
||||
listDisplays: () => displays,
|
||||
primaryDisplay: () => displays[0],
|
||||
dipToScreenPoint: (point) => ({ x: point.x, y: point.y }),
|
||||
displayMatching: () => displays[0],
|
||||
captureDisplay: async (displayId: string, wantWidth: number, wantHeight: number) => {
|
||||
// 模拟真实 captureDisplay 的对位规则,但不做 sources[0] 兜底:
|
||||
// 保持"主屏无画面 → 跳过该屏"的旧多屏语义
|
||||
const source = electronState.sources.find((item) => item.display_id === displayId && item.thumbnail.getSize().width === wantWidth)
|
||||
|| electronState.sources.find((item) => item.thumbnail.getSize().width === wantWidth);
|
||||
if (!source || source.thumbnail.isEmpty()) throw new Error(`无法获取显示器 ${displayId} 的屏幕截图`);
|
||||
void wantHeight;
|
||||
return source.thumbnail.toPNG();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
describe("CV vision capabilities", () => {
|
||||
beforeEach(() => {
|
||||
powershellMock.mockClear();
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { register } from "../engine/registry";
|
||||
import { RuntimeError, type CapabilityHandler } from "../engine/types";
|
||||
import { requireScreenService } from "../engine/host-bridge";
|
||||
import { powershell } from "../driver/windows/powershell";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
@@ -44,30 +45,27 @@ async function gdiGrabScreen(imagePath: string): Promise<void> {
|
||||
/** 一次屏幕截图: 图片左上角在虚拟桌面上的绝对 DIP 坐标 + 物理/DIP 缩放比 */
|
||||
interface ScreenShot { imagePath: string; dipX: number; dipY: number; scale: number }
|
||||
|
||||
/** desktopCapturer 逐显示器截屏(每屏原生分辨率), 不可用时回退 GDI 全虚拟桌面单图 */
|
||||
/** 宿主屏幕服务逐显示器截屏(每屏原生分辨率), 不可用时回退 GDI 全虚拟桌面单图 */
|
||||
async function grabScreenAll(baseDir: string): Promise<ScreenShot[]> {
|
||||
fs.mkdirSync(baseDir, { recursive: true });
|
||||
try {
|
||||
const { desktopCapturer, screen } = await import("electron");
|
||||
if (desktopCapturer && screen?.getAllDisplays) {
|
||||
const shots: ScreenShot[] = [];
|
||||
const displays = screen.getAllDisplays();
|
||||
for (const display of displays) {
|
||||
const scale = display.scaleFactor || 1;
|
||||
const sources = await desktopCapturer.getSources({
|
||||
types: ["screen"],
|
||||
thumbnailSize: { width: Math.round(display.bounds.width * scale), height: Math.round(display.bounds.height * scale) }
|
||||
});
|
||||
const source = sources.find(item => item.display_id === String(display.id))
|
||||
|| sources.find(item => item.thumbnail.getSize().width === Math.round(display.bounds.width * scale))
|
||||
|| sources[0];
|
||||
if (!source || source.thumbnail.isEmpty()) continue;
|
||||
const screenHost = requireScreenService();
|
||||
const shots: ScreenShot[] = [];
|
||||
for (const display of screenHost.listDisplays()) {
|
||||
const scale = display.scaleFactor || 1;
|
||||
try {
|
||||
const pngBytes = await screenHost.captureDisplay(
|
||||
display.id,
|
||||
Math.round(display.bounds.width * scale),
|
||||
Math.round(display.bounds.height * scale)
|
||||
);
|
||||
if (!pngBytes?.length) continue;
|
||||
const imagePath = path.join(baseDir, `rpa-cv-screen-${display.id}-${Date.now()}.png`);
|
||||
fs.writeFileSync(imagePath, source.thumbnail.toPNG());
|
||||
fs.writeFileSync(imagePath, Buffer.from(pngBytes));
|
||||
shots.push({ imagePath, dipX: display.bounds.x, dipY: display.bounds.y, scale });
|
||||
}
|
||||
if (shots.length) return shots;
|
||||
} catch {}
|
||||
}
|
||||
if (shots.length) return shots;
|
||||
} catch {}
|
||||
const imagePath = path.join(baseDir, `rpa-cv-screen-${Date.now()}.png`);
|
||||
await gdiGrabScreen(imagePath);
|
||||
@@ -207,14 +205,11 @@ function coordinateScript(x: number, y: number, action: "hover" | "click" | "dou
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
/** DIP → 物理像素(SetCursorPos 在 DPI 感知进程里用物理坐标); 无 Electron API 时按原值(缩放 100% 等价) */
|
||||
/** DIP → 物理像素(SetCursorPos 在 DPI 感知进程里用物理坐标); 无宿主屏幕服务时按原值(缩放 100% 等价) */
|
||||
async function toPhysical(x: number, y: number): Promise<{ x: number; y: number }> {
|
||||
try {
|
||||
const { screen } = await import("electron");
|
||||
if (screen && typeof screen.dipToScreenPoint === "function") {
|
||||
const p = screen.dipToScreenPoint({ x: Math.round(x), y: Math.round(y) });
|
||||
return { x: p.x, y: p.y };
|
||||
}
|
||||
const point = requireScreenService().dipToScreenPoint({ x: Math.round(x), y: Math.round(y) });
|
||||
return { x: point.x, y: point.y };
|
||||
} catch {}
|
||||
return { x: Math.round(x), y: Math.round(y) };
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ import { promises as fs } from "node:fs";
|
||||
import { PNG } from "pngjs";
|
||||
import { register } from "../engine/registry";
|
||||
import { RuntimeError } from "../engine/types";
|
||||
import { requireScreenService, type HostDisplay } from "../engine/host-bridge";
|
||||
import { powershell, psString } from "../driver/windows/powershell";
|
||||
|
||||
function text(args: Record<string, unknown>, keys: string[], fallback = ""): string {
|
||||
@@ -15,28 +16,20 @@ function text(args: Record<string, unknown>, keys: string[], fallback = ""): str
|
||||
* 逐屏截图后按各自物理原点拼到一张画布上——目标在副屏也能搜到。
|
||||
* 返回 originX/originY = 拼接画布左上角对应的屏幕物理坐标(副屏在主屏左侧时为负),用于结果坐标平移。 */
|
||||
async function captureVirtualScreenPng(): Promise<{ png: PNG; originX: number; originY: number }> {
|
||||
const { desktopCapturer, screen } = await import("electron");
|
||||
const displays = screen.getAllDisplays();
|
||||
const screenHost = requireScreenService();
|
||||
const displays = screenHost.listDisplays();
|
||||
if (displays.length === 0) throw new RuntimeError("desktop_error", "无法获取屏幕信息");
|
||||
// 每屏按自己的物理分辨率截图(display_id 匹配 → 缩略图尺寸匹配 → 按物理原点排序对位)
|
||||
const captures: { display: Electron.Display; origin: { x: number; y: number }; png: PNG }[] = [];
|
||||
// 每屏按自己的物理分辨率截图(宿主按 displayId 对位 → 按物理原点排序对位)
|
||||
const captures: { display: HostDisplay; origin: { x: number; y: number }; png: PNG }[] = [];
|
||||
for (const display of displays) {
|
||||
const scale = display.scaleFactor || 1;
|
||||
const wantW = Math.round(display.bounds.width * scale);
|
||||
const wantH = Math.round(display.bounds.height * scale);
|
||||
const sources = await desktopCapturer.getSources({
|
||||
types: ["screen"],
|
||||
thumbnailSize: { width: wantW, height: wantH }
|
||||
});
|
||||
const source = sources.find(item => item.display_id === String(display.id))
|
||||
|| sources.find(item => item.thumbnail.getSize().width === wantW)
|
||||
|| sources[0];
|
||||
if (!source || source.thumbnail.isEmpty()) {
|
||||
throw new RuntimeError("desktop_error", "无法获取屏幕截图");
|
||||
}
|
||||
const pngBytes = await screenHost.captureDisplay(display.id, wantW, wantH);
|
||||
if (!pngBytes?.length) throw new RuntimeError("desktop_error", "无法获取屏幕截图");
|
||||
// 显示器 DIP 原点的物理坐标 = 该屏物理原点(屏内线性映射: 物理 = (DIP-原点)*缩放 + 物理原点)
|
||||
const origin = screen.dipToScreenPoint({ x: display.bounds.x, y: display.bounds.y });
|
||||
captures.push({ display, origin, png: PNG.sync.read(source.thumbnail.toPNG()) });
|
||||
const origin = screenHost.dipToScreenPoint({ x: display.bounds.x, y: display.bounds.y });
|
||||
captures.push({ display, origin, png: PNG.sync.read(Buffer.from(pngBytes)) });
|
||||
}
|
||||
const originX = Math.min(...captures.map(c => c.origin.x));
|
||||
const originY = Math.min(...captures.map(c => c.origin.y));
|
||||
@@ -63,9 +56,9 @@ async function captureVirtualScreenPng(): Promise<{ png: PNG; originX: number; o
|
||||
|
||||
/** DIP 区域 → 物理像素区域:左上角经 dipToScreenPoint 换算(天然支持副屏/负坐标),宽高乘所在屏缩放比 */
|
||||
async function dipRegionToPhysical(region: { x: number; y: number; width: number; height: number }): Promise<{ x: number; y: number; width: number; height: number }> {
|
||||
const { screen } = await import("electron");
|
||||
const topLeft = screen.dipToScreenPoint({ x: Math.round(region.x), y: Math.round(region.y) });
|
||||
const display = screen.getDisplayMatching({
|
||||
const screenHost = requireScreenService();
|
||||
const topLeft = screenHost.dipToScreenPoint({ x: Math.round(region.x), y: Math.round(region.y) });
|
||||
const display = screenHost.displayMatching({
|
||||
x: region.x, y: region.y,
|
||||
width: Math.max(1, Math.round(region.width)), height: Math.max(1, Math.round(region.height))
|
||||
});
|
||||
@@ -429,7 +422,7 @@ async function locate(args: Record<string, unknown>, evidenceDir: string, nodeId
|
||||
const rawAccuracy = Number(text(args, ["accuracy"], "0.8"));
|
||||
const similarity = Math.max(0.05, Math.min(1, Number.isFinite(rawAccuracy) && rawAccuracy > 0 ? rawAccuracy : 0.8));
|
||||
// 原生 1:1 匹配失败时, 按缩放比对模板放大/缩小重试——兼容旧版 DIP 截图存的模板与任意来源的图片
|
||||
const scale = (await import("electron")).screen.getPrimaryDisplay().scaleFactor || 1;
|
||||
const scale = requireScreenService().primaryDisplay().scaleFactor || 1;
|
||||
const result = await findImage(template, screenshot, 1 - similarity, scale !== 1 ? [scale, 1 / scale] : []);
|
||||
if (!result) throw new RuntimeError("not_found", `屏幕中未找到目标图像(要求相似度 ${(similarity * 100).toFixed(0)}%,未达到阈值)`, { nodeId, recommendation: "重新框选模板(框小一点、只框目标图案),或适当调低相似度(如 0.8);目标在副屏时确认其清晰可见" });
|
||||
return { ...result, x: result.x + offsetX, y: result.y + offsetY };
|
||||
|
||||
Reference in new issue
Block a user