feat: 视觉能力去 Electron 化,经 HostBridge.screen 注入(迁移阶段1-A9)

- HostBridge 新增可选 screen 服务(listDisplays/primaryDisplay/dipToScreenPoint/
  displayMatching/captureDisplay),vision 3 处、cv-vision 2 处动态 import(electron)
  全部改为 requireScreenService();未注入时抛稳定的 host_unavailable
- demo 宿主 src/main/index.ts 用 desktopCapturer/screen 实现 screen 服务
- check-deps.mjs 升级:新增动态 import()/require() 扫描(旧正则实测 0 命中的盲区),
  合成用例验证命中,升级后真实源码 0 违规
- 包依赖闭包:补声明 @cantoo/pdf-lib、@napi-rs/canvas、pdfjs-dist
- cv-vision 测试从 vi.mock(electron) 改为宿主桥注入,11/11 通过
This commit is contained in:
yu committed 2026-11-06 10:53:42 +08:00
1 parent 1da00827ac
commit 1a20dd72d1
7 files changed
+161 -62

No files matched your search

+13 -20
View File
@@ -3,6 +3,7 @@ import { promises as fs } from "node:fs";
import { PNG } from "pngjs";
import { register } from "../engine/registry";
import { RuntimeError } from "../engine/types";
import { requireScreenService, type HostDisplay } from "../engine/host-bridge";
import { powershell, psString } from "../driver/windows/powershell";
function text(args: Record<string, unknown>, keys: string[], fallback = ""): string {
@@ -15,28 +16,20 @@ function text(args: Record<string, unknown>, keys: string[], fallback = ""): str
* 逐屏截图后按各自物理原点拼到一张画布上——目标在副屏也能搜到。
* 返回 originX/originY = 拼接画布左上角对应的屏幕物理坐标(副屏在主屏左侧时为负),用于结果坐标平移。 */
async function captureVirtualScreenPng(): Promise<{ png: PNG; originX: number; originY: number }> {
const { desktopCapturer, screen } = await import("electron");
const displays = screen.getAllDisplays();
const screenHost = requireScreenService();
const displays = screenHost.listDisplays();
if (displays.length === 0) throw new RuntimeError("desktop_error", "无法获取屏幕信息");
// 每屏按自己的物理分辨率截图(display_id 匹配 → 缩略图尺寸匹配 → 按物理原点排序对位)
const captures: { display: Electron.Display; origin: { x: number; y: number }; png: PNG }[] = [];
// 每屏按自己的物理分辨率截图(宿主按 displayId 对位 → 按物理原点排序对位)
const captures: { display: HostDisplay; origin: { x: number; y: number }; png: PNG }[] = [];
for (const display of displays) {
const scale = display.scaleFactor || 1;
const wantW = Math.round(display.bounds.width * scale);
const wantH = Math.round(display.bounds.height * scale);
const sources = await desktopCapturer.getSources({
types: ["screen"],
thumbnailSize: { width: wantW, height: wantH }
});
const source = sources.find(item => item.display_id === String(display.id))
|| sources.find(item => item.thumbnail.getSize().width === wantW)
|| sources[0];
if (!source || source.thumbnail.isEmpty()) {
throw new RuntimeError("desktop_error", "无法获取屏幕截图");
}
const pngBytes = await screenHost.captureDisplay(display.id, wantW, wantH);
if (!pngBytes?.length) throw new RuntimeError("desktop_error", "无法获取屏幕截图");
// 显示器 DIP 原点的物理坐标 = 该屏物理原点(屏内线性映射: 物理 = (DIP-原点)*缩放 + 物理原点)
const origin = screen.dipToScreenPoint({ x: display.bounds.x, y: display.bounds.y });
captures.push({ display, origin, png: PNG.sync.read(source.thumbnail.toPNG()) });
const origin = screenHost.dipToScreenPoint({ x: display.bounds.x, y: display.bounds.y });
captures.push({ display, origin, png: PNG.sync.read(Buffer.from(pngBytes)) });
}
const originX = Math.min(...captures.map(c => c.origin.x));
const originY = Math.min(...captures.map(c => c.origin.y));
@@ -63,9 +56,9 @@ async function captureVirtualScreenPng(): Promise<{ png: PNG; originX: number; o
/** DIP 区域 → 物理像素区域:左上角经 dipToScreenPoint 换算(天然支持副屏/负坐标),宽高乘所在屏缩放比 */
async function dipRegionToPhysical(region: { x: number; y: number; width: number; height: number }): Promise<{ x: number; y: number; width: number; height: number }> {
const { screen } = await import("electron");
const topLeft = screen.dipToScreenPoint({ x: Math.round(region.x), y: Math.round(region.y) });
const display = screen.getDisplayMatching({
const screenHost = requireScreenService();
const topLeft = screenHost.dipToScreenPoint({ x: Math.round(region.x), y: Math.round(region.y) });
const display = screenHost.displayMatching({
x: region.x, y: region.y,
width: Math.max(1, Math.round(region.width)), height: Math.max(1, Math.round(region.height))
});
@@ -429,7 +422,7 @@ async function locate(args: Record<string, unknown>, evidenceDir: string, nodeId
const rawAccuracy = Number(text(args, ["accuracy"], "0.8"));
const similarity = Math.max(0.05, Math.min(1, Number.isFinite(rawAccuracy) && rawAccuracy > 0 ? rawAccuracy : 0.8));
// 原生 1:1 匹配失败时, 按缩放比对模板放大/缩小重试——兼容旧版 DIP 截图存的模板与任意来源的图片
const scale = (await import("electron")).screen.getPrimaryDisplay().scaleFactor || 1;
const scale = requireScreenService().primaryDisplay().scaleFactor || 1;
const result = await findImage(template, screenshot, 1 - similarity, scale !== 1 ? [scale, 1 / scale] : []);
if (!result) throw new RuntimeError("not_found", `屏幕中未找到目标图像(要求相似度 ${(similarity * 100).toFixed(0)}%,未达到阈值)`, { nodeId, recommendation: "重新框选模板(框小一点、只框目标图案),或适当调低相似度(如 0.8);目标在副屏时确认其清晰可见" });
return { ...result, x: result.x + offsetX, y: result.y + offsetY };