fix: 截屏改用 desktopCapturer, 修复截 Electron 窗口全黑

- GDI CopyFromScreen 截不到 GPU 合成渲染的窗口内容, 用户框选 Electron 编辑器界面时截图全黑, OCR 无从识别
- pickScreenRegion 改用 desktopCapturer 截屏后按 scaleFactor 裁剪(与 AI 拾取 capturePaneForAi 同款方案)
- cv-vision grabScreen 同步改为 desktopCapturer 优先, 不可用时回退 GDI 脚本
This commit is contained in:
yu 2026-11-03 09:17:23 +08:00
parent d98185d1cf
commit 22054fed37
2 changed files with 42 additions and 13 deletions

View File

@ -23,8 +23,7 @@ function cvTarget(args: Record<string, unknown>, nodeId: string): string {
throw new RuntimeError("validation_error", "CV 目标为空", { nodeId, componentId: nodeId, recommendation: "填写要在屏幕上查找的目标文本" });
}
async function grabScreen(imagePath: string): Promise<void> {
fs.mkdirSync(path.dirname(imagePath), { recursive: true });
async function gdiGrabScreen(imagePath: string): Promise<void> {
const psScript = [
// 与点击脚本一致: 先声明 DPI 感知, 否则缩放非 100% 时 Bounds 是逻辑像素, 截屏只覆盖屏幕左上角
"Add-Type -MemberDefinition '[DllImport(\"user32.dll\")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value);' -Name RpaCvDpi -Namespace RpaCv | Out-Null;",
@ -42,6 +41,26 @@ async function grabScreen(imagePath: string): Promise<void> {
await powershell(psScript, 30000);
}
async function grabScreen(imagePath: string): Promise<void> {
fs.mkdirSync(path.dirname(imagePath), { recursive: true });
// GDI CopyFromScreen 截不到 GPU 合成窗口(Electron 自身界面, 会截出全黑),
// 优先用 desktopCapturer 走系统合成器; 不可用时(如单测环境)回退 GDI 脚本
try {
const { desktopCapturer, screen } = await import("electron");
const display = screen.getPrimaryDisplay();
const sources = await desktopCapturer.getSources({
types: ["screen"],
thumbnailSize: { width: display.size.width, height: display.size.height }
});
const source = sources.find(item => item.display_id === String(display.id)) || sources[0];
if (source && !source.thumbnail.isEmpty()) {
fs.writeFileSync(imagePath, source.thumbnail.toPNG());
return;
}
} catch {}
await gdiGrabScreen(imagePath);
}
interface OcrLine { text: string; box?: number[][]; mean?: number }
/** 搜索区域: 框选生成时含 imagePath(区域截图)+x/y(屏幕偏移); 也可只给坐标范围 */

View File

@ -744,17 +744,27 @@ export async function pickScreenRegion(): Promise<{ x: number; y: number; width:
const outDir = path.join(app.getPath("userData"), "region-picks");
fs.mkdirSync(outDir, { recursive: true });
const imagePath = path.join(outDir, `pick-region-${Date.now()}.png`);
const psScript = [
"Add-Type -AssemblyName System.Drawing | Out-Null;",
"$bitmap = New-Object System.Drawing.Bitmap " + region.width + ", " + region.height + ";",
"$graphics = [System.Drawing.Graphics]::FromImage();",
"$graphics.CopyFromScreen(" + region.x + ", " + region.y + ", 0, 0, $bitmap.Size);",
"$graphics.Dispose();",
"$bitmap.Save(" + JSON.stringify(imagePath) + ");",
"$bitmap.Dispose();",
"Write-Output ok"
].join("");
await run("powershell.exe", ["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-Command", psScript], { timeout: 30000, windowsHide: true });
// GDI CopyFromScreen 截不到 GPU 合成窗口(Electron 自身界面), 会截出全黑;
// 改用 desktopCapturer 走系统合成器截屏。覆盖层坐标是 DIP, 裁剪时按 scaleFactor 换算物理像素
const dipX = sx + region.x;
const dipY = sy + region.y;
const target = displayForPoint(dipX + region.width / 2, dipY + region.height / 2);
const scale = target.scaleFactor || 1;
const sources = await desktopCapturer.getSources({
types: ["screen"],
thumbnailSize: { width: Math.round(target.bounds.width * scale), height: Math.round(target.bounds.height * scale) }
});
const source = sources.find(item => item.display_id === String(target.id)) || sources[0];
if (!source || source.thumbnail.isEmpty()) {
throw new RuntimeError("pick_capture_failed", "框选截图失败:未获取到屏幕画面");
}
const thumbnail = source.thumbnail;
const actualScale = thumbnail.getSize().width / Math.max(1, target.bounds.width);
const cropX = Math.max(0, Math.min(thumbnail.getSize().width - 1, Math.round((dipX - target.bounds.x) * actualScale)));
const cropY = Math.max(0, Math.min(thumbnail.getSize().height - 1, Math.round((dipY - target.bounds.y) * actualScale)));
const cropW = Math.max(1, Math.min(thumbnail.getSize().width - cropX, Math.round(region.width * actualScale)));
const cropH = Math.max(1, Math.min(thumbnail.getSize().height - cropY, Math.round(region.height * actualScale)));
fs.writeFileSync(imagePath, thumbnail.crop({ x: cropX, y: cropY, width: cropW, height: cropH }).toPNG());
return { ...region, imagePath };
}