fix: 截屏改用 desktopCapturer, 修复截 Electron 窗口全黑
- GDI CopyFromScreen 截不到 GPU 合成渲染的窗口内容, 用户框选 Electron 编辑器界面时截图全黑, OCR 无从识别 - pickScreenRegion 改用 desktopCapturer 截屏后按 scaleFactor 裁剪(与 AI 拾取 capturePaneForAi 同款方案) - cv-vision grabScreen 同步改为 desktopCapturer 优先, 不可用时回退 GDI 脚本
This commit is contained in:
parent
d98185d1cf
commit
22054fed37
@ -23,8 +23,7 @@ function cvTarget(args: Record<string, unknown>, nodeId: string): string {
|
||||
throw new RuntimeError("validation_error", "CV 目标为空", { nodeId, componentId: nodeId, recommendation: "填写要在屏幕上查找的目标文本" });
|
||||
}
|
||||
|
||||
async function grabScreen(imagePath: string): Promise<void> {
|
||||
fs.mkdirSync(path.dirname(imagePath), { recursive: true });
|
||||
async function gdiGrabScreen(imagePath: string): Promise<void> {
|
||||
const psScript = [
|
||||
// 与点击脚本一致: 先声明 DPI 感知, 否则缩放非 100% 时 Bounds 是逻辑像素, 截屏只覆盖屏幕左上角
|
||||
"Add-Type -MemberDefinition '[DllImport(\"user32.dll\")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value);' -Name RpaCvDpi -Namespace RpaCv | Out-Null;",
|
||||
@ -42,6 +41,26 @@ async function grabScreen(imagePath: string): Promise<void> {
|
||||
await powershell(psScript, 30000);
|
||||
}
|
||||
|
||||
async function grabScreen(imagePath: string): Promise<void> {
|
||||
fs.mkdirSync(path.dirname(imagePath), { recursive: true });
|
||||
// GDI CopyFromScreen 截不到 GPU 合成窗口(Electron 自身界面, 会截出全黑),
|
||||
// 优先用 desktopCapturer 走系统合成器; 不可用时(如单测环境)回退 GDI 脚本
|
||||
try {
|
||||
const { desktopCapturer, screen } = await import("electron");
|
||||
const display = screen.getPrimaryDisplay();
|
||||
const sources = await desktopCapturer.getSources({
|
||||
types: ["screen"],
|
||||
thumbnailSize: { width: display.size.width, height: display.size.height }
|
||||
});
|
||||
const source = sources.find(item => item.display_id === String(display.id)) || sources[0];
|
||||
if (source && !source.thumbnail.isEmpty()) {
|
||||
fs.writeFileSync(imagePath, source.thumbnail.toPNG());
|
||||
return;
|
||||
}
|
||||
} catch {}
|
||||
await gdiGrabScreen(imagePath);
|
||||
}
|
||||
|
||||
interface OcrLine { text: string; box?: number[][]; mean?: number }
|
||||
|
||||
/** 搜索区域: 框选生成时含 imagePath(区域截图)+x/y(屏幕偏移); 也可只给坐标范围 */
|
||||
|
||||
@ -744,17 +744,27 @@ export async function pickScreenRegion(): Promise<{ x: number; y: number; width:
|
||||
const outDir = path.join(app.getPath("userData"), "region-picks");
|
||||
fs.mkdirSync(outDir, { recursive: true });
|
||||
const imagePath = path.join(outDir, `pick-region-${Date.now()}.png`);
|
||||
const psScript = [
|
||||
"Add-Type -AssemblyName System.Drawing | Out-Null;",
|
||||
"$bitmap = New-Object System.Drawing.Bitmap " + region.width + ", " + region.height + ";",
|
||||
"$graphics = [System.Drawing.Graphics]::FromImage();",
|
||||
"$graphics.CopyFromScreen(" + region.x + ", " + region.y + ", 0, 0, $bitmap.Size);",
|
||||
"$graphics.Dispose();",
|
||||
"$bitmap.Save(" + JSON.stringify(imagePath) + ");",
|
||||
"$bitmap.Dispose();",
|
||||
"Write-Output ok"
|
||||
].join("");
|
||||
await run("powershell.exe", ["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-Command", psScript], { timeout: 30000, windowsHide: true });
|
||||
// GDI CopyFromScreen 截不到 GPU 合成窗口(Electron 自身界面), 会截出全黑;
|
||||
// 改用 desktopCapturer 走系统合成器截屏。覆盖层坐标是 DIP, 裁剪时按 scaleFactor 换算物理像素
|
||||
const dipX = sx + region.x;
|
||||
const dipY = sy + region.y;
|
||||
const target = displayForPoint(dipX + region.width / 2, dipY + region.height / 2);
|
||||
const scale = target.scaleFactor || 1;
|
||||
const sources = await desktopCapturer.getSources({
|
||||
types: ["screen"],
|
||||
thumbnailSize: { width: Math.round(target.bounds.width * scale), height: Math.round(target.bounds.height * scale) }
|
||||
});
|
||||
const source = sources.find(item => item.display_id === String(target.id)) || sources[0];
|
||||
if (!source || source.thumbnail.isEmpty()) {
|
||||
throw new RuntimeError("pick_capture_failed", "框选截图失败:未获取到屏幕画面");
|
||||
}
|
||||
const thumbnail = source.thumbnail;
|
||||
const actualScale = thumbnail.getSize().width / Math.max(1, target.bounds.width);
|
||||
const cropX = Math.max(0, Math.min(thumbnail.getSize().width - 1, Math.round((dipX - target.bounds.x) * actualScale)));
|
||||
const cropY = Math.max(0, Math.min(thumbnail.getSize().height - 1, Math.round((dipY - target.bounds.y) * actualScale)));
|
||||
const cropW = Math.max(1, Math.min(thumbnail.getSize().width - cropX, Math.round(region.width * actualScale)));
|
||||
const cropH = Math.max(1, Math.min(thumbnail.getSize().height - cropY, Math.round(region.height * actualScale)));
|
||||
fs.writeFileSync(imagePath, thumbnail.crop({ x: cropX, y: cropY, width: cropW, height: cropH }).toPNG());
|
||||
return { ...region, imagePath };
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user