diff --git a/src/main/capabilities/cv-vision.ts b/src/main/capabilities/cv-vision.ts index 8a809be..434e9da 100644 --- a/src/main/capabilities/cv-vision.ts +++ b/src/main/capabilities/cv-vision.ts @@ -23,8 +23,7 @@ function cvTarget(args: Record, nodeId: string): string { throw new RuntimeError("validation_error", "CV 目标为空", { nodeId, componentId: nodeId, recommendation: "填写要在屏幕上查找的目标文本" }); } -async function grabScreen(imagePath: string): Promise { - fs.mkdirSync(path.dirname(imagePath), { recursive: true }); +async function gdiGrabScreen(imagePath: string): Promise { const psScript = [ // 与点击脚本一致: 先声明 DPI 感知, 否则缩放非 100% 时 Bounds 是逻辑像素, 截屏只覆盖屏幕左上角 "Add-Type -MemberDefinition '[DllImport(\"user32.dll\")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value);' -Name RpaCvDpi -Namespace RpaCv | Out-Null;", @@ -42,6 +41,26 @@ async function grabScreen(imagePath: string): Promise { await powershell(psScript, 30000); } +async function grabScreen(imagePath: string): Promise { + fs.mkdirSync(path.dirname(imagePath), { recursive: true }); + // GDI CopyFromScreen 截不到 GPU 合成窗口(Electron 自身界面, 会截出全黑), + // 优先用 desktopCapturer 走系统合成器; 不可用时(如单测环境)回退 GDI 脚本 + try { + const { desktopCapturer, screen } = await import("electron"); + const display = screen.getPrimaryDisplay(); + const sources = await desktopCapturer.getSources({ + types: ["screen"], + thumbnailSize: { width: display.size.width, height: display.size.height } + }); + const source = sources.find(item => item.display_id === String(display.id)) || sources[0]; + if (source && !source.thumbnail.isEmpty()) { + fs.writeFileSync(imagePath, source.thumbnail.toPNG()); + return; + } + } catch {} + await gdiGrabScreen(imagePath); +} + interface OcrLine { text: string; box?: number[][]; mean?: number } /** 搜索区域: 框选生成时含 imagePath(区域截图)+x/y(屏幕偏移); 也可只给坐标范围 */ diff --git a/src/main/desktop-pick.ts b/src/main/desktop-pick.ts index ce923bd..b7d7931 100644 --- a/src/main/desktop-pick.ts +++ b/src/main/desktop-pick.ts @@ -744,17 +744,27 @@ export async function pickScreenRegion(): Promise<{ x: number; y: number; width: const outDir = path.join(app.getPath("userData"), "region-picks"); fs.mkdirSync(outDir, { recursive: true }); const imagePath = path.join(outDir, `pick-region-${Date.now()}.png`); - const psScript = [ - "Add-Type -AssemblyName System.Drawing | Out-Null;", - "$bitmap = New-Object System.Drawing.Bitmap " + region.width + ", " + region.height + ";", - "$graphics = [System.Drawing.Graphics]::FromImage();", - "$graphics.CopyFromScreen(" + region.x + ", " + region.y + ", 0, 0, $bitmap.Size);", - "$graphics.Dispose();", - "$bitmap.Save(" + JSON.stringify(imagePath) + ");", - "$bitmap.Dispose();", - "Write-Output ok" - ].join(""); - await run("powershell.exe", ["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-Command", psScript], { timeout: 30000, windowsHide: true }); + // GDI CopyFromScreen 截不到 GPU 合成窗口(Electron 自身界面), 会截出全黑; + // 改用 desktopCapturer 走系统合成器截屏。覆盖层坐标是 DIP, 裁剪时按 scaleFactor 换算物理像素 + const dipX = sx + region.x; + const dipY = sy + region.y; + const target = displayForPoint(dipX + region.width / 2, dipY + region.height / 2); + const scale = target.scaleFactor || 1; + const sources = await desktopCapturer.getSources({ + types: ["screen"], + thumbnailSize: { width: Math.round(target.bounds.width * scale), height: Math.round(target.bounds.height * scale) } + }); + const source = sources.find(item => item.display_id === String(target.id)) || sources[0]; + if (!source || source.thumbnail.isEmpty()) { + throw new RuntimeError("pick_capture_failed", "框选截图失败:未获取到屏幕画面"); + } + const thumbnail = source.thumbnail; + const actualScale = thumbnail.getSize().width / Math.max(1, target.bounds.width); + const cropX = Math.max(0, Math.min(thumbnail.getSize().width - 1, Math.round((dipX - target.bounds.x) * actualScale))); + const cropY = Math.max(0, Math.min(thumbnail.getSize().height - 1, Math.round((dipY - target.bounds.y) * actualScale))); + const cropW = Math.max(1, Math.min(thumbnail.getSize().width - cropX, Math.round(region.width * actualScale))); + const cropH = Math.max(1, Math.min(thumbnail.getSize().height - cropY, Math.round(region.height * actualScale))); + fs.writeFileSync(imagePath, thumbnail.crop({ x: cropX, y: cropY, width: cropW, height: cropH }).toPNG()); return { ...region, imagePath }; }