From e7669acd16af0a701518e2bcc815bad5bb737447 Mon Sep 17 00:00:00 2001 From: yu <191972707@qq.com> Date: Tue, 3 Nov 2026 11:48:41 +0800 Subject: [PATCH 1/2] =?UTF-8?q?feat:=20=E5=9B=BE=E5=83=8F=E6=9F=A5?= =?UTF-8?q?=E6=89=BE=E6=90=9C=E7=B4=A2=E6=95=B4=E4=B8=AA=E8=99=9A=E6=8B=9F?= =?UTF-8?q?=E6=A1=8C=E9=9D=A2(=E5=A4=9A=E5=B1=8F=E6=8B=BC=E6=8E=A5),=20?= =?UTF-8?q?=E7=AA=97=E5=8F=A3=E7=9F=A9=E5=BD=A2=E4=B8=8E=E5=8C=BA=E5=9F=9F?= =?UTF-8?q?JSON=E7=BB=9F=E4=B8=80DPI=E6=8D=A2=E7=AE=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/capabilities/vision.ts | 144 +++++++++++++++++++++----------- 1 file changed, 94 insertions(+), 50 deletions(-) diff --git a/src/main/capabilities/vision.ts b/src/main/capabilities/vision.ts index 45d918f..ba4ee00 100644 --- a/src/main/capabilities/vision.ts +++ b/src/main/capabilities/vision.ts @@ -10,36 +10,69 @@ function text(args: Record, keys: string[], fallback = ""): str return fallback; } -/** 主屏物理分辨率截图。desktopCapturer thumbnailSize 乘 scaleFactor， - * 与「屏幕框选」生成的模板（物理像素裁剪）尺度一致—— - * 若用 DIP 逻辑分辨率, 缩放≠100% 的显示器上模板比截图里的同款 UI 大 scaleFactor 倍, 像素匹配永远对不齐 */ -async function capturePrimaryScreenPng(): Promise { +/** 全虚拟桌面（所有显示器拼接）物理分辨率截图。 + * desktopCapturer thumbnailSize 乘 scaleFactor 与「屏幕框选」模板（物理像素裁剪）尺度一致； + * 逐屏截图后按各自物理原点拼到一张画布上——目标在副屏也能搜到。 + * 返回 originX/originY = 拼接画布左上角对应的屏幕物理坐标（副屏在主屏左侧时为负），用于结果坐标平移。 */ +async function captureVirtualScreenPng(): Promise<{ png: PNG; originX: number; originY: number }> { const { desktopCapturer, screen } = await import("electron"); - const primary = screen.getPrimaryDisplay(); - const scale = primary.scaleFactor || 1; - const sources = await desktopCapturer.getSources({ - types: ["screen"], - thumbnailSize: { - width: Math.round(primary.size.width * scale), - height: Math.round(primary.size.height * scale) + const displays = screen.getAllDisplays(); + if (displays.length === 0) throw new RuntimeError("desktop_error", "无法获取屏幕信息"); + // 每屏按自己的物理分辨率截图（display_id 匹配 → 缩略图尺寸匹配 → 按物理原点排序对位） + const captures: { display: Electron.Display; origin: { x: number; y: number }; png: PNG }[] = []; + for (const display of displays) { + const scale = display.scaleFactor || 1; + const wantW = Math.round(display.bounds.width * scale); + const wantH = Math.round(display.bounds.height * scale); + const sources = await desktopCapturer.getSources({ + types: ["screen"], + thumbnailSize: { width: wantW, height: wantH } + }); + const source = sources.find(item => item.display_id === String(display.id)) + || sources.find(item => item.thumbnail.getSize().width === wantW) + || sources[0]; + if (!source || source.thumbnail.isEmpty()) { + throw new RuntimeError("desktop_error", "无法获取屏幕截图"); } - }); - const source = sources[0]; - if (!source) throw new RuntimeError("desktop_error", "无法获取屏幕截图"); - return PNG.sync.read(source.thumbnail.toPNG()); + // 显示器 DIP 原点的物理坐标 = 该屏物理原点（屏内线性映射: 物理 = (DIP-原点)*缩放 + 物理原点） + const origin = screen.dipToScreenPoint({ x: display.bounds.x, y: display.bounds.y }); + captures.push({ display, origin, png: PNG.sync.read(source.thumbnail.toPNG()) }); + } + const originX = Math.min(...captures.map(c => c.origin.x)); + const originY = Math.min(...captures.map(c => c.origin.y)); + const endX = Math.max(...captures.map(c => c.origin.x + c.png.width)); + const endY = Math.max(...captures.map(c => c.origin.y + c.png.height)); + const width = endX - originX; + const height = endY - originY; + const { createCanvas } = await import("@napi-rs/canvas"); + const canvas = createCanvas(width, height); + const context = canvas.getContext("2d"); + for (const capture of captures) { + const sourceCanvas = createCanvas(capture.png.width, capture.png.height); + const sourceContext = sourceCanvas.getContext("2d"); + const imageData = sourceContext.createImageData(capture.png.width, capture.png.height); + imageData.data.set(capture.png.data); + sourceContext.putImageData(imageData, 0, 0); + context.drawImage(sourceCanvas, capture.origin.x - originX, capture.origin.y - originY); + } + const stitched = context.getImageData(0, 0, width, height); + const png = new PNG({ width, height }); + png.data = Buffer.from(stitched.data); + return { png, originX, originY }; } -/** 主屏缩放比：区域 JSON 等参数按 DIP 语义存储，裁剪物理分辨率截图前需换算 */ -async function primaryScale(): Promise { +/** DIP 区域 → 物理像素区域：左上角经 dipToScreenPoint 换算（天然支持副屏/负坐标），宽高乘所在屏缩放比 */ +async function dipRegionToPhysical(region: { x: number; y: number; width: number; height: number }): Promise<{ x: number; y: number; width: number; height: number }> { const { screen } = await import("electron"); - return screen.getPrimaryDisplay().scaleFactor || 1; -} - -/** DIP 区域 → 物理像素区域（截图已改为物理分辨率，坐标需同步放大） */ -function scaleRegion(region: { x: number; y: number; width: number; height: number }, scale: number) { + const topLeft = screen.dipToScreenPoint({ x: Math.round(region.x), y: Math.round(region.y) }); + const display = screen.getDisplayMatching({ + x: region.x, y: region.y, + width: Math.max(1, Math.round(region.width)), height: Math.max(1, Math.round(region.height)) + }); + const scale = display.scaleFactor || 1; return { - x: Math.round(region.x * scale), - y: Math.round(region.y * scale), + x: topLeft.x, + y: topLeft.y, width: Math.max(1, Math.round(region.width * scale)), height: Math.max(1, Math.round(region.height * scale)) }; @@ -153,6 +186,7 @@ register("Image_Screenshot", async (ctx, node, args) => { let image: PNG; let output: string; + let screenOrigin: { x: number; y: number } | null = null; if (source === "2") { // 指定图片文件来源：image_path 为输入图片；输出走 输出目录+文件名，两者均未配置时直接返回源文件 if (!imageArg.trim()) { @@ -165,7 +199,10 @@ register("Image_Screenshot", async (ctx, node, args) => { output = folder ? path.resolve(path.join(folder, name)) : sourceFile; } else { // 当前屏幕来源：image_path 即保存路径（文件夹则自动追加默认文件名），缺省用 输出目录+文件名；再缺省落流程证据目录（路径可预期） - image = await capturePrimaryScreenPng(); + // 截取整个虚拟桌面（含副屏），物理分辨率 + const shot = await captureVirtualScreenPng(); + image = shot.png; + screenOrigin = { x: shot.originX, y: shot.originY }; output = await resolveOutputPath( path.resolve(imageArg || (folder ? path.join(folder, name) : path.join(ctx.evidenceDir, name))), name @@ -179,9 +216,19 @@ register("Image_Screenshot", async (ctx, node, args) => { recommendation: '在 position 填写 {"x":0,"y":0,"width":100,"height":80} 或 [x, y, width, height]，或将截图范围改回「整个屏幕」' }); } - // 区域 JSON 是 DIP 坐标，截图是物理分辨率，裁剪前换算 - const scale = await primaryScale(); - image = cropPng(image, clampRegion(scaleRegion(parseRegion(regionRaw, image), scale), image)); + if (screenOrigin) { + // 屏幕来源: 区域 JSON 是 DIP 坐标（支持副屏），换算物理坐标后减去拼接画布原点再裁剪 + const phys = await dipRegionToPhysical(parseRegion(regionRaw, image)); + image = cropPng(image, clampRegion({ + x: phys.x - screenOrigin.x, + y: phys.y - screenOrigin.y, + width: phys.width, + height: phys.height + }, image)); + } else { + // 图片文件来源: 区域按该图片自身像素坐标 + image = cropPng(image, parseRegion(regionRaw, image)); + } } await fs.mkdir(path.dirname(output), { recursive: true }); @@ -191,20 +238,15 @@ register("Image_Screenshot", async (ctx, node, args) => { interface MatchResult { x: number; y: number; width: number; height: number; score: number; } -async function captureScreen(target: string): Promise { - await fs.mkdir(path.dirname(target), { recursive: true }); - await fs.writeFile(target, PNG.sync.write(await capturePrimaryScreenPng())); - return target; -} - interface WindowRect { x: number; y: number; width: number; height: number } -/** 按标题（精确→包含）或进程名查找窗口，前置并返回窗口矩形（屏幕坐标） */ +/** 按标题（精确→包含）或进程名查找窗口，前置并返回窗口矩形（物理屏幕坐标） */ async function focusWindowRect(windowObj: string): Promise { const quoted = psString(windowObj); const stdout = await powershell(`Add-Type @' using System; using System.Runtime.InteropServices; public static class WinRect { + [DllImport("user32.dll")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value); [DllImport("user32.dll")] public static extern IntPtr FindWindow(string cls, string title); [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr h); [DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr h, int cmd); @@ -212,6 +254,8 @@ public static class WinRect { public struct RECT { public int Left; public int Top; public int Right; public int Bottom; } } '@; +# 不声明 DPI 感知时 GetWindowRect 返回虚拟化坐标, 与物理分辨率截图对不上 +try { [void][WinRect]::SetProcessDpiAwarenessContext([IntPtr](-4)) } catch {}; $h = [WinRect]::FindWindow($null, ${quoted}); if ($h -eq [IntPtr]::Zero) { $p = Get-Process | Where-Object { $_.MainWindowTitle -and ($_.MainWindowTitle -eq ${quoted} -or $_.MainWindowTitle.Contains(${quoted})) } | Select-Object -First 1; if ($p) { $h = $p.MainWindowHandle } } if ($h -eq [IntPtr]::Zero) { Write-Output 'NOT_FOUND'; exit } @@ -337,31 +381,32 @@ async function locate(args: Record, evidenceDir: string, nodeId if (!rawTemplate.trim()) throw new RuntimeError("validation_error", "模板图片路径为空", { nodeId }); const template = path.resolve(rawTemplate); - // 查找范围：1 整个屏幕（默认）/ 2 指定窗口 / 3 指定区域；窗口/区域模式下结果坐标会平移回屏幕坐标系 + // 查找范围：1 整个虚拟桌面（默认，含副屏）/ 2 指定窗口 / 3 指定区域； + // 截图统一为物理分辨率拼接画布, 结果坐标加回画布原点即屏幕物理坐标（可直接用于点击） const rangeMode = text(args, ["find_range"], "1"); const screenshot = path.join(evidenceDir, `${nodeId}_screen.png`); + const shot = await captureVirtualScreenPng(); let offsetX = 0; let offsetY = 0; if (rangeMode === "2") { const windowObj = text(args, ["window_obj"]); if (!windowObj.trim()) throw new RuntimeError("validation_error", "查找范围为「指定窗口」时，窗口对象不能为空", { nodeId, recommendation: "填写窗口标题（或标题关键字、进程名）" }); const rect = await focusWindowRect(windowObj); - const screen = await capturePrimaryScreenPng(); - await fs.writeFile(screenshot, PNG.sync.write(cropPng(screen, clampRegion(rect, screen)))); + const cropRect = clampRegion({ x: rect.x - shot.originX, y: rect.y - shot.originY, width: rect.width, height: rect.height }, shot.png); + await fs.writeFile(screenshot, PNG.sync.write(cropPng(shot.png, cropRect))); offsetX = rect.x; offsetY = rect.y; } else if (rangeMode === "3") { const regionRaw = text(args, ["shot_position", "action", "position"]); if (!regionRaw.trim()) throw new RuntimeError("validation_error", "查找范围为「指定区域」时，需要填写区域 JSON", { nodeId, recommendation: '在 shot_position 填写 {"x":0,"y":0,"width":100,"height":80} 或 [x, y, width, height]' }); - const scale = await primaryScale(); - const screen = await capturePrimaryScreenPng(); - // 区域 JSON 是 DIP 坐标（框选工具按 DIP 返回），截图是物理分辨率，裁剪前放大 - const region = clampRegion(scaleRegion(parseRegion(regionRaw, screen), scale), screen); - await fs.writeFile(screenshot, PNG.sync.write(cropPng(screen, region))); - offsetX = region.x; - offsetY = region.y; + // 区域 JSON 是 DIP 坐标（框选工具按 DIP 返回, 支持副屏负坐标），换算物理坐标后减画布原点裁剪 + const phys = await dipRegionToPhysical(parseRegion(regionRaw, shot.png)); + const cropRect = clampRegion({ x: phys.x - shot.originX, y: phys.y - shot.originY, width: phys.width, height: phys.height }, shot.png); + await fs.writeFile(screenshot, PNG.sync.write(cropPng(shot.png, cropRect))); + offsetX = phys.x; + offsetY = phys.y; } else { - await captureScreen(screenshot); + await fs.writeFile(screenshot, PNG.sync.write(shot.png)); } // accuracy 字段语义为「相似度」(0~1, 越大越严格, 0.9=至少90%像素一致)——与界面文案一致。 @@ -369,11 +414,10 @@ async function locate(args: Record, evidenceDir: string, nodeId // 用户填 0.9 反而放宽到"接受90%像素差异", 导致到处误命中。 const rawAccuracy = Number(text(args, ["accuracy"], "0.8")); const similarity = Math.max(0.05, Math.min(1, Number.isFinite(rawAccuracy) && rawAccuracy > 0 ? rawAccuracy : 0.8)); - // 原生 1:1 匹配失败时, 按主屏缩放比对模板放大/缩小重试—— - // 兼容旧版 DIP 截图存的模板与任意来源的图片 - const scale = await primaryScale(); + // 原生 1:1 匹配失败时, 按缩放比对模板放大/缩小重试——兼容旧版 DIP 截图存的模板与任意来源的图片 + const scale = (await import("electron")).screen.getPrimaryDisplay().scaleFactor || 1; const result = await findImage(template, screenshot, 1 - similarity, scale !== 1 ? [scale, 1 / scale] : []); - if (!result) throw new RuntimeError("not_found", `屏幕中未找到目标图像（要求相似度 ${(similarity * 100).toFixed(0)}%，未达到阈值）`, { nodeId, recommendation: "重新框选模板（框小一点、只框目标图案），或适当调低相似度（如 0.8）" }); + if (!result) throw new RuntimeError("not_found", `屏幕中未找到目标图像（要求相似度 ${(similarity * 100).toFixed(0)}%，未达到阈值）`, { nodeId, recommendation: "重新框选模板（框小一点、只框目标图案），或适当调低相似度（如 0.8）；目标在副屏时确认其清晰可见" }); return { ...result, x: result.x + offsetX, y: result.y + offsetY }; } From 3922447689010e730d731d372d31602741c311a7 Mon Sep 17 00:00:00 2001 From: yu <191972707@qq.com> Date: Tue, 3 Nov 2026 13:07:12 +0800 Subject: [PATCH 2/2] =?UTF-8?q?fix:=20=E5=85=A8=E6=A1=8C=E9=9D=A2=E6=9F=A5?= =?UTF-8?q?=E6=89=BE=E7=BB=93=E6=9E=9C=E5=8A=A0=E5=9B=9E=E6=8B=BC=E6=8E=A5?= =?UTF-8?q?=E7=94=BB=E5=B8=83=E7=89=A9=E7=90=86=E5=8E=9F=E7=82=B9,=20?= =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=89=AF=E5=B1=8F=E7=82=B9=E5=87=BB=E5=9E=82?= =?UTF-8?q?=E7=9B=B4=E5=81=8F=E7=A7=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/capabilities/vision.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/capabilities/vision.ts b/src/main/capabilities/vision.ts index ba4ee00..a8e893b 100644 --- a/src/main/capabilities/vision.ts +++ b/src/main/capabilities/vision.ts @@ -407,6 +407,10 @@ async function locate(args: Record, evidenceDir: string, nodeId offsetY = phys.y; } else { await fs.writeFile(screenshot, PNG.sync.write(shot.png)); + // 拼接画布坐标 → 屏幕物理坐标必须加回画布原点(副屏在主屏左侧/上方时原点为负); + // 漏加会导致点击偏移(实测副屏上移 177px 时点击正好偏下 177px) + offsetX = shot.originX; + offsetY = shot.originY; } // accuracy 字段语义为「相似度」(0~1, 越大越严格, 0.9=至少90%像素一致)——与界面文案一致。