import path from "node:path"; import { promises as fs } from "node:fs"; import { PNG } from "pngjs"; import { register } from "../engine/registry"; import { RuntimeError } from "../engine/types"; import { requireScreenService, type HostDisplay } from "../engine/host-bridge"; import { powershell, psString } from "../driver/windows/powershell"; function text(args: Record, keys: string[], fallback = ""): string { for (const key of keys) if (args[key] !== undefined && args[key] !== null && args[key] !== "") return String(args[key]); return fallback; } /** 全虚拟桌面(所有显示器拼接)物理分辨率截图。 * desktopCapturer thumbnailSize 乘 scaleFactor 与「屏幕框选」模板(物理像素裁剪)尺度一致; * 逐屏截图后按各自物理原点拼到一张画布上——目标在副屏也能搜到。 * 返回 originX/originY = 拼接画布左上角对应的屏幕物理坐标(副屏在主屏左侧时为负),用于结果坐标平移。 */ async function captureVirtualScreenPng(): Promise<{ png: PNG; originX: number; originY: number }> { const screenHost = requireScreenService(); const displays = screenHost.listDisplays(); if (displays.length === 0) throw new RuntimeError("desktop_error", "无法获取屏幕信息"); // 每屏按自己的物理分辨率截图(宿主按 displayId 对位 → 按物理原点排序对位) const captures: { display: HostDisplay; origin: { x: number; y: number }; png: PNG }[] = []; for (const display of displays) { const scale = display.scaleFactor || 1; const wantW = Math.round(display.bounds.width * scale); const wantH = Math.round(display.bounds.height * scale); const pngBytes = await screenHost.captureDisplay(display.id, wantW, wantH); if (!pngBytes?.length) throw new RuntimeError("desktop_error", "无法获取屏幕截图"); // 显示器 DIP 原点的物理坐标 = 该屏物理原点(屏内线性映射: 物理 = (DIP-原点)*缩放 + 物理原点) const origin = screenHost.dipToScreenPoint({ x: display.bounds.x, y: display.bounds.y }); captures.push({ display, origin, png: PNG.sync.read(Buffer.from(pngBytes)) }); } const originX = Math.min(...captures.map(c => c.origin.x)); const originY = Math.min(...captures.map(c => c.origin.y)); const endX = Math.max(...captures.map(c => c.origin.x + c.png.width)); const endY = Math.max(...captures.map(c => c.origin.y + c.png.height)); const width = endX - originX; const height = endY - originY; const { createCanvas } = await import("@napi-rs/canvas"); const canvas = createCanvas(width, height); const context = canvas.getContext("2d"); for (const capture of captures) { const sourceCanvas = createCanvas(capture.png.width, capture.png.height); const sourceContext = sourceCanvas.getContext("2d"); const imageData = sourceContext.createImageData(capture.png.width, capture.png.height); imageData.data.set(capture.png.data); sourceContext.putImageData(imageData, 0, 0); context.drawImage(sourceCanvas, capture.origin.x - originX, capture.origin.y - originY); } const stitched = context.getImageData(0, 0, width, height); const png = new PNG({ width, height }); png.data = Buffer.from(stitched.data); return { png, originX, originY }; } /** DIP 区域 → 物理像素区域:左上角经 dipToScreenPoint 换算(天然支持副屏/负坐标),宽高乘所在屏缩放比 */ async function dipRegionToPhysical(region: { x: number; y: number; width: number; height: number }): Promise<{ x: number; y: number; width: number; height: number }> { const screenHost = requireScreenService(); const topLeft = screenHost.dipToScreenPoint({ x: Math.round(region.x), y: Math.round(region.y) }); const display = screenHost.displayMatching({ x: region.x, y: region.y, width: Math.max(1, Math.round(region.width)), height: Math.max(1, Math.round(region.height)) }); const scale = display.scaleFactor || 1; return { x: topLeft.x, y: topLeft.y, width: Math.max(1, Math.round(region.width * scale)), height: Math.max(1, Math.round(region.height * scale)) }; } /** 按魔数判断是否 PNG(89 50 4E 47) */ function isPngBuffer(buffer: Buffer): boolean { return buffer.length > 8 && buffer[0] === 0x89 && buffer[1] === 0x50 && buffer[2] === 0x4e && buffer[3] === 0x47; } /** 解码用户提供的图片文件为 pngjs PNG。pngjs 只认 PNG, 模板/输入图常是 JPG—— * 非 PNG 时走 @napi-rs/canvas 解码(JPG/BMP/WebP 等), 否则报 "unrecognised content at end of stream" */ async function decodeImageFile(filePath: string): Promise { let buffer: Buffer; try { buffer = await fs.readFile(filePath); } catch (error) { if ((error as NodeJS.ErrnoException)?.code === "ENOENT") { throw new RuntimeError("validation_error", `找不到图片文件:${filePath}`, { recommendation: "确认路径正确,或直接点参数旁的「屏幕框选」截取一张" }); } throw error; } if (isPngBuffer(buffer)) return PNG.sync.read(buffer); const { loadImage, createCanvas } = await import("@napi-rs/canvas"); const image = await loadImage(buffer); if (!image.width || !image.height) { throw new RuntimeError("validation_error", "图片解码失败", { detail: filePath }); } const canvas = createCanvas(image.width, image.height); const context = canvas.getContext("2d"); context.drawImage(image, 0, 0); const imageData = context.getImageData(0, 0, image.width, image.height); const png = new PNG({ width: image.width, height: image.height }); png.data = Buffer.from(imageData.data); return png; } /** 解析区域 JSON:对象 {x,y,width,height}(兼容 left/top、x1y1x2y2 角点)或数组 [x,y,width,height],并夹取到图片边界内 */ export function parseRegion(raw: string, image: PNG): { x: number; y: number; width: number; height: number } { let parsed: unknown; try { parsed = JSON.parse(raw); } catch { throw new RuntimeError("validation_error", "区域 JSON 无法解析", { detail: raw.slice(0, 200) }); } const numberOf = (value: unknown): number | null => { const n = Number(value); return Number.isFinite(n) ? n : null; }; if (Array.isArray(parsed) && parsed.length === 4) { const [x, y, width, height] = parsed.map(numberOf); if (x !== null && y !== null && width !== null && height !== null) { return clampRegion({ x, y, width, height }, image); } } if (parsed && typeof parsed === "object") { const record = parsed as Record; const x = numberOf(record.x ?? record.left); const y = numberOf(record.y ?? record.top); const width = numberOf(record.width); const height = numberOf(record.height); if (x !== null && y !== null && width !== null && height !== null) { return clampRegion({ x, y, width, height }, image); } const x2 = numberOf(record.x2 ?? record.right); const y2 = numberOf(record.y2 ?? record.bottom); if (x !== null && y !== null && x2 !== null && y2 !== null) { return clampRegion({ x, y, width: x2 - x, height: y2 - y }, image); } } throw new RuntimeError("validation_error", "区域 JSON 不符合要求", { detail: raw.slice(0, 200), recommendation: '使用 {"x":0,"y":0,"width":100,"height":80} 或 [x, y, width, height]' }); } export function clampRegion(region: { x: number; y: number; width: number; height: number }, image: PNG): { x: number; y: number; width: number; height: number } { const x = Math.max(0, Math.min(Math.round(region.x), image.width - 1)); const y = Math.max(0, Math.min(Math.round(region.y), image.height - 1)); const width = Math.max(1, Math.min(Math.round(region.width), image.width - x)); const height = Math.max(1, Math.min(Math.round(region.height), image.height - y)); return { x, y, width, height }; } export function cropPng(source: PNG, region: { x: number; y: number; width: number; height: number }): PNG { const cropped = new PNG({ width: region.width, height: region.height }); PNG.bitblt(source, cropped, region.x, region.y, region.width, region.height, 0, 0); return cropped; } /** 目标路径容错:填的是文件夹(已存在或以分隔符结尾)时,自动往里存默认文件名 */ async function resolveOutputPath(outputPath: string, defaultName: string): Promise { if (/[\\/]$/.test(outputPath)) return path.join(outputPath, defaultName); try { const stat = await fs.stat(outputPath); if (stat.isDirectory()) return path.join(outputPath, defaultName); } catch { // 路径不存在说明是要新建的文件,原样返回 } return outputPath; } register("Image_Screenshot", async (ctx, node, args) => { const folder = text(args, ["outImagePath", "folder"]); const name = text(args, ["outImageName", "name"], `screenshot_${Date.now()}.png`); const imageArg = text(args, ["image_path"]); const source = text(args, ["pictureSource"], "1"); const range = text(args, ["shotRange"], "1"); let image: PNG; let output: string; let screenOrigin: { x: number; y: number } | null = null; if (source === "2") { // 指定图片文件来源:image_path 为输入图片;输出走 输出目录+文件名,两者均未配置时直接返回源文件 if (!imageArg.trim()) { throw new RuntimeError("validation_error", "图像来源为「指定图片文件」时,截图保存路径需填写源图片路径", { recommendation: "在截图保存路径选择一张已存在的图片,或将图像来源改回「当前屏幕」" }); } const sourceFile = path.resolve(imageArg); image = await decodeImageFile(sourceFile); output = folder ? path.resolve(path.join(folder, name)) : sourceFile; } else { // 当前屏幕来源:image_path 即保存路径(文件夹则自动追加默认文件名),缺省用 输出目录+文件名;再缺省落流程证据目录(路径可预期) // 截取整个虚拟桌面(含副屏),物理分辨率 const shot = await captureVirtualScreenPng(); image = shot.png; screenOrigin = { x: shot.originX, y: shot.originY }; output = await resolveOutputPath( path.resolve(imageArg || (folder ? path.join(folder, name) : path.join(ctx.evidenceDir, name))), name ); } if (range === "2") { const regionRaw = text(args, ["position", "action", "position_img"]); if (!regionRaw.trim()) { throw new RuntimeError("validation_error", "截图范围为「框选区域」时,截图区域需填写区域 JSON", { recommendation: '在「截图区域」点屏幕框选,或填写 {"x":0,"y":0,"width":100,"height":80},或将截图范围改回「整个屏幕」' }); } if (screenOrigin) { // 屏幕来源: 区域 JSON 是 DIP 坐标(支持副屏),换算物理坐标后减去拼接画布原点再裁剪 const phys = await dipRegionToPhysical(parseRegion(regionRaw, image)); image = cropPng(image, clampRegion({ x: phys.x - screenOrigin.x, y: phys.y - screenOrigin.y, width: phys.width, height: phys.height }, image)); } else { // 图片文件来源: 区域按该图片自身像素坐标 image = cropPng(image, parseRegion(regionRaw, image)); } } await fs.mkdir(path.dirname(output), { recursive: true }); await fs.writeFile(output, PNG.sync.write(image)); return output; }); interface MatchResult { x: number; y: number; width: number; height: number; score: number; } interface WindowRect { x: number; y: number; width: number; height: number } /** 按标题(精确→包含)或进程名查找窗口,前置并返回窗口矩形(物理屏幕坐标) */ async function focusWindowRect(windowObj: string): Promise { const quoted = psString(windowObj); const stdout = await powershell(`Add-Type @' using System; using System.Runtime.InteropServices; public static class WinRect { [DllImport("user32.dll")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value); [DllImport("user32.dll")] public static extern IntPtr FindWindow(string cls, string title); [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr h); [DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr h, int cmd); [DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr h, out RECT r); public struct RECT { public int Left; public int Top; public int Right; public int Bottom; } } '@; # 不声明 DPI 感知时 GetWindowRect 返回虚拟化坐标, 与物理分辨率截图对不上 try { [void][WinRect]::SetProcessDpiAwarenessContext([IntPtr](-4)) } catch {}; $h = [WinRect]::FindWindow($null, ${quoted}); if ($h -eq [IntPtr]::Zero) { $p = Get-Process | Where-Object { $_.MainWindowTitle -and ($_.MainWindowTitle -eq ${quoted} -or $_.MainWindowTitle.Contains(${quoted})) } | Select-Object -First 1; if ($p) { $h = $p.MainWindowHandle } } if ($h -eq [IntPtr]::Zero) { Write-Output 'NOT_FOUND'; exit } [WinRect]::ShowWindow($h, 9) | Out-Null; [WinRect]::SetForegroundWindow($h) | Out-Null; Start-Sleep -Milliseconds 300; $r = New-Object WinRect+RECT; [WinRect]::GetWindowRect($h, [ref]$r) | Out-Null; Write-Output ("{0},{1},{2},{3}" -f $r.Left, $r.Top, $r.Right, $r.Bottom)`); const line = stdout.trim().split(/\r?\n/).pop() ?? ""; if (line === "NOT_FOUND") { throw new RuntimeError("not_found", `未找到窗口「${windowObj}」`, { recommendation: "确认窗口已打开,窗口对象可填完整标题、标题包含关键字或进程名" }); } const parts = line.split(",").map(Number); if (parts.length !== 4 || parts.some((v) => !Number.isFinite(v))) { throw new RuntimeError("desktop_error", "窗口矩形获取失败", { detail: line.slice(0, 120) }); } const [left, top, right, bottom] = parts; return { x: left, y: top, width: Math.max(1, right - left), height: Math.max(1, bottom - top) }; } /** 单通道噪声死区:JPG 压缩伪影/抗锯齿带来 ±10 级内的像素波动,不应计入差异 */ const NOISE_FLOOR = 10; function pixelDifference(screen: PNG, template: PNG, offsetX: number, offsetY: number, sampleStep: number, cutoff: number): number { let difference = 0; let samples = 0; for (let y = 0; y < template.height; y += sampleStep) { for (let x = 0; x < template.width; x += sampleStep) { const sourceIndex = ((offsetY + y) * screen.width + offsetX + x) * 4; const templateIndex = (y * template.width + x) * 4; for (let c = 0; c < 3; c++) { const d = Math.abs(screen.data[sourceIndex + c] - template.data[templateIndex + c]); difference += d <= NOISE_FLOOR ? 0 : d - NOISE_FLOOR; } samples += 3; // cutoff 是 0~1 的归一化差异上限; difference 是通道差累加值(0~255 每通道), // 提前退出必须按 255 折算——之前漏乘导致几乎所有位置在第一个像素就被剪掉, 得分全乱 if (difference > cutoff * samples * 255) return difference / (samples * 255); } } return samples ? difference / (samples * 255) : 1; } /** 在屏幕图里扫描模板的最佳位置(粗扫网格 + 精化) */ function scanBest(screen: PNG, template: PNG, maximumDifference: number): MatchResult | null { const sampleStep = Math.max(1, Math.floor(Math.min(template.width, template.height) / 18)); const scanStep = Math.max(1, Math.floor(Math.min(template.width, template.height) / 12)); let best: MatchResult | null = null; for (let y = 0; y <= screen.height - template.height; y += scanStep) { for (let x = 0; x <= screen.width - template.width; x += scanStep) { // 尚无 best 时不剪枝(完整评估): 若初始就按 maximumDifference 剪枝, // 均匀背景位置在少数采样后返回"刚好压线"的部分均值抢占 best, // 而真位置(常不在网格上)附近的网格点因部分重叠被剪, 精化永远够不到—— // 相似度 0.95 时实测 score=1.0 的真位置被判"未达到阈值" const cutoff = best ? 1 - best.score : 1; const difference = pixelDifference(screen, template, x, y, sampleStep, cutoff); const score = 1 - difference; if (!best || score > best.score) best = { x, y, width: template.width, height: template.height, score }; } } if (!best) return null; // 注意: 阈值判定必须放在精化之后——粗扫网格(步长=模板短边/12)通常够不到精确位置, // 粗扫 best 常差在阈值边缘, 精化后才能提到满分; 先判阈值会把真命中误杀 const refine = Math.max(1, scanStep); const minX = Math.max(0, best.x - refine); const maxX = Math.min(screen.width - template.width, best.x + refine); const minY = Math.max(0, best.y - refine); const maxY = Math.min(screen.height - template.height, best.y + refine); for (let y = minY; y <= maxY; y += 1) { for (let x = minX; x <= maxX; x += 1) { // 精化区域只有几百个位置, 完整评估避免同样的剪枝偏置 const difference = pixelDifference(screen, template, x, y, sampleStep, 1); const score = 1 - difference; if (score > best.score) best = { x, y, width: template.width, height: template.height, score }; } } // 阈值判定: 精化后的最终得分才与 maximumDifference 比较 if (1 - best.score > maximumDifference) return null; return best; } /** 用 canvas 双线性缩放模板(匹配尺度兜底用) */ async function resizeTemplate(template: PNG, scale: number): Promise { const { createCanvas } = await import("@napi-rs/canvas"); const width = Math.max(1, Math.round(template.width * scale)); const height = Math.max(1, Math.round(template.height * scale)); const source = createCanvas(template.width, template.height); const sourceContext = source.getContext("2d"); const imageData = sourceContext.createImageData(template.width, template.height); imageData.data.set(template.data); sourceContext.putImageData(imageData, 0, 0); const target = createCanvas(width, height); const targetContext = target.getContext("2d"); targetContext.imageSmoothingEnabled = true; targetContext.imageSmoothingQuality = "high"; targetContext.drawImage(source, 0, 0, width, height); const scaled = targetContext.getImageData(0, 0, width, height); const png = new PNG({ width, height }); png.data = Buffer.from(scaled.data); return png; } /** * 模板匹配。extraScales 提供尺度兜底:截图与模板的获取渠道分辨率不一致时 * (如旧版 DIP 截图存下的模板 vs 现在的物理分辨率截图),按给定缩放比重试。 */ export async function findImage(templatePath: string, screenshotPath: string, accuracy: number, extraScales: number[] = []): Promise { const [screenBuffer, templateBuffer] = await Promise.all([fs.readFile(screenshotPath), fs.readFile(templatePath)]); const screen = PNG.sync.read(screenBuffer); // 模板图可能是 JPG 等非 PNG 格式, 按魔数选择解码器 const template = isPngBuffer(templateBuffer) ? PNG.sync.read(templateBuffer) : await decodeImageFile(templatePath); if (template.width > screen.width || template.height > screen.height) { throw new RuntimeError("validation_error", `模板图片(${template.width}×${template.height})比截图画面(${screen.width}×${screen.height})还大,无法查找`, { recommendation: "模板应是屏幕上要寻找的小图案(按钮、图标、文字块),不要用整屏大图" }); } const maximumDifference = Math.max(0.01, Math.min(0.95, accuracy || 0.2)); const best = scanBest(screen, template, maximumDifference); if (best) return best; for (const scale of extraScales) { if (!Number.isFinite(scale) || scale === 1) continue; const scaled = await resizeTemplate(template, scale); if (scaled.width > screen.width || scaled.height > screen.height) continue; const scaledBest = scanBest(screen, scaled, maximumDifference); if (scaledBest) return scaledBest; } return null; } async function locate(args: Record, evidenceDir: string, nodeId: string): Promise { const rawTemplate = text(args, ["image", "image_path", "template"]); if (!rawTemplate.trim()) throw new RuntimeError("validation_error", "模板图片路径为空", { nodeId }); const template = path.resolve(rawTemplate); // 查找范围:1 整个虚拟桌面(默认,含副屏)/ 2 指定窗口 / 3 指定区域; // 截图统一为物理分辨率拼接画布, 结果坐标加回画布原点即屏幕物理坐标(可直接用于点击) const rangeMode = text(args, ["find_range"], "1"); const screenshot = path.join(evidenceDir, `${nodeId}_screen.png`); const shot = await captureVirtualScreenPng(); let offsetX = 0; let offsetY = 0; if (rangeMode === "2") { const windowObj = text(args, ["window_obj"]); if (!windowObj.trim()) throw new RuntimeError("validation_error", "查找范围为「指定窗口」时,窗口对象不能为空", { nodeId, recommendation: "填写窗口标题(或标题关键字、进程名)" }); const rect = await focusWindowRect(windowObj); const cropRect = clampRegion({ x: rect.x - shot.originX, y: rect.y - shot.originY, width: rect.width, height: rect.height }, shot.png); await fs.writeFile(screenshot, PNG.sync.write(cropPng(shot.png, cropRect))); offsetX = rect.x; offsetY = rect.y; } else if (rangeMode === "3" || rangeMode === "4") { const regionRaw = text(args, ["shot_position", "action", "position"]); if (!regionRaw.trim()) throw new RuntimeError("validation_error", "查找范围为「指定区域」时,需要填写区域 JSON", { nodeId, recommendation: '在 shot_position 填写 {"x":0,"y":0,"width":100,"height":80} 或 [x, y, width, height]' }); // 区域 JSON 是 DIP 坐标(框选工具按 DIP 返回, 支持副屏负坐标),换算物理坐标后减画布原点裁剪 const phys = await dipRegionToPhysical(parseRegion(regionRaw, shot.png)); const cropRect = clampRegion({ x: phys.x - shot.originX, y: phys.y - shot.originY, width: phys.width, height: phys.height }, shot.png); await fs.writeFile(screenshot, PNG.sync.write(cropPng(shot.png, cropRect))); offsetX = phys.x; offsetY = phys.y; } else { await fs.writeFile(screenshot, PNG.sync.write(shot.png)); // 拼接画布坐标 → 屏幕物理坐标必须加回画布原点(副屏在主屏左侧/上方时原点为负); // 漏加会导致点击偏移(实测副屏上移 177px 时点击正好偏下 177px) offsetX = shot.originX; offsetY = shot.originY; } // accuracy 字段语义为「相似度」(0~1, 越大越严格, 0.9=至少90%像素一致)——与界面文案一致。 // 引擎内部用最大允许差异 = 1 - 相似度。此前引擎直接把该值当差异阈值, // 用户填 0.9 反而放宽到"接受90%像素差异", 导致到处误命中。 const rawAccuracy = Number(text(args, ["accuracy"], "0.8")); const similarity = Math.max(0.05, Math.min(1, Number.isFinite(rawAccuracy) && rawAccuracy > 0 ? rawAccuracy : 0.8)); // 原生 1:1 匹配失败时, 按缩放比对模板放大/缩小重试——兼容旧版 DIP 截图存的模板与任意来源的图片 const scale = requireScreenService().primaryDisplay().scaleFactor || 1; const result = await findImage(template, screenshot, 1 - similarity, scale !== 1 ? [scale, 1 / scale] : []); if (!result) throw new RuntimeError("not_found", `屏幕中未找到目标图像(要求相似度 ${(similarity * 100).toFixed(0)}%,未达到阈值)`, { nodeId, recommendation: "重新框选模板(框小一点、只框目标图案),或适当调低相似度(如 0.8);目标在副屏时确认其清晰可见" }); return { ...result, x: result.x + offsetX, y: result.y + offsetY }; } register(["Image_Find", "Image_Find_V1"], async (ctx, node, args) => locate(args, ctx.evidenceDir, node.id)); // 图像位置: 与查找图像同义(历史重复组件), 同管线返回坐标 for (const id of ["Image_Location", "Image_Location_V1"]) register(id, async (ctx, node, args) => locate(args, ctx.evidenceDir, node.id)); // 判断图像: 找到返回 true / 找不到返回 false(供条件分支), 仅有 not_found 转为 false; // 模板路径为空/文件不存在等配置错误仍然抛出, 避免静默 false 难排查。 // timeout>0 时在窗口内每 300ms 重试(等待图像出现再判断的场景) for (const id of ["Image_Exists", "Image_Exists_V1"]) register(id, async (ctx, node, args) => { const rawTimeout = Number(text(args, ["timeout", "time_out"], "0")); const deadline = Date.now() + (Number.isFinite(rawTimeout) && rawTimeout > 0 ? rawTimeout : 0) * 1000; for (;;) { try { await locate(args, ctx.evidenceDir, node.id); return true; } catch (error) { if (error instanceof RuntimeError && error.code === "not_found") { if (Date.now() >= deadline) return false; } else { throw error; } } await new Promise(resolve => setTimeout(resolve, 300)); } }); // 等待图像: 在超时窗口内每 300ms 重试一次查找, 命中即返回坐标, 超时仍无则抛 not_found for (const id of ["Image_Wait", "Image_Wait_V1"]) register(id, async (ctx, node, args) => { const rawTimeout = Number(text(args, ["timeout", "time_out"], "20")); const timeout = Number.isFinite(rawTimeout) && rawTimeout >= 0 ? rawTimeout : 20; const deadline = Date.now() + timeout * 1000; for (;;) { try { return await locate(args, ctx.evidenceDir, node.id); } catch (error) { if (!(error instanceof RuntimeError) || error.code !== "not_found" || Date.now() >= deadline) throw error; } await new Promise(resolve => setTimeout(resolve, 300)); } }); // 移动鼠标到图像: 定位后把光标移到匹配区域中心; // SetCursorPos 传物理坐标, 脚本必须声明 per-monitor DPI 感知(与 injectClick 同理), 否则被系统虚拟化导致偏移 for (const id of ["Image_Move", "Image_Move_V1", "Image_Move_V2"]) register(id, async (ctx, node, args) => { const hit = await locate(args, ctx.evidenceDir, node.id); const point = clickPoint(hit, text(args, ["position"], "center")); const x = point.x; const y = point.y; await powershell("Add-Type -MemberDefinition '[DllImport(\"user32.dll\")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value); [DllImport(\"user32.dll\")] public static extern bool SetCursorPos(int X, int Y);' -Name RpaImageMoveInput -Namespace RpaImageMove | Out-Null; try { [void][RpaImageMove.RpaImageMoveInput]::SetProcessDpiAwarenessContext([IntPtr](-4)) } catch {}; [RpaImageMove.RpaImageMoveInput]::SetCursorPos(" + x + "," + y + ")"); return { ...hit, x, y }; }); /** 点击位置锚点(相对匹配区域的比例坐标),支持九宫格取值 */ const POSITION_ANCHORS: Record = { center: [0.5, 0.5], leftTop: [0, 0], top: [0.5, 0], centerTop: [0.5, 0], rightTop: [1, 0], left: [0, 0.5], centerLeft: [0, 0.5], right: [1, 0.5], centerRight: [1, 0.5], leftBottom: [0, 1], bottom: [0.5, 1], centerBottom: [0.5, 1], rightBottom: [1, 1] }; export function clickPoint(result: MatchResult, position: string): { x: number; y: number } { const raw = position.trim(); if (raw && raw.toLowerCase() !== "center") { if (raw.startsWith("{")) { try { const offset = JSON.parse(raw) as { x?: unknown; y?: unknown }; const dx = Number(offset.x); const dy = Number(offset.y); if (Number.isFinite(dx) && Number.isFinite(dy)) { return { x: Math.round(result.x + dx), y: Math.round(result.y + dy) }; } } catch { // 非法 JSON 落回中心 } } const anchor = POSITION_ANCHORS[raw] ?? POSITION_ANCHORS[raw.toLowerCase()] ?? POSITION_ANCHORS[raw.replace(/-([a-z])/g, (_: string, char: string) => char.toUpperCase())]; if (anchor) { return { x: Math.round(result.x + result.width * anchor[0]), y: Math.round(result.y + result.height * anchor[1]) }; } } return { x: Math.round(result.x + result.width / 2), y: Math.round(result.y + result.height / 2) }; } const CLICK_ACTIONS: Record = { left: "[Mouse]::mouse_event(2,0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 30; [Mouse]::mouse_event(4,0,0,0,[UIntPtr]::Zero)", right: "[Mouse]::mouse_event(8,0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 30; [Mouse]::mouse_event(16,0,0,0,[UIntPtr]::Zero)", middle: "[Mouse]::mouse_event(32,0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 30; [Mouse]::mouse_event(64,0,0,0,[UIntPtr]::Zero)", double: "[Mouse]::mouse_event(2,0,0,0,[UIntPtr]::Zero); [Mouse]::mouse_event(4,0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 60; [Mouse]::mouse_event(2,0,0,0,[UIntPtr]::Zero); [Mouse]::mouse_event(4,0,0,0,[UIntPtr]::Zero)" }; async function injectClick(x: number, y: number, clickKey: string, showTrack: boolean): Promise { const action = CLICK_ACTIONS[clickKey] ?? CLICK_ACTIONS.left; // show_track=1 时从当前光标位置平滑移动过去(20 步插值),否则直接跳转 const move = showTrack ? `for ($i = 1; $i -le 20; $i++) { [Mouse]::SetCursorPos([int]($p.X + (${x} - $p.X) * $i / 20), [int]($p.Y + (${y} - $p.Y) * $i / 20)); Start-Sleep -Milliseconds 12 }` : `[Mouse]::SetCursorPos(${x}, ${y})`; await powershell(`Add-Type @' using System; using System.Runtime.InteropServices; public static class Mouse { [DllImport("user32.dll")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value); [DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y); [DllImport("user32.dll")] public static extern bool GetCursorPos(out POINT p); [DllImport("user32.dll")] public static extern void mouse_event(uint f, uint x, uint y, uint d, UIntPtr e); public struct POINT { public int X; public int Y; } } '@; # 传入的是物理像素坐标(与物理分辨率截图匹配所得), 必须声明 DPI 感知—— # 否则进程被 DPI 虚拟化, 坐标再被系统放大一次, 点击成比例偏右下 try { [void][Mouse]::SetProcessDpiAwarenessContext([IntPtr](-4)) } catch {}; $p = New-Object Mouse+POINT; [Mouse]::GetCursorPos([ref]$p) | Out-Null; ${move}; Start-Sleep -Milliseconds 60; ${action}`); } register(["Image_Click", "Image_Click_V1", "Image_Click_V2", "Image_Click_V3"], async (ctx, node, args) => { const result = await locate(args, ctx.evidenceDir, node.id); const point = clickPoint(result, text(args, ["position"], "center")); const clickKey = text(args, ["click_key"], "left"); const showTrack = String(args.show_track ?? "0") === "1" || args.show_track === true; await injectClick(point.x, point.y, clickKey, showTrack); ctx.log(`已${clickKey === "right" ? "右键" : clickKey === "double" ? "双击" : "点击"}图像 (${point.x}, ${point.y}),匹配得分 ${result.score.toFixed(3)}`, "info", node.id); return { ...result, click: point }; });