import path from "node:path"; import { promises as fs } from "node:fs"; import { PNG } from "pngjs"; import { register } from "../engine/registry"; import { RuntimeError } from "../engine/types"; import { powershell, psString } from "./desktop"; function text(args: Record, keys: string[], fallback = ""): string { for (const key of keys) if (args[key] !== undefined && args[key] !== null && args[key] !== "") return String(args[key]); return fallback; } async function capturePrimaryScreenPng(): Promise { const { desktopCapturer, screen } = await import("electron"); const primary = screen.getPrimaryDisplay(); const sources = await desktopCapturer.getSources({ types: ["screen"], thumbnailSize: { width: primary.size.width, height: primary.size.height } }); const source = sources[0]; if (!source) throw new RuntimeError("desktop_error", "无法获取屏幕截图"); return PNG.sync.read(source.thumbnail.toPNG()); } /** 解析区域 JSON:对象 {x,y,width,height}(兼容 left/top、x1y1x2y2 角点)或数组 [x,y,width,height],并夹取到图片边界内 */ function parseRegion(raw: string, image: PNG): { x: number; y: number; width: number; height: number } { let parsed: unknown; try { parsed = JSON.parse(raw); } catch { throw new RuntimeError("validation_error", "区域 JSON 无法解析", { detail: raw.slice(0, 200) }); } const numberOf = (value: unknown): number | null => { const n = Number(value); return Number.isFinite(n) ? n : null; }; if (Array.isArray(parsed) && parsed.length === 4) { const [x, y, width, height] = parsed.map(numberOf); if (x !== null && y !== null && width !== null && height !== null) { return clampRegion({ x, y, width, height }, image); } } if (parsed && typeof parsed === "object") { const record = parsed as Record; const x = numberOf(record.x ?? record.left); const y = numberOf(record.y ?? record.top); const width = numberOf(record.width); const height = numberOf(record.height); if (x !== null && y !== null && width !== null && height !== null) { return clampRegion({ x, y, width, height }, image); } const x2 = numberOf(record.x2 ?? record.right); const y2 = numberOf(record.y2 ?? record.bottom); if (x !== null && y !== null && x2 !== null && y2 !== null) { return clampRegion({ x, y, width: x2 - x, height: y2 - y }, image); } } throw new RuntimeError("validation_error", "区域 JSON 不符合要求", { detail: raw.slice(0, 200), recommendation: '使用 {"x":0,"y":0,"width":100,"height":80} 或 [x, y, width, height]' }); } function clampRegion(region: { x: number; y: number; width: number; height: number }, image: PNG): { x: number; y: number; width: number; height: number } { const x = Math.max(0, Math.min(Math.round(region.x), image.width - 1)); const y = Math.max(0, Math.min(Math.round(region.y), image.height - 1)); const width = Math.max(1, Math.min(Math.round(region.width), image.width - x)); const height = Math.max(1, Math.min(Math.round(region.height), image.height - y)); return { x, y, width, height }; } function cropPng(source: PNG, region: { x: number; y: number; width: number; height: number }): PNG { const cropped = new PNG({ width: region.width, height: region.height }); PNG.bitblt(source, cropped, region.x, region.y, region.width, region.height, 0, 0); return cropped; } register("Image_Screenshot", async (_ctx, node, args) => { const folder = text(args, ["outImagePath", "folder"]); const name = text(args, ["outImageName", "name"], `screenshot_${Date.now()}.png`); const imageArg = text(args, ["image_path"]); const source = text(args, ["pictureSource"], "1"); const range = text(args, ["shotRange"], "1"); let image: PNG; let output: string; if (source === "2") { // 指定图片文件来源:image_path 为输入图片;输出走 输出目录+文件名,两者均未配置时直接返回源文件 if (!imageArg.trim()) { throw new RuntimeError("validation_error", "图像来源为「指定图片文件」时,目标文件路径需填写源图片路径", { recommendation: "在目标文件路径选择一张已存在的图片,或将图像来源改回「当前屏幕」" }); } const sourceFile = path.resolve(imageArg); image = PNG.sync.read(await fs.readFile(sourceFile)); output = folder ? path.resolve(path.join(folder, name)) : sourceFile; } else { // 当前屏幕来源:image_path 即保存路径,缺省用 输出目录+文件名 image = await capturePrimaryScreenPng(); output = path.resolve(imageArg || (folder ? path.join(folder, name) : name)); } if (range === "2") { const regionRaw = text(args, ["position", "action", "position_img"]); if (!regionRaw.trim()) { throw new RuntimeError("validation_error", "截图范围为「框选区域」时,position 需填写区域 JSON", { recommendation: '在 position 填写 {"x":0,"y":0,"width":100,"height":80} 或 [x, y, width, height],或将截图范围改回「整个屏幕」' }); } image = cropPng(image, parseRegion(regionRaw, image)); } await fs.mkdir(path.dirname(output), { recursive: true }); await fs.writeFile(output, PNG.sync.write(image)); return output; }); interface MatchResult { x: number; y: number; width: number; height: number; score: number; } async function captureScreen(target: string): Promise { await fs.mkdir(path.dirname(target), { recursive: true }); await powershell(`Add-Type -AssemblyName System.Drawing; Add-Type -AssemblyName System.Windows.Forms; $bounds = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds; $bitmap = New-Object System.Drawing.Bitmap $bounds.Width,$bounds.Height; $graphics = [System.Drawing.Graphics]::FromImage($bitmap); $graphics.CopyFromScreen($bounds.Location,[System.Drawing.Point]::Empty,$bounds.Size); $bitmap.Save(${psString(target)},[System.Drawing.Imaging.ImageFormat]::Png); $graphics.Dispose(); $bitmap.Dispose()`); return target; } function pixelDifference(screen: PNG, template: PNG, offsetX: number, offsetY: number, sampleStep: number, cutoff: number): number { let difference = 0; let samples = 0; for (let y = 0; y < template.height; y += sampleStep) { for (let x = 0; x < template.width; x += sampleStep) { const sourceIndex = ((offsetY + y) * screen.width + offsetX + x) * 4; const templateIndex = (y * template.width + x) * 4; difference += Math.abs(screen.data[sourceIndex] - template.data[templateIndex]); difference += Math.abs(screen.data[sourceIndex + 1] - template.data[templateIndex + 1]); difference += Math.abs(screen.data[sourceIndex + 2] - template.data[templateIndex + 2]); samples += 3; if (difference > cutoff * samples) return difference / (samples * 255); } } return samples ? difference / (samples * 255) : 1; } export async function findImage(templatePath: string, screenshotPath: string, accuracy: number): Promise { const [screenBuffer, templateBuffer] = await Promise.all([fs.readFile(screenshotPath), fs.readFile(templatePath)]); const screen = PNG.sync.read(screenBuffer); const template = PNG.sync.read(templateBuffer); if (template.width > screen.width || template.height > screen.height) return null; const maximumDifference = Math.max(0.01, Math.min(0.95, accuracy || 0.2)); const sampleStep = Math.max(1, Math.floor(Math.min(template.width, template.height) / 18)); const scanStep = Math.max(1, Math.floor(Math.min(template.width, template.height) / 12)); let best: MatchResult | null = null; for (let y = 0; y <= screen.height - template.height; y += scanStep) { for (let x = 0; x <= screen.width - template.width; x += scanStep) { const difference = pixelDifference(screen, template, x, y, sampleStep, best ? 1 - best.score : maximumDifference); const score = 1 - difference; if (!best || score > best.score) best = { x, y, width: template.width, height: template.height, score }; } } if (!best || 1 - best.score > maximumDifference) return null; const refine = Math.max(1, scanStep); const minX = Math.max(0, best.x - refine); const maxX = Math.min(screen.width - template.width, best.x + refine); const minY = Math.max(0, best.y - refine); const maxY = Math.min(screen.height - template.height, best.y + refine); for (let y = minY; y <= maxY; y += 1) { for (let x = minX; x <= maxX; x += 1) { const difference = pixelDifference(screen, template, x, y, sampleStep, 1 - best.score); const score = 1 - difference; if (score > best.score) best = { x, y, width: template.width, height: template.height, score }; } } return best; } async function locate(args: Record, evidenceDir: string, nodeId: string): Promise { const rawTemplate = text(args, ["image", "image_path", "template"]); if (!rawTemplate.trim()) throw new RuntimeError("validation_error", "模板图片路径为空", { nodeId }); const template = path.resolve(rawTemplate); const screenshot = path.join(evidenceDir, `${nodeId}_screen.png`); await captureScreen(screenshot); const result = await findImage(template, screenshot, Number(text(args, ["accuracy"], "0.2"))); if (!result) throw new RuntimeError("not_found", "屏幕中未找到目标图像", { nodeId, recommendation: "重新截取模板,或调低匹配精度" }); return result; } register(["Image_Find", "Image_Find_V1"], async (ctx, node, args) => locate(args, ctx.evidenceDir, node.id)); register(["Image_Click", "Image_Click_V1", "Image_Click_V2", "Image_Click_V3"], async (ctx, node, args) => { const result = await locate(args, ctx.evidenceDir, node.id); const x = Math.round(result.x + result.width / 2); const y = Math.round(result.y + result.height / 2); await powershell(`Add-Type @'\nusing System; using System.Runtime.InteropServices; public static class Clicker { [DllImport("user32.dll")] public static extern bool SetCursorPos(int X,int Y); [DllImport("user32.dll")] public static extern void mouse_event(uint f,uint x,uint y,uint d,UIntPtr e); }\n'@; [Clicker]::SetCursorPos(${x},${y}); [Clicker]::mouse_event(2,0,0,0,[UIntPtr]::Zero); [Clicker]::mouse_event(4,0,0,0,[UIntPtr]::Zero)`); return { ...result, click: { x, y } }; });