342145cc46
- vision.ts 重写 Image_Screenshot：pictureSource=1 当前屏幕(desktopCapturer)/2 指定图片文件(image_path 作输入, 输出走输出目录+文件名, 未配置则原样返回)；shotRange=2 框选区域按 position/action 区域 JSON 裁剪(pngjs bitblt)，区域支持 {x,y,width,height}/[x,y,w,h]/left,top/角点写法并夹取边界
- 移除 browser.ts 重复注册，实现单点维护
- catalog: pictureSource/shotRange 补下拉选项(当前屏幕/指定图片文件, 整个屏幕/框选区域)
- typecheck 通过；browser/cv-vision/capability-audit/app-system 回归 50 项全过
194 lines
10 KiB
TypeScript
194 lines
10 KiB
TypeScript
import path from "node:path";
|
||
import { promises as fs } from "node:fs";
|
||
import { PNG } from "pngjs";
|
||
import { register } from "../engine/registry";
|
||
import { RuntimeError } from "../engine/types";
|
||
import { powershell, psString } from "./desktop";
|
||
|
||
function text(args: Record<string, unknown>, keys: string[], fallback = ""): string {
|
||
for (const key of keys) if (args[key] !== undefined && args[key] !== null && args[key] !== "") return String(args[key]);
|
||
return fallback;
|
||
}
|
||
|
||
async function capturePrimaryScreenPng(): Promise<PNG> {
|
||
const { desktopCapturer, screen } = await import("electron");
|
||
const primary = screen.getPrimaryDisplay();
|
||
const sources = await desktopCapturer.getSources({
|
||
types: ["screen"],
|
||
thumbnailSize: { width: primary.size.width, height: primary.size.height }
|
||
});
|
||
const source = sources[0];
|
||
if (!source) throw new RuntimeError("desktop_error", "无法获取屏幕截图");
|
||
return PNG.sync.read(source.thumbnail.toPNG());
|
||
}
|
||
|
||
/** 解析区域 JSON：对象 {x,y,width,height}（兼容 left/top、x1y1x2y2 角点）或数组 [x,y,width,height]，并夹取到图片边界内 */
|
||
function parseRegion(raw: string, image: PNG): { x: number; y: number; width: number; height: number } {
|
||
let parsed: unknown;
|
||
try {
|
||
parsed = JSON.parse(raw);
|
||
} catch {
|
||
throw new RuntimeError("validation_error", "区域 JSON 无法解析", { detail: raw.slice(0, 200) });
|
||
}
|
||
const numberOf = (value: unknown): number | null => {
|
||
const n = Number(value);
|
||
return Number.isFinite(n) ? n : null;
|
||
};
|
||
if (Array.isArray(parsed) && parsed.length === 4) {
|
||
const [x, y, width, height] = parsed.map(numberOf);
|
||
if (x !== null && y !== null && width !== null && height !== null) {
|
||
return clampRegion({ x, y, width, height }, image);
|
||
}
|
||
}
|
||
if (parsed && typeof parsed === "object") {
|
||
const record = parsed as Record<string, unknown>;
|
||
const x = numberOf(record.x ?? record.left);
|
||
const y = numberOf(record.y ?? record.top);
|
||
const width = numberOf(record.width);
|
||
const height = numberOf(record.height);
|
||
if (x !== null && y !== null && width !== null && height !== null) {
|
||
return clampRegion({ x, y, width, height }, image);
|
||
}
|
||
const x2 = numberOf(record.x2 ?? record.right);
|
||
const y2 = numberOf(record.y2 ?? record.bottom);
|
||
if (x !== null && y !== null && x2 !== null && y2 !== null) {
|
||
return clampRegion({ x, y, width: x2 - x, height: y2 - y }, image);
|
||
}
|
||
}
|
||
throw new RuntimeError("validation_error", "区域 JSON 不符合要求", {
|
||
detail: raw.slice(0, 200),
|
||
recommendation: '使用 {"x":0,"y":0,"width":100,"height":80} 或 [x, y, width, height]'
|
||
});
|
||
}
|
||
|
||
function clampRegion(region: { x: number; y: number; width: number; height: number }, image: PNG): { x: number; y: number; width: number; height: number } {
|
||
const x = Math.max(0, Math.min(Math.round(region.x), image.width - 1));
|
||
const y = Math.max(0, Math.min(Math.round(region.y), image.height - 1));
|
||
const width = Math.max(1, Math.min(Math.round(region.width), image.width - x));
|
||
const height = Math.max(1, Math.min(Math.round(region.height), image.height - y));
|
||
return { x, y, width, height };
|
||
}
|
||
|
||
function cropPng(source: PNG, region: { x: number; y: number; width: number; height: number }): PNG {
|
||
const cropped = new PNG({ width: region.width, height: region.height });
|
||
PNG.bitblt(source, cropped, region.x, region.y, region.width, region.height, 0, 0);
|
||
return cropped;
|
||
}
|
||
|
||
register("Image_Screenshot", async (_ctx, node, args) => {
|
||
const folder = text(args, ["outImagePath", "folder"]);
|
||
const name = text(args, ["outImageName", "name"], `screenshot_${Date.now()}.png`);
|
||
const imageArg = text(args, ["image_path"]);
|
||
const source = text(args, ["pictureSource"], "1");
|
||
const range = text(args, ["shotRange"], "1");
|
||
|
||
let image: PNG;
|
||
let output: string;
|
||
if (source === "2") {
|
||
// 指定图片文件来源：image_path 为输入图片；输出走 输出目录+文件名，两者均未配置时直接返回源文件
|
||
if (!imageArg.trim()) {
|
||
throw new RuntimeError("validation_error", "图像来源为「指定图片文件」时，目标文件路径需填写源图片路径", {
|
||
recommendation: "在目标文件路径选择一张已存在的图片，或将图像来源改回「当前屏幕」"
|
||
});
|
||
}
|
||
const sourceFile = path.resolve(imageArg);
|
||
image = PNG.sync.read(await fs.readFile(sourceFile));
|
||
output = folder ? path.resolve(path.join(folder, name)) : sourceFile;
|
||
} else {
|
||
// 当前屏幕来源：image_path 即保存路径，缺省用 输出目录+文件名
|
||
image = await capturePrimaryScreenPng();
|
||
output = path.resolve(imageArg || (folder ? path.join(folder, name) : name));
|
||
}
|
||
|
||
if (range === "2") {
|
||
const regionRaw = text(args, ["position", "action", "position_img"]);
|
||
if (!regionRaw.trim()) {
|
||
throw new RuntimeError("validation_error", "截图范围为「框选区域」时，position 需填写区域 JSON", {
|
||
recommendation: '在 position 填写 {"x":0,"y":0,"width":100,"height":80} 或 [x, y, width, height]，或将截图范围改回「整个屏幕」'
|
||
});
|
||
}
|
||
image = cropPng(image, parseRegion(regionRaw, image));
|
||
}
|
||
|
||
await fs.mkdir(path.dirname(output), { recursive: true });
|
||
await fs.writeFile(output, PNG.sync.write(image));
|
||
return output;
|
||
});
|
||
|
||
interface MatchResult { x: number; y: number; width: number; height: number; score: number; }
|
||
|
||
async function captureScreen(target: string): Promise<string> {
|
||
await fs.mkdir(path.dirname(target), { recursive: true });
|
||
await powershell(`Add-Type -AssemblyName System.Drawing; Add-Type -AssemblyName System.Windows.Forms; $bounds = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds; $bitmap = New-Object System.Drawing.Bitmap $bounds.Width,$bounds.Height; $graphics = [System.Drawing.Graphics]::FromImage($bitmap); $graphics.CopyFromScreen($bounds.Location,[System.Drawing.Point]::Empty,$bounds.Size); $bitmap.Save(${psString(target)},[System.Drawing.Imaging.ImageFormat]::Png); $graphics.Dispose(); $bitmap.Dispose()`);
|
||
return target;
|
||
}
|
||
|
||
function pixelDifference(screen: PNG, template: PNG, offsetX: number, offsetY: number, sampleStep: number, cutoff: number): number {
|
||
let difference = 0;
|
||
let samples = 0;
|
||
for (let y = 0; y < template.height; y += sampleStep) {
|
||
for (let x = 0; x < template.width; x += sampleStep) {
|
||
const sourceIndex = ((offsetY + y) * screen.width + offsetX + x) * 4;
|
||
const templateIndex = (y * template.width + x) * 4;
|
||
difference += Math.abs(screen.data[sourceIndex] - template.data[templateIndex]);
|
||
difference += Math.abs(screen.data[sourceIndex + 1] - template.data[templateIndex + 1]);
|
||
difference += Math.abs(screen.data[sourceIndex + 2] - template.data[templateIndex + 2]);
|
||
samples += 3;
|
||
if (difference > cutoff * samples) return difference / (samples * 255);
|
||
}
|
||
}
|
||
return samples ? difference / (samples * 255) : 1;
|
||
}
|
||
|
||
export async function findImage(templatePath: string, screenshotPath: string, accuracy: number): Promise<MatchResult | null> {
|
||
const [screenBuffer, templateBuffer] = await Promise.all([fs.readFile(screenshotPath), fs.readFile(templatePath)]);
|
||
const screen = PNG.sync.read(screenBuffer);
|
||
const template = PNG.sync.read(templateBuffer);
|
||
if (template.width > screen.width || template.height > screen.height) return null;
|
||
const maximumDifference = Math.max(0.01, Math.min(0.95, accuracy || 0.2));
|
||
const sampleStep = Math.max(1, Math.floor(Math.min(template.width, template.height) / 18));
|
||
const scanStep = Math.max(1, Math.floor(Math.min(template.width, template.height) / 12));
|
||
let best: MatchResult | null = null;
|
||
for (let y = 0; y <= screen.height - template.height; y += scanStep) {
|
||
for (let x = 0; x <= screen.width - template.width; x += scanStep) {
|
||
const difference = pixelDifference(screen, template, x, y, sampleStep, best ? 1 - best.score : maximumDifference);
|
||
const score = 1 - difference;
|
||
if (!best || score > best.score) best = { x, y, width: template.width, height: template.height, score };
|
||
}
|
||
}
|
||
if (!best || 1 - best.score > maximumDifference) return null;
|
||
const refine = Math.max(1, scanStep);
|
||
const minX = Math.max(0, best.x - refine);
|
||
const maxX = Math.min(screen.width - template.width, best.x + refine);
|
||
const minY = Math.max(0, best.y - refine);
|
||
const maxY = Math.min(screen.height - template.height, best.y + refine);
|
||
for (let y = minY; y <= maxY; y += 1) {
|
||
for (let x = minX; x <= maxX; x += 1) {
|
||
const difference = pixelDifference(screen, template, x, y, sampleStep, 1 - best.score);
|
||
const score = 1 - difference;
|
||
if (score > best.score) best = { x, y, width: template.width, height: template.height, score };
|
||
}
|
||
}
|
||
return best;
|
||
}
|
||
|
||
async function locate(args: Record<string, unknown>, evidenceDir: string, nodeId: string): Promise<MatchResult> {
|
||
const rawTemplate = text(args, ["image", "image_path", "template"]);
|
||
if (!rawTemplate.trim()) throw new RuntimeError("validation_error", "模板图片路径为空", { nodeId });
|
||
const template = path.resolve(rawTemplate);
|
||
const screenshot = path.join(evidenceDir, `${nodeId}_screen.png`);
|
||
await captureScreen(screenshot);
|
||
const result = await findImage(template, screenshot, Number(text(args, ["accuracy"], "0.2")));
|
||
if (!result) throw new RuntimeError("not_found", "屏幕中未找到目标图像", { nodeId, recommendation: "重新截取模板，或调低匹配精度" });
|
||
return result;
|
||
}
|
||
|
||
register(["Image_Find", "Image_Find_V1"], async (ctx, node, args) => locate(args, ctx.evidenceDir, node.id));
|
||
register(["Image_Click", "Image_Click_V1", "Image_Click_V2", "Image_Click_V3"], async (ctx, node, args) => {
|
||
const result = await locate(args, ctx.evidenceDir, node.id);
|
||
const x = Math.round(result.x + result.width / 2);
|
||
const y = Math.round(result.y + result.height / 2);
|
||
await powershell(`Add-Type @'\nusing System; using System.Runtime.InteropServices; public static class Clicker { [DllImport("user32.dll")] public static extern bool SetCursorPos(int X,int Y); [DllImport("user32.dll")] public static extern void mouse_event(uint f,uint x,uint y,uint d,UIntPtr e); }\n'@; [Clicker]::SetCursorPos(${x},${y}); [Clicker]::mouse_event(2,0,0,0,[UIntPtr]::Zero); [Clicker]::mouse_event(4,0,0,0,[UIntPtr]::Zero)`);
|
||
return { ...result, click: { x, y } };
|
||
});
|