feat: 屏幕截图组件实现图像来源与截图范围（原 P2 问题）
- vision.ts 重写 Image_Screenshot：pictureSource=1 当前屏幕(desktopCapturer)/2 指定图片文件(image_path 作输入, 输出走输出目录+文件名, 未配置则原样返回)；shotRange=2 框选区域按 position/action 区域 JSON 裁剪(pngjs bitblt)，区域支持 {x,y,width,height}/[x,y,w,h]/left,top/角点写法并夹取边界
- 移除 browser.ts 重复注册，实现单点维护
- catalog: pictureSource/shotRange 补下拉选项(当前屏幕/指定图片文件, 整个屏幕/框选区域)
- typecheck 通过；browser/cv-vision/capability-audit/app-system 回归 50 项全过
This commit is contained in:
yu
parent
c11e1d1527
commit
342145cc46
@ -61865,7 +61865,17 @@
|
||||
"placeholder": "",
|
||||
"description": "选择要截图的图像来源，可以是当前屏幕，也可以是指定的图片文件。",
|
||||
"advanced": false,
|
||||
"valueTypes": []
|
||||
"valueTypes": [],
|
||||
"options": [
|
||||
{
|
||||
"label": "当前屏幕",
|
||||
"value": "1"
|
||||
},
|
||||
{
|
||||
"label": "指定图片文件",
|
||||
"value": "2"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "shotRange",
|
||||
@ -61878,7 +61888,17 @@
|
||||
"placeholder": "",
|
||||
"description": "选择要截图的范围，可以使截取整个屏幕，或手动选择截图区域",
|
||||
"advanced": false,
|
||||
"valueTypes": []
|
||||
"valueTypes": [],
|
||||
"options": [
|
||||
{
|
||||
"label": "整个屏幕",
|
||||
"value": "1"
|
||||
},
|
||||
{
|
||||
"label": "框选区域",
|
||||
"value": "2"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "image_path",
|
||||
|
||||
@ -178,21 +178,4 @@ register("WebBrowser_Screenshot", async (ctx, node, args) => {
|
||||
return file;
|
||||
});
|
||||
|
||||
register("Image_Screenshot", async (ctx, node, args) => {
|
||||
const { desktopCapturer, screen } = await import("electron");
|
||||
const folder = String(args.outImagePath ?? args.image_path ?? ctx.evidenceDir);
|
||||
const name = String(args.outImageName ?? `${node.id}.png`);
|
||||
const { promises: fs } = await import("node:fs");
|
||||
const path = await import("node:path");
|
||||
const target = path.resolve(folder.endsWith(".png") ? folder : path.join(folder, name));
|
||||
const primary = screen.getPrimaryDisplay();
|
||||
const sources = await desktopCapturer.getSources({
|
||||
types: ["screen"],
|
||||
thumbnailSize: { width: primary.size.width, height: primary.size.height }
|
||||
});
|
||||
const source = sources[0];
|
||||
if (!source) throw new RuntimeError("desktop_error", "无法获取屏幕截图");
|
||||
await fs.mkdir(path.dirname(target), { recursive: true });
|
||||
await fs.writeFile(target, source.thumbnail.toPNG());
|
||||
return target;
|
||||
});
|
||||
// Image_Screenshot 统一在 vision.ts 实现（屏幕/文件来源 + 区域裁剪），此处不再重复注册
|
||||
|
||||
@ -10,12 +10,108 @@ function text(args: Record<string, unknown>, keys: string[], fallback = ""): str
|
||||
return fallback;
|
||||
}
|
||||
|
||||
async function capturePrimaryScreenPng(): Promise<PNG> {
|
||||
const { desktopCapturer, screen } = await import("electron");
|
||||
const primary = screen.getPrimaryDisplay();
|
||||
const sources = await desktopCapturer.getSources({
|
||||
types: ["screen"],
|
||||
thumbnailSize: { width: primary.size.width, height: primary.size.height }
|
||||
});
|
||||
const source = sources[0];
|
||||
if (!source) throw new RuntimeError("desktop_error", "无法获取屏幕截图");
|
||||
return PNG.sync.read(source.thumbnail.toPNG());
|
||||
}
|
||||
|
||||
/** 解析区域 JSON：对象 {x,y,width,height}（兼容 left/top、x1y1x2y2 角点）或数组 [x,y,width,height]，并夹取到图片边界内 */
|
||||
function parseRegion(raw: string, image: PNG): { x: number; y: number; width: number; height: number } {
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(raw);
|
||||
} catch {
|
||||
throw new RuntimeError("validation_error", "区域 JSON 无法解析", { detail: raw.slice(0, 200) });
|
||||
}
|
||||
const numberOf = (value: unknown): number | null => {
|
||||
const n = Number(value);
|
||||
return Number.isFinite(n) ? n : null;
|
||||
};
|
||||
if (Array.isArray(parsed) && parsed.length === 4) {
|
||||
const [x, y, width, height] = parsed.map(numberOf);
|
||||
if (x !== null && y !== null && width !== null && height !== null) {
|
||||
return clampRegion({ x, y, width, height }, image);
|
||||
}
|
||||
}
|
||||
if (parsed && typeof parsed === "object") {
|
||||
const record = parsed as Record<string, unknown>;
|
||||
const x = numberOf(record.x ?? record.left);
|
||||
const y = numberOf(record.y ?? record.top);
|
||||
const width = numberOf(record.width);
|
||||
const height = numberOf(record.height);
|
||||
if (x !== null && y !== null && width !== null && height !== null) {
|
||||
return clampRegion({ x, y, width, height }, image);
|
||||
}
|
||||
const x2 = numberOf(record.x2 ?? record.right);
|
||||
const y2 = numberOf(record.y2 ?? record.bottom);
|
||||
if (x !== null && y !== null && x2 !== null && y2 !== null) {
|
||||
return clampRegion({ x, y, width: x2 - x, height: y2 - y }, image);
|
||||
}
|
||||
}
|
||||
throw new RuntimeError("validation_error", "区域 JSON 不符合要求", {
|
||||
detail: raw.slice(0, 200),
|
||||
recommendation: '使用 {"x":0,"y":0,"width":100,"height":80} 或 [x, y, width, height]'
|
||||
});
|
||||
}
|
||||
|
||||
function clampRegion(region: { x: number; y: number; width: number; height: number }, image: PNG): { x: number; y: number; width: number; height: number } {
|
||||
const x = Math.max(0, Math.min(Math.round(region.x), image.width - 1));
|
||||
const y = Math.max(0, Math.min(Math.round(region.y), image.height - 1));
|
||||
const width = Math.max(1, Math.min(Math.round(region.width), image.width - x));
|
||||
const height = Math.max(1, Math.min(Math.round(region.height), image.height - y));
|
||||
return { x, y, width, height };
|
||||
}
|
||||
|
||||
function cropPng(source: PNG, region: { x: number; y: number; width: number; height: number }): PNG {
|
||||
const cropped = new PNG({ width: region.width, height: region.height });
|
||||
PNG.bitblt(source, cropped, region.x, region.y, region.width, region.height, 0, 0);
|
||||
return cropped;
|
||||
}
|
||||
|
||||
register("Image_Screenshot", async (_ctx, node, args) => {
|
||||
const folder = text(args, ["outImagePath", "folder"]);
|
||||
const name = text(args, ["outImageName", "name"], `screenshot_${Date.now()}.png`);
|
||||
const output = path.resolve(text(args, ["image_path"], folder ? path.join(folder, name) : name));
|
||||
const imageArg = text(args, ["image_path"]);
|
||||
const source = text(args, ["pictureSource"], "1");
|
||||
const range = text(args, ["shotRange"], "1");
|
||||
|
||||
let image: PNG;
|
||||
let output: string;
|
||||
if (source === "2") {
|
||||
// 指定图片文件来源：image_path 为输入图片；输出走 输出目录+文件名，两者均未配置时直接返回源文件
|
||||
if (!imageArg.trim()) {
|
||||
throw new RuntimeError("validation_error", "图像来源为「指定图片文件」时，目标文件路径需填写源图片路径", {
|
||||
recommendation: "在目标文件路径选择一张已存在的图片，或将图像来源改回「当前屏幕」"
|
||||
});
|
||||
}
|
||||
const sourceFile = path.resolve(imageArg);
|
||||
image = PNG.sync.read(await fs.readFile(sourceFile));
|
||||
output = folder ? path.resolve(path.join(folder, name)) : sourceFile;
|
||||
} else {
|
||||
// 当前屏幕来源：image_path 即保存路径，缺省用 输出目录+文件名
|
||||
image = await capturePrimaryScreenPng();
|
||||
output = path.resolve(imageArg || (folder ? path.join(folder, name) : name));
|
||||
}
|
||||
|
||||
if (range === "2") {
|
||||
const regionRaw = text(args, ["position", "action", "position_img"]);
|
||||
if (!regionRaw.trim()) {
|
||||
throw new RuntimeError("validation_error", "截图范围为「框选区域」时，position 需填写区域 JSON", {
|
||||
recommendation: '在 position 填写 {"x":0,"y":0,"width":100,"height":80} 或 [x, y, width, height]，或将截图范围改回「整个屏幕」'
|
||||
});
|
||||
}
|
||||
image = cropPng(image, parseRegion(regionRaw, image));
|
||||
}
|
||||
|
||||
await fs.mkdir(path.dirname(output), { recursive: true });
|
||||
await powershell(`Add-Type -AssemblyName System.Drawing; Add-Type -AssemblyName System.Windows.Forms; $bounds = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds; $bitmap = New-Object System.Drawing.Bitmap $bounds.Width,$bounds.Height; $graphics = [System.Drawing.Graphics]::FromImage($bitmap); $graphics.CopyFromScreen($bounds.Location,[System.Drawing.Point]::Empty,$bounds.Size); $bitmap.Save(${psString(output)},[System.Drawing.Imaging.ImageFormat]::Png); $graphics.Dispose(); $bitmap.Dispose()`);
|
||||
await fs.writeFile(output, PNG.sync.write(image));
|
||||
return output;
|
||||
});
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user