feat: 屏幕截图组件实现图像来源与截图范围(原 P2 问题)
- vision.ts 重写 Image_Screenshot:pictureSource=1 当前屏幕(desktopCapturer)/2 指定图片文件(image_path 作输入, 输出走输出目录+文件名, 未配置则原样返回);shotRange=2 框选区域按 position/action 区域 JSON 裁剪(pngjs bitblt),区域支持 {x,y,width,height}/[x,y,w,h]/left,top/角点写法并夹取边界
- 移除 browser.ts 重复注册,实现单点维护
- catalog: pictureSource/shotRange 补下拉选项(当前屏幕/指定图片文件, 整个屏幕/框选区域)
- typecheck 通过;browser/cv-vision/capability-audit/app-system 回归 50 项全过
This commit is contained in:
1 parent
c11e1d1527
commit
342145cc46
3 files changed
+121
-22
No files matched your search
+22
-2
@@ -61865,7 +61865,17 @@
|
|||||||
"placeholder": "",
|
"placeholder": "",
|
||||||
"description": "选择要截图的图像来源,可以是当前屏幕,也可以是指定的图片文件。",
|
"description": "选择要截图的图像来源,可以是当前屏幕,也可以是指定的图片文件。",
|
||||||
"advanced": false,
|
"advanced": false,
|
||||||
"valueTypes": []
|
"valueTypes": [],
|
||||||
|
"options": [
|
||||||
|
{
|
||||||
|
"label": "当前屏幕",
|
||||||
|
"value": "1"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "指定图片文件",
|
||||||
|
"value": "2"
|
||||||
|
}
|
||||||
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"key": "shotRange",
|
"key": "shotRange",
|
||||||
@@ -61878,7 +61888,17 @@
|
|||||||
"placeholder": "",
|
"placeholder": "",
|
||||||
"description": "选择要截图的范围,可以使截取整个屏幕,或手动选择截图区域",
|
"description": "选择要截图的范围,可以使截取整个屏幕,或手动选择截图区域",
|
||||||
"advanced": false,
|
"advanced": false,
|
||||||
"valueTypes": []
|
"valueTypes": [],
|
||||||
|
"options": [
|
||||||
|
{
|
||||||
|
"label": "整个屏幕",
|
||||||
|
"value": "1"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "框选区域",
|
||||||
|
"value": "2"
|
||||||
|
}
|
||||||
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"key": "image_path",
|
"key": "image_path",
|
||||||
|
|||||||
@@ -178,21 +178,4 @@ register("WebBrowser_Screenshot", async (ctx, node, args) => {
|
|||||||
return file;
|
return file;
|
||||||
});
|
});
|
||||||
|
|
||||||
register("Image_Screenshot", async (ctx, node, args) => {
|
// Image_Screenshot 统一在 vision.ts 实现(屏幕/文件来源 + 区域裁剪),此处不再重复注册
|
||||||
const { desktopCapturer, screen } = await import("electron");
|
|
||||||
const folder = String(args.outImagePath ?? args.image_path ?? ctx.evidenceDir);
|
|
||||||
const name = String(args.outImageName ?? `${node.id}.png`);
|
|
||||||
const { promises: fs } = await import("node:fs");
|
|
||||||
const path = await import("node:path");
|
|
||||||
const target = path.resolve(folder.endsWith(".png") ? folder : path.join(folder, name));
|
|
||||||
const primary = screen.getPrimaryDisplay();
|
|
||||||
const sources = await desktopCapturer.getSources({
|
|
||||||
types: ["screen"],
|
|
||||||
thumbnailSize: { width: primary.size.width, height: primary.size.height }
|
|
||||||
});
|
|
||||||
const source = sources[0];
|
|
||||||
if (!source) throw new RuntimeError("desktop_error", "无法获取屏幕截图");
|
|
||||||
await fs.mkdir(path.dirname(target), { recursive: true });
|
|
||||||
await fs.writeFile(target, source.thumbnail.toPNG());
|
|
||||||
return target;
|
|
||||||
});
|
|
||||||
@@ -10,12 +10,108 @@ function text(args: Record<string, unknown>, keys: string[], fallback = ""): str
|
|||||||
return fallback;
|
return fallback;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function capturePrimaryScreenPng(): Promise<PNG> {
|
||||||
|
const { desktopCapturer, screen } = await import("electron");
|
||||||
|
const primary = screen.getPrimaryDisplay();
|
||||||
|
const sources = await desktopCapturer.getSources({
|
||||||
|
types: ["screen"],
|
||||||
|
thumbnailSize: { width: primary.size.width, height: primary.size.height }
|
||||||
|
});
|
||||||
|
const source = sources[0];
|
||||||
|
if (!source) throw new RuntimeError("desktop_error", "无法获取屏幕截图");
|
||||||
|
return PNG.sync.read(source.thumbnail.toPNG());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 解析区域 JSON:对象 {x,y,width,height}(兼容 left/top、x1y1x2y2 角点)或数组 [x,y,width,height],并夹取到图片边界内 */
|
||||||
|
function parseRegion(raw: string, image: PNG): { x: number; y: number; width: number; height: number } {
|
||||||
|
let parsed: unknown;
|
||||||
|
try {
|
||||||
|
parsed = JSON.parse(raw);
|
||||||
|
} catch {
|
||||||
|
throw new RuntimeError("validation_error", "区域 JSON 无法解析", { detail: raw.slice(0, 200) });
|
||||||
|
}
|
||||||
|
const numberOf = (value: unknown): number | null => {
|
||||||
|
const n = Number(value);
|
||||||
|
return Number.isFinite(n) ? n : null;
|
||||||
|
};
|
||||||
|
if (Array.isArray(parsed) && parsed.length === 4) {
|
||||||
|
const [x, y, width, height] = parsed.map(numberOf);
|
||||||
|
if (x !== null && y !== null && width !== null && height !== null) {
|
||||||
|
return clampRegion({ x, y, width, height }, image);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (parsed && typeof parsed === "object") {
|
||||||
|
const record = parsed as Record<string, unknown>;
|
||||||
|
const x = numberOf(record.x ?? record.left);
|
||||||
|
const y = numberOf(record.y ?? record.top);
|
||||||
|
const width = numberOf(record.width);
|
||||||
|
const height = numberOf(record.height);
|
||||||
|
if (x !== null && y !== null && width !== null && height !== null) {
|
||||||
|
return clampRegion({ x, y, width, height }, image);
|
||||||
|
}
|
||||||
|
const x2 = numberOf(record.x2 ?? record.right);
|
||||||
|
const y2 = numberOf(record.y2 ?? record.bottom);
|
||||||
|
if (x !== null && y !== null && x2 !== null && y2 !== null) {
|
||||||
|
return clampRegion({ x, y, width: x2 - x, height: y2 - y }, image);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw new RuntimeError("validation_error", "区域 JSON 不符合要求", {
|
||||||
|
detail: raw.slice(0, 200),
|
||||||
|
recommendation: '使用 {"x":0,"y":0,"width":100,"height":80} 或 [x, y, width, height]'
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function clampRegion(region: { x: number; y: number; width: number; height: number }, image: PNG): { x: number; y: number; width: number; height: number } {
|
||||||
|
const x = Math.max(0, Math.min(Math.round(region.x), image.width - 1));
|
||||||
|
const y = Math.max(0, Math.min(Math.round(region.y), image.height - 1));
|
||||||
|
const width = Math.max(1, Math.min(Math.round(region.width), image.width - x));
|
||||||
|
const height = Math.max(1, Math.min(Math.round(region.height), image.height - y));
|
||||||
|
return { x, y, width, height };
|
||||||
|
}
|
||||||
|
|
||||||
|
function cropPng(source: PNG, region: { x: number; y: number; width: number; height: number }): PNG {
|
||||||
|
const cropped = new PNG({ width: region.width, height: region.height });
|
||||||
|
PNG.bitblt(source, cropped, region.x, region.y, region.width, region.height, 0, 0);
|
||||||
|
return cropped;
|
||||||
|
}
|
||||||
|
|
||||||
register("Image_Screenshot", async (_ctx, node, args) => {
|
register("Image_Screenshot", async (_ctx, node, args) => {
|
||||||
const folder = text(args, ["outImagePath", "folder"]);
|
const folder = text(args, ["outImagePath", "folder"]);
|
||||||
const name = text(args, ["outImageName", "name"], `screenshot_${Date.now()}.png`);
|
const name = text(args, ["outImageName", "name"], `screenshot_${Date.now()}.png`);
|
||||||
const output = path.resolve(text(args, ["image_path"], folder ? path.join(folder, name) : name));
|
const imageArg = text(args, ["image_path"]);
|
||||||
|
const source = text(args, ["pictureSource"], "1");
|
||||||
|
const range = text(args, ["shotRange"], "1");
|
||||||
|
|
||||||
|
let image: PNG;
|
||||||
|
let output: string;
|
||||||
|
if (source === "2") {
|
||||||
|
// 指定图片文件来源:image_path 为输入图片;输出走 输出目录+文件名,两者均未配置时直接返回源文件
|
||||||
|
if (!imageArg.trim()) {
|
||||||
|
throw new RuntimeError("validation_error", "图像来源为「指定图片文件」时,目标文件路径需填写源图片路径", {
|
||||||
|
recommendation: "在目标文件路径选择一张已存在的图片,或将图像来源改回「当前屏幕」"
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const sourceFile = path.resolve(imageArg);
|
||||||
|
image = PNG.sync.read(await fs.readFile(sourceFile));
|
||||||
|
output = folder ? path.resolve(path.join(folder, name)) : sourceFile;
|
||||||
|
} else {
|
||||||
|
// 当前屏幕来源:image_path 即保存路径,缺省用 输出目录+文件名
|
||||||
|
image = await capturePrimaryScreenPng();
|
||||||
|
output = path.resolve(imageArg || (folder ? path.join(folder, name) : name));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (range === "2") {
|
||||||
|
const regionRaw = text(args, ["position", "action", "position_img"]);
|
||||||
|
if (!regionRaw.trim()) {
|
||||||
|
throw new RuntimeError("validation_error", "截图范围为「框选区域」时,position 需填写区域 JSON", {
|
||||||
|
recommendation: '在 position 填写 {"x":0,"y":0,"width":100,"height":80} 或 [x, y, width, height],或将截图范围改回「整个屏幕」'
|
||||||
|
});
|
||||||
|
}
|
||||||
|
image = cropPng(image, parseRegion(regionRaw, image));
|
||||||
|
}
|
||||||
|
|
||||||
await fs.mkdir(path.dirname(output), { recursive: true });
|
await fs.mkdir(path.dirname(output), { recursive: true });
|
||||||
await powershell(`Add-Type -AssemblyName System.Drawing; Add-Type -AssemblyName System.Windows.Forms; $bounds = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds; $bitmap = New-Object System.Drawing.Bitmap $bounds.Width,$bounds.Height; $graphics = [System.Drawing.Graphics]::FromImage($bitmap); $graphics.CopyFromScreen($bounds.Location,[System.Drawing.Point]::Empty,$bounds.Size); $bitmap.Save(${psString(output)},[System.Drawing.Imaging.ImageFormat]::Png); $graphics.Dispose(); $bitmap.Dispose()`);
|
await fs.writeFile(output, PNG.sync.write(image));
|
||||||
return output;
|
return output;
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user