feat: 图像查找搜索整个虚拟桌面(多屏拼接), 窗口矩形与区域JSON统一DPI换算
This commit is contained in:
yu
parent
68e16282de
commit
e7669acd16
@ -10,36 +10,69 @@ function text(args: Record<string, unknown>, keys: string[], fallback = ""): str
|
||||
return fallback;
|
||||
}
|
||||
|
||||
/** 主屏物理分辨率截图。desktopCapturer thumbnailSize 乘 scaleFactor，
|
||||
* 与「屏幕框选」生成的模板（物理像素裁剪）尺度一致——
|
||||
* 若用 DIP 逻辑分辨率, 缩放≠100% 的显示器上模板比截图里的同款 UI 大 scaleFactor 倍, 像素匹配永远对不齐 */
|
||||
async function capturePrimaryScreenPng(): Promise<PNG> {
|
||||
/** 全虚拟桌面（所有显示器拼接）物理分辨率截图。
|
||||
* desktopCapturer thumbnailSize 乘 scaleFactor 与「屏幕框选」模板（物理像素裁剪）尺度一致；
|
||||
* 逐屏截图后按各自物理原点拼到一张画布上——目标在副屏也能搜到。
|
||||
* 返回 originX/originY = 拼接画布左上角对应的屏幕物理坐标（副屏在主屏左侧时为负），用于结果坐标平移。 */
|
||||
async function captureVirtualScreenPng(): Promise<{ png: PNG; originX: number; originY: number }> {
|
||||
const { desktopCapturer, screen } = await import("electron");
|
||||
const primary = screen.getPrimaryDisplay();
|
||||
const scale = primary.scaleFactor || 1;
|
||||
const sources = await desktopCapturer.getSources({
|
||||
types: ["screen"],
|
||||
thumbnailSize: {
|
||||
width: Math.round(primary.size.width * scale),
|
||||
height: Math.round(primary.size.height * scale)
|
||||
const displays = screen.getAllDisplays();
|
||||
if (displays.length === 0) throw new RuntimeError("desktop_error", "无法获取屏幕信息");
|
||||
// 每屏按自己的物理分辨率截图（display_id 匹配 → 缩略图尺寸匹配 → 按物理原点排序对位）
|
||||
const captures: { display: Electron.Display; origin: { x: number; y: number }; png: PNG }[] = [];
|
||||
for (const display of displays) {
|
||||
const scale = display.scaleFactor || 1;
|
||||
const wantW = Math.round(display.bounds.width * scale);
|
||||
const wantH = Math.round(display.bounds.height * scale);
|
||||
const sources = await desktopCapturer.getSources({
|
||||
types: ["screen"],
|
||||
thumbnailSize: { width: wantW, height: wantH }
|
||||
});
|
||||
const source = sources.find(item => item.display_id === String(display.id))
|
||||
|| sources.find(item => item.thumbnail.getSize().width === wantW)
|
||||
|| sources[0];
|
||||
if (!source || source.thumbnail.isEmpty()) {
|
||||
throw new RuntimeError("desktop_error", "无法获取屏幕截图");
|
||||
}
|
||||
});
|
||||
const source = sources[0];
|
||||
if (!source) throw new RuntimeError("desktop_error", "无法获取屏幕截图");
|
||||
return PNG.sync.read(source.thumbnail.toPNG());
|
||||
// 显示器 DIP 原点的物理坐标 = 该屏物理原点（屏内线性映射: 物理 = (DIP-原点)*缩放 + 物理原点）
|
||||
const origin = screen.dipToScreenPoint({ x: display.bounds.x, y: display.bounds.y });
|
||||
captures.push({ display, origin, png: PNG.sync.read(source.thumbnail.toPNG()) });
|
||||
}
|
||||
const originX = Math.min(...captures.map(c => c.origin.x));
|
||||
const originY = Math.min(...captures.map(c => c.origin.y));
|
||||
const endX = Math.max(...captures.map(c => c.origin.x + c.png.width));
|
||||
const endY = Math.max(...captures.map(c => c.origin.y + c.png.height));
|
||||
const width = endX - originX;
|
||||
const height = endY - originY;
|
||||
const { createCanvas } = await import("@napi-rs/canvas");
|
||||
const canvas = createCanvas(width, height);
|
||||
const context = canvas.getContext("2d");
|
||||
for (const capture of captures) {
|
||||
const sourceCanvas = createCanvas(capture.png.width, capture.png.height);
|
||||
const sourceContext = sourceCanvas.getContext("2d");
|
||||
const imageData = sourceContext.createImageData(capture.png.width, capture.png.height);
|
||||
imageData.data.set(capture.png.data);
|
||||
sourceContext.putImageData(imageData, 0, 0);
|
||||
context.drawImage(sourceCanvas, capture.origin.x - originX, capture.origin.y - originY);
|
||||
}
|
||||
const stitched = context.getImageData(0, 0, width, height);
|
||||
const png = new PNG({ width, height });
|
||||
png.data = Buffer.from(stitched.data);
|
||||
return { png, originX, originY };
|
||||
}
|
||||
|
||||
/** 主屏缩放比：区域 JSON 等参数按 DIP 语义存储，裁剪物理分辨率截图前需换算 */
|
||||
async function primaryScale(): Promise<number> {
|
||||
/** DIP 区域 → 物理像素区域：左上角经 dipToScreenPoint 换算（天然支持副屏/负坐标），宽高乘所在屏缩放比 */
|
||||
async function dipRegionToPhysical(region: { x: number; y: number; width: number; height: number }): Promise<{ x: number; y: number; width: number; height: number }> {
|
||||
const { screen } = await import("electron");
|
||||
return screen.getPrimaryDisplay().scaleFactor || 1;
|
||||
}
|
||||
|
||||
/** DIP 区域 → 物理像素区域（截图已改为物理分辨率，坐标需同步放大） */
|
||||
function scaleRegion(region: { x: number; y: number; width: number; height: number }, scale: number) {
|
||||
const topLeft = screen.dipToScreenPoint({ x: Math.round(region.x), y: Math.round(region.y) });
|
||||
const display = screen.getDisplayMatching({
|
||||
x: region.x, y: region.y,
|
||||
width: Math.max(1, Math.round(region.width)), height: Math.max(1, Math.round(region.height))
|
||||
});
|
||||
const scale = display.scaleFactor || 1;
|
||||
return {
|
||||
x: Math.round(region.x * scale),
|
||||
y: Math.round(region.y * scale),
|
||||
x: topLeft.x,
|
||||
y: topLeft.y,
|
||||
width: Math.max(1, Math.round(region.width * scale)),
|
||||
height: Math.max(1, Math.round(region.height * scale))
|
||||
};
|
||||
@ -153,6 +186,7 @@ register("Image_Screenshot", async (ctx, node, args) => {
|
||||
|
||||
let image: PNG;
|
||||
let output: string;
|
||||
let screenOrigin: { x: number; y: number } | null = null;
|
||||
if (source === "2") {
|
||||
// 指定图片文件来源：image_path 为输入图片；输出走 输出目录+文件名，两者均未配置时直接返回源文件
|
||||
if (!imageArg.trim()) {
|
||||
@ -165,7 +199,10 @@ register("Image_Screenshot", async (ctx, node, args) => {
|
||||
output = folder ? path.resolve(path.join(folder, name)) : sourceFile;
|
||||
} else {
|
||||
// 当前屏幕来源：image_path 即保存路径（文件夹则自动追加默认文件名），缺省用 输出目录+文件名；再缺省落流程证据目录（路径可预期）
|
||||
image = await capturePrimaryScreenPng();
|
||||
// 截取整个虚拟桌面（含副屏），物理分辨率
|
||||
const shot = await captureVirtualScreenPng();
|
||||
image = shot.png;
|
||||
screenOrigin = { x: shot.originX, y: shot.originY };
|
||||
output = await resolveOutputPath(
|
||||
path.resolve(imageArg || (folder ? path.join(folder, name) : path.join(ctx.evidenceDir, name))),
|
||||
name
|
||||
@ -179,9 +216,19 @@ register("Image_Screenshot", async (ctx, node, args) => {
|
||||
recommendation: '在 position 填写 {"x":0,"y":0,"width":100,"height":80} 或 [x, y, width, height]，或将截图范围改回「整个屏幕」'
|
||||
});
|
||||
}
|
||||
// 区域 JSON 是 DIP 坐标，截图是物理分辨率，裁剪前换算
|
||||
const scale = await primaryScale();
|
||||
image = cropPng(image, clampRegion(scaleRegion(parseRegion(regionRaw, image), scale), image));
|
||||
if (screenOrigin) {
|
||||
// 屏幕来源: 区域 JSON 是 DIP 坐标（支持副屏），换算物理坐标后减去拼接画布原点再裁剪
|
||||
const phys = await dipRegionToPhysical(parseRegion(regionRaw, image));
|
||||
image = cropPng(image, clampRegion({
|
||||
x: phys.x - screenOrigin.x,
|
||||
y: phys.y - screenOrigin.y,
|
||||
width: phys.width,
|
||||
height: phys.height
|
||||
}, image));
|
||||
} else {
|
||||
// 图片文件来源: 区域按该图片自身像素坐标
|
||||
image = cropPng(image, parseRegion(regionRaw, image));
|
||||
}
|
||||
}
|
||||
|
||||
await fs.mkdir(path.dirname(output), { recursive: true });
|
||||
@ -191,20 +238,15 @@ register("Image_Screenshot", async (ctx, node, args) => {
|
||||
|
||||
interface MatchResult { x: number; y: number; width: number; height: number; score: number; }
|
||||
|
||||
async function captureScreen(target: string): Promise<string> {
|
||||
await fs.mkdir(path.dirname(target), { recursive: true });
|
||||
await fs.writeFile(target, PNG.sync.write(await capturePrimaryScreenPng()));
|
||||
return target;
|
||||
}
|
||||
|
||||
interface WindowRect { x: number; y: number; width: number; height: number }
|
||||
|
||||
/** 按标题（精确→包含）或进程名查找窗口，前置并返回窗口矩形（屏幕坐标） */
|
||||
/** 按标题（精确→包含）或进程名查找窗口，前置并返回窗口矩形（物理屏幕坐标） */
|
||||
async function focusWindowRect(windowObj: string): Promise<WindowRect> {
|
||||
const quoted = psString(windowObj);
|
||||
const stdout = await powershell(`Add-Type @'
|
||||
using System; using System.Runtime.InteropServices;
|
||||
public static class WinRect {
|
||||
[DllImport("user32.dll")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value);
|
||||
[DllImport("user32.dll")] public static extern IntPtr FindWindow(string cls, string title);
|
||||
[DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr h);
|
||||
[DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr h, int cmd);
|
||||
@ -212,6 +254,8 @@ public static class WinRect {
|
||||
public struct RECT { public int Left; public int Top; public int Right; public int Bottom; }
|
||||
}
|
||||
'@;
|
||||
# 不声明 DPI 感知时 GetWindowRect 返回虚拟化坐标, 与物理分辨率截图对不上
|
||||
try { [void][WinRect]::SetProcessDpiAwarenessContext([IntPtr](-4)) } catch {};
|
||||
$h = [WinRect]::FindWindow($null, ${quoted});
|
||||
if ($h -eq [IntPtr]::Zero) { $p = Get-Process | Where-Object { $_.MainWindowTitle -and ($_.MainWindowTitle -eq ${quoted} -or $_.MainWindowTitle.Contains(${quoted})) } | Select-Object -First 1; if ($p) { $h = $p.MainWindowHandle } }
|
||||
if ($h -eq [IntPtr]::Zero) { Write-Output 'NOT_FOUND'; exit }
|
||||
@ -337,31 +381,32 @@ async function locate(args: Record<string, unknown>, evidenceDir: string, nodeId
|
||||
if (!rawTemplate.trim()) throw new RuntimeError("validation_error", "模板图片路径为空", { nodeId });
|
||||
const template = path.resolve(rawTemplate);
|
||||
|
||||
// 查找范围：1 整个屏幕（默认）/ 2 指定窗口 / 3 指定区域；窗口/区域模式下结果坐标会平移回屏幕坐标系
|
||||
// 查找范围：1 整个虚拟桌面（默认，含副屏）/ 2 指定窗口 / 3 指定区域；
|
||||
// 截图统一为物理分辨率拼接画布, 结果坐标加回画布原点即屏幕物理坐标（可直接用于点击）
|
||||
const rangeMode = text(args, ["find_range"], "1");
|
||||
const screenshot = path.join(evidenceDir, `${nodeId}_screen.png`);
|
||||
const shot = await captureVirtualScreenPng();
|
||||
let offsetX = 0;
|
||||
let offsetY = 0;
|
||||
if (rangeMode === "2") {
|
||||
const windowObj = text(args, ["window_obj"]);
|
||||
if (!windowObj.trim()) throw new RuntimeError("validation_error", "查找范围为「指定窗口」时，窗口对象不能为空", { nodeId, recommendation: "填写窗口标题（或标题关键字、进程名）" });
|
||||
const rect = await focusWindowRect(windowObj);
|
||||
const screen = await capturePrimaryScreenPng();
|
||||
await fs.writeFile(screenshot, PNG.sync.write(cropPng(screen, clampRegion(rect, screen))));
|
||||
const cropRect = clampRegion({ x: rect.x - shot.originX, y: rect.y - shot.originY, width: rect.width, height: rect.height }, shot.png);
|
||||
await fs.writeFile(screenshot, PNG.sync.write(cropPng(shot.png, cropRect)));
|
||||
offsetX = rect.x;
|
||||
offsetY = rect.y;
|
||||
} else if (rangeMode === "3") {
|
||||
const regionRaw = text(args, ["shot_position", "action", "position"]);
|
||||
if (!regionRaw.trim()) throw new RuntimeError("validation_error", "查找范围为「指定区域」时，需要填写区域 JSON", { nodeId, recommendation: '在 shot_position 填写 {"x":0,"y":0,"width":100,"height":80} 或 [x, y, width, height]' });
|
||||
const scale = await primaryScale();
|
||||
const screen = await capturePrimaryScreenPng();
|
||||
// 区域 JSON 是 DIP 坐标（框选工具按 DIP 返回），截图是物理分辨率，裁剪前放大
|
||||
const region = clampRegion(scaleRegion(parseRegion(regionRaw, screen), scale), screen);
|
||||
await fs.writeFile(screenshot, PNG.sync.write(cropPng(screen, region)));
|
||||
offsetX = region.x;
|
||||
offsetY = region.y;
|
||||
// 区域 JSON 是 DIP 坐标（框选工具按 DIP 返回, 支持副屏负坐标），换算物理坐标后减画布原点裁剪
|
||||
const phys = await dipRegionToPhysical(parseRegion(regionRaw, shot.png));
|
||||
const cropRect = clampRegion({ x: phys.x - shot.originX, y: phys.y - shot.originY, width: phys.width, height: phys.height }, shot.png);
|
||||
await fs.writeFile(screenshot, PNG.sync.write(cropPng(shot.png, cropRect)));
|
||||
offsetX = phys.x;
|
||||
offsetY = phys.y;
|
||||
} else {
|
||||
await captureScreen(screenshot);
|
||||
await fs.writeFile(screenshot, PNG.sync.write(shot.png));
|
||||
}
|
||||
|
||||
// accuracy 字段语义为「相似度」(0~1, 越大越严格, 0.9=至少90%像素一致)——与界面文案一致。
|
||||
@ -369,11 +414,10 @@ async function locate(args: Record<string, unknown>, evidenceDir: string, nodeId
|
||||
// 用户填 0.9 反而放宽到"接受90%像素差异", 导致到处误命中。
|
||||
const rawAccuracy = Number(text(args, ["accuracy"], "0.8"));
|
||||
const similarity = Math.max(0.05, Math.min(1, Number.isFinite(rawAccuracy) && rawAccuracy > 0 ? rawAccuracy : 0.8));
|
||||
// 原生 1:1 匹配失败时, 按主屏缩放比对模板放大/缩小重试——
|
||||
// 兼容旧版 DIP 截图存的模板与任意来源的图片
|
||||
const scale = await primaryScale();
|
||||
// 原生 1:1 匹配失败时, 按缩放比对模板放大/缩小重试——兼容旧版 DIP 截图存的模板与任意来源的图片
|
||||
const scale = (await import("electron")).screen.getPrimaryDisplay().scaleFactor || 1;
|
||||
const result = await findImage(template, screenshot, 1 - similarity, scale !== 1 ? [scale, 1 / scale] : []);
|
||||
if (!result) throw new RuntimeError("not_found", `屏幕中未找到目标图像（要求相似度 ${(similarity * 100).toFixed(0)}%，未达到阈值）`, { nodeId, recommendation: "重新框选模板（框小一点、只框目标图案），或适当调低相似度（如 0.8）" });
|
||||
if (!result) throw new RuntimeError("not_found", `屏幕中未找到目标图像（要求相似度 ${(similarity * 100).toFixed(0)}%，未达到阈值）`, { nodeId, recommendation: "重新框选模板（框小一点、只框目标图案），或适当调低相似度（如 0.8）；目标在副屏时确认其清晰可见" });
|
||||
return { ...result, x: result.x + offsetX, y: result.y + offsetY };
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user