feat: 框选与 CV 查找支持多显示器副屏
- pickScreenRegion 每个显示器各开一个框选覆盖层, 副屏可直接框选; 返回屏幕绝对 DIP 坐标+该屏 scaleFactor - cv-vision 全屏查找改为逐显示器 desktopCapturer 截屏(含负坐标副屏), 命中换算为绝对 DIP 坐标; 区域截图命中按 scaleFactor 换算 - 点击/悬停/输入坐标经 screen.dipToScreenPoint 转物理像素, 修复缩放非 100% 时点击偏移 - 新增 2 个多显示器测试用例(副屏命中映射/区域图缩放换算), 11/11 通过
This commit is contained in:
yu
parent
22054fed37
commit
b40c15d5b2
@ -1,4 +1,7 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
const powershellMock = vi.hoisted(() => vi.fn(async () => '{"ok":true}'));
|
||||
const detectMock = vi.hoisted(() => vi.fn(async () => ({
|
||||
@ -7,11 +10,26 @@ const detectMock = vi.hoisted(() => vi.fn(async () => ({
|
||||
{ text: "确认", box: [[100, 200], [130, 200], [130, 210], [100, 210]] }
|
||||
]
|
||||
})));
|
||||
// desktopCapturer 模拟: 默认无画面(走 GDI 回退), 多屏用例按请求宽度提供对应显示器画面
|
||||
const electronState = vi.hoisted(() => ({ sources: [] as Array<{ display_id: string; thumbnail: any }> }));
|
||||
|
||||
vi.mock("./desktop", () => ({ powershell: powershellMock }));
|
||||
vi.mock("@gutenye/ocr-node", () => ({
|
||||
default: { create: vi.fn(async () => ({ detect: detectMock })) }
|
||||
}));
|
||||
vi.mock("electron", () => ({
|
||||
desktopCapturer: {
|
||||
getSources: vi.fn(async (opts: { thumbnailSize: { width: number } }) =>
|
||||
electronState.sources.filter(s => s.thumbnail.getSize().width === opts.thumbnailSize.width))
|
||||
},
|
||||
screen: {
|
||||
getAllDisplays: vi.fn(() => [
|
||||
{ id: 1, bounds: { x: 0, y: 0, width: 1920, height: 1080 }, scaleFactor: 1 },
|
||||
{ id: 2, bounds: { x: 1920, y: 0, width: 1280, height: 720 }, scaleFactor: 2 }
|
||||
]),
|
||||
dipToScreenPoint: vi.fn((p: { x: number; y: number }) => ({ x: p.x, y: p.y }))
|
||||
}
|
||||
}));
|
||||
|
||||
import { getHandler } from "../engine/registry";
|
||||
import "./cv-vision";
|
||||
@ -20,6 +38,7 @@ describe("CV vision capabilities", () => {
|
||||
beforeEach(() => {
|
||||
powershellMock.mockClear();
|
||||
detectMock.mockClear();
|
||||
electronState.sources = [];
|
||||
});
|
||||
|
||||
it.each(["CV_GetText", "CV_SetText", "CV_MouseHover", "CV_MouseClick"])("registers %s", (id) => {
|
||||
@ -67,4 +86,41 @@ describe("CV vision capabilities", () => {
|
||||
searchRegion: JSON.stringify({ x: 500, y: 500, width: 50, height: 50 })
|
||||
}, {} as any)).rejects.toThrow("未找到目标文本");
|
||||
});
|
||||
|
||||
it("locates targets on a secondary display and maps DIP coordinates", async () => {
|
||||
// 主屏无画面, 副屏(bounds 1920,0 1280x720 缩放2)物理中心(450,210) → DIP(2145,105)
|
||||
const fakeThumb = (w: number, h: number): any => ({
|
||||
isEmpty: () => false,
|
||||
getSize: () => ({ width: w, height: h }),
|
||||
toPNG: () => Buffer.from([137, 80, 78, 71])
|
||||
});
|
||||
electronState.sources = [{ display_id: "2", thumbnail: fakeThumb(2560, 1440) }];
|
||||
detectMock.mockImplementationOnce(async () => ({
|
||||
texts: [{ text: "副屏按钮", box: [[400, 200], [500, 200], [500, 220], [400, 220]] }]
|
||||
}));
|
||||
await expect(getHandler("CV_MouseClick")!({ log: vi.fn() } as any, { id: "n5", componentId: "CV_MouseClick" } as any, {
|
||||
cvSelector: "副屏按钮"
|
||||
}, {} as any)).resolves.toBe(true);
|
||||
const script = String((powershellMock.mock.calls as unknown as Array<[string]>).at(-1)?.[0] ?? "");
|
||||
expect(script).toContain("SetCursorPos(2145, 105)");
|
||||
});
|
||||
|
||||
it("maps region-image hits through the display scale", async () => {
|
||||
// 框选图来自缩放2的副屏: 图内物理中心(120,90) + 框选DIP原点(1920,0) → DIP(1980,45)
|
||||
const regionShot = path.join(os.tmpdir(), "rpa-cv-region-test-" + Date.now() + ".png");
|
||||
fs.writeFileSync(regionShot, Buffer.from([137, 80, 78, 71]));
|
||||
try {
|
||||
detectMock.mockImplementationOnce(async () => ({
|
||||
texts: [{ text: "确认", box: [[100, 80], [140, 80], [140, 100], [100, 100]] }]
|
||||
}));
|
||||
await expect(getHandler("CV_MouseHover")!({ log: vi.fn() } as any, { id: "n6", componentId: "CV_MouseHover" } as any, {
|
||||
cvSelector: "确认",
|
||||
searchRegion: JSON.stringify({ x: 1920, y: 0, width: 640, height: 360, scale: 2, imagePath: regionShot })
|
||||
}, {} as any)).resolves.toBe(true);
|
||||
const script = String((powershellMock.mock.calls as unknown as Array<[string]>).at(-1)?.[0] ?? "");
|
||||
expect(script).toContain("SetCursorPos(1980, 45)");
|
||||
} finally {
|
||||
try { fs.unlinkSync(regionShot); } catch {}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@ -41,30 +41,43 @@ async function gdiGrabScreen(imagePath: string): Promise<void> {
|
||||
await powershell(psScript, 30000);
|
||||
}
|
||||
|
||||
async function grabScreen(imagePath: string): Promise<void> {
|
||||
fs.mkdirSync(path.dirname(imagePath), { recursive: true });
|
||||
// GDI CopyFromScreen 截不到 GPU 合成窗口(Electron 自身界面, 会截出全黑),
|
||||
// 优先用 desktopCapturer 走系统合成器; 不可用时(如单测环境)回退 GDI 脚本
|
||||
/** 一次屏幕截图: 图片左上角在虚拟桌面上的绝对 DIP 坐标 + 物理/DIP 缩放比 */
|
||||
interface ScreenShot { imagePath: string; dipX: number; dipY: number; scale: number }
|
||||
|
||||
/** desktopCapturer 逐显示器截屏(每屏原生分辨率), 不可用时回退 GDI 全虚拟桌面单图 */
|
||||
async function grabScreenAll(baseDir: string): Promise<ScreenShot[]> {
|
||||
fs.mkdirSync(baseDir, { recursive: true });
|
||||
try {
|
||||
const { desktopCapturer, screen } = await import("electron");
|
||||
const display = screen.getPrimaryDisplay();
|
||||
const sources = await desktopCapturer.getSources({
|
||||
types: ["screen"],
|
||||
thumbnailSize: { width: display.size.width, height: display.size.height }
|
||||
});
|
||||
const source = sources.find(item => item.display_id === String(display.id)) || sources[0];
|
||||
if (source && !source.thumbnail.isEmpty()) {
|
||||
fs.writeFileSync(imagePath, source.thumbnail.toPNG());
|
||||
return;
|
||||
if (desktopCapturer && screen?.getAllDisplays) {
|
||||
const shots: ScreenShot[] = [];
|
||||
const displays = screen.getAllDisplays();
|
||||
for (const display of displays) {
|
||||
const scale = display.scaleFactor || 1;
|
||||
const sources = await desktopCapturer.getSources({
|
||||
types: ["screen"],
|
||||
thumbnailSize: { width: Math.round(display.bounds.width * scale), height: Math.round(display.bounds.height * scale) }
|
||||
});
|
||||
const source = sources.find(item => item.display_id === String(display.id))
|
||||
|| sources.find(item => item.thumbnail.getSize().width === Math.round(display.bounds.width * scale))
|
||||
|| sources[0];
|
||||
if (!source || source.thumbnail.isEmpty()) continue;
|
||||
const imagePath = path.join(baseDir, `rpa-cv-screen-${display.id}-${Date.now()}.png`);
|
||||
fs.writeFileSync(imagePath, source.thumbnail.toPNG());
|
||||
shots.push({ imagePath, dipX: display.bounds.x, dipY: display.bounds.y, scale });
|
||||
}
|
||||
if (shots.length) return shots;
|
||||
}
|
||||
} catch {}
|
||||
const imagePath = path.join(baseDir, `rpa-cv-screen-${Date.now()}.png`);
|
||||
await gdiGrabScreen(imagePath);
|
||||
return [{ imagePath, dipX: 0, dipY: 0, scale: 1 }];
|
||||
}
|
||||
|
||||
interface OcrLine { text: string; box?: number[][]; mean?: number }
|
||||
|
||||
/** 搜索区域: 框选生成时含 imagePath(区域截图)+x/y(屏幕偏移); 也可只给坐标范围 */
|
||||
interface CvRegion { imagePath?: string; x?: number; y?: number; width?: number; height?: number }
|
||||
/** 搜索区域: 框选生成时含 imagePath(区域截图)+x/y(屏幕绝对 DIP 坐标)+scale(该屏缩放); 也可只给坐标范围 */
|
||||
interface CvRegion { imagePath?: string; x?: number; y?: number; width?: number; height?: number; scale?: number }
|
||||
|
||||
function parseRegionArg(value: unknown): CvRegion | null {
|
||||
if (!value) return null;
|
||||
@ -89,46 +102,63 @@ export async function locateTextOnScreen(
|
||||
nodeId: string,
|
||||
region?: CvRegion | null
|
||||
): Promise<{ target: string; text: string; cx: number; cy: number }> {
|
||||
let shotPath: string | null = null;
|
||||
let imagePath: string;
|
||||
let offsetX = 0;
|
||||
let offsetY = 0;
|
||||
if (region?.imagePath && fs.existsSync(region.imagePath)) {
|
||||
// 框选截图: OCR 只识别该图, 命中坐标加上框选时的屏幕偏移得到绝对坐标
|
||||
imagePath = region.imagePath;
|
||||
offsetX = Number(region.x ?? 0) || 0;
|
||||
offsetY = Number(region.y ?? 0) || 0;
|
||||
} else {
|
||||
shotPath = path.join(os.tmpdir(), "rpa-cv-screen-" + Date.now() + ".png");
|
||||
await grabScreen(shotPath);
|
||||
imagePath = shotPath;
|
||||
}
|
||||
const ocr = await getOcr();
|
||||
const result = await ocr.detect(imagePath);
|
||||
const texts = result.texts ?? [];
|
||||
const keyword = target.trim();
|
||||
const candidates = texts.filter(t => t.text && t.text.includes(keyword) && t.box && t.box.length >= 4);
|
||||
let hit: OcrLine | undefined = candidates[0];
|
||||
// 未用框选图但给了坐标范围时, 过滤中心落在区域内的命中
|
||||
if (hit && !region?.imagePath && region?.width && region?.height) {
|
||||
const rx = Number(region.x ?? 0) || 0;
|
||||
const ry = Number(region.y ?? 0) || 0;
|
||||
hit = candidates.find(t => {
|
||||
const c = boxCenter(t.box!);
|
||||
return c.cx >= rx && c.cx <= rx + region.width! && c.cy >= ry && c.cy <= ry + region.height!;
|
||||
});
|
||||
// 命中中心换算为屏幕绝对 DIP 坐标
|
||||
let found: { text: string; cx: number; cy: number } | null = null;
|
||||
const cleanups: string[] = [];
|
||||
|
||||
if (region?.imagePath && fs.existsSync(region.imagePath)) {
|
||||
// 框选截图: OCR 只识别该图; 图内物理坐标 ÷ 该屏缩放 + 框选绝对 DIP 原点
|
||||
const scale = Number(region.scale ?? 1) || 1;
|
||||
const result = await ocr.detect(region.imagePath);
|
||||
const hit = (result.texts ?? []).find(t => t.text && t.text.includes(keyword) && t.box && t.box.length >= 4);
|
||||
if (hit?.box) {
|
||||
const c = boxCenter(hit.box);
|
||||
found = {
|
||||
text: hit.text,
|
||||
cx: (Number(region.x ?? 0) || 0) + c.cx / scale,
|
||||
cy: (Number(region.y ?? 0) || 0) + c.cy / scale
|
||||
};
|
||||
}
|
||||
} else {
|
||||
// 全屏查找: 逐显示器截屏识别, 覆盖副屏(含负坐标屏); 命中即停
|
||||
const shots = await grabScreenAll(path.join(os.tmpdir(), "rpa-cv-shots"));
|
||||
for (const shot of shots) {
|
||||
const result = await ocr.detect(shot.imagePath);
|
||||
const candidates = (result.texts ?? []).filter(t => t.text && t.text.includes(keyword) && t.box && t.box.length >= 4);
|
||||
let hit: OcrLine | undefined = candidates[0];
|
||||
// 给了坐标范围(绝对 DIP)时, 过滤中心落在区域内的命中
|
||||
if (hit && region?.width && region?.height) {
|
||||
const rx = Number(region.x ?? 0) || 0;
|
||||
const ry = Number(region.y ?? 0) || 0;
|
||||
hit = candidates.find(t => {
|
||||
const c = boxCenter(t.box!);
|
||||
const dx = shot.dipX + c.cx / shot.scale;
|
||||
const dy = shot.dipY + c.cy / shot.scale;
|
||||
return dx >= rx && dx <= rx + region.width! && dy >= ry && dy <= ry + region.height!;
|
||||
});
|
||||
}
|
||||
if (hit?.box) {
|
||||
const c = boxCenter(hit.box);
|
||||
found = { text: hit.text, cx: shot.dipX + c.cx / shot.scale, cy: shot.dipY + c.cy / shot.scale };
|
||||
break;
|
||||
}
|
||||
cleanups.push(shot.imagePath);
|
||||
}
|
||||
// 成功命中时清理全部截图, 未命中保留全部以便排查
|
||||
if (found) for (const p of shots.map(s => s.imagePath)) { try { fs.unlinkSync(p); } catch {} }
|
||||
}
|
||||
if (!hit || !hit.box || hit.box.length < 4) {
|
||||
// 找不到目标时保留截屏, 便于排查(截屏范围/遮挡/清晰度)
|
||||
|
||||
if (!found) {
|
||||
throw new RuntimeError("cv_not_found", "屏幕上未找到目标文本：" + keyword + (region?.imagePath ? "（搜索范围: 框选区域）" : ""), {
|
||||
nodeId,
|
||||
componentId: nodeId,
|
||||
recommendation: "确认目标文本在" + (region?.imagePath ? "框选区域内" : "屏幕上") + "可见且清晰；本次识别图片已保留: " + (region?.imagePath ?? shotPath)
|
||||
recommendation: "确认目标文本在" + (region?.imagePath ? "框选区域内" : "屏幕上(含副屏)") + "可见且清晰"
|
||||
});
|
||||
}
|
||||
if (shotPath) { try { fs.unlinkSync(shotPath); } catch {} }
|
||||
const c = boxCenter(hit.box);
|
||||
return { target: keyword, text: hit.text, cx: c.cx + offsetX, cy: c.cy + offsetY };
|
||||
for (const p of cleanups) { try { fs.unlinkSync(p); } catch {} }
|
||||
return { target: keyword, text: found.text, cx: found.cx, cy: found.cy };
|
||||
}
|
||||
function screenUser32(): string {
|
||||
return [
|
||||
@ -177,9 +207,21 @@ function coordinateScript(x: number, y: number, action: "hover" | "click" | "dou
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
/** DIP → 物理像素(SetCursorPos 在 DPI 感知进程里用物理坐标); 无 Electron API 时按原值(缩放 100% 等价) */
|
||||
async function toPhysical(x: number, y: number): Promise<{ x: number; y: number }> {
|
||||
try {
|
||||
const { screen } = await import("electron");
|
||||
if (screen && typeof screen.dipToScreenPoint === "function") {
|
||||
const p = screen.dipToScreenPoint({ x: Math.round(x), y: Math.round(y) });
|
||||
return { x: p.x, y: p.y };
|
||||
}
|
||||
} catch {}
|
||||
return { x: Math.round(x), y: Math.round(y) };
|
||||
}
|
||||
|
||||
async function locateFor(args: Record<string, unknown>, node: { id: string; componentId: string }) {
|
||||
const target = cvTarget(args, node.id);
|
||||
// 可选搜索区域: 框选生成 {x,y,width,height,imagePath}, 限定 OCR 查找范围(字段明天接入 UI)
|
||||
// 可选搜索区域: 框选生成 {x,y,width,height,imagePath,scale}, 限定 OCR 查找范围
|
||||
const region = parseRegionArg(args.searchRegion ?? args.region);
|
||||
const located = await locateTextOnScreen(target, node.id, region);
|
||||
const offsetX = Number(args.offsetX ?? args.offset_x ?? 0) || 0;
|
||||
@ -198,24 +240,27 @@ const setTextGround: CapabilityHandler = async (ctx, node, args) => {
|
||||
const value = String(args.value ?? args.text ?? "");
|
||||
if (!value) throw new RuntimeError("validation_error", "CV 文本输入内容为空", { nodeId: node.id, componentId: node.componentId });
|
||||
const { x, y } = await locateFor(args, node);
|
||||
await powershell(coordinateScript(x, y, "paste", value), 30000);
|
||||
ctx.log("CV_SetText input at " + x + "," + y, "info", node.id);
|
||||
const phys = await toPhysical(x, y);
|
||||
await powershell(coordinateScript(phys.x, phys.y, "paste", value), 30000);
|
||||
ctx.log("CV_SetText input at " + phys.x + "," + phys.y, "info", node.id);
|
||||
return true;
|
||||
};
|
||||
|
||||
const mouseHoverGround: CapabilityHandler = async (ctx, node, args) => {
|
||||
const { x, y } = await locateFor(args, node);
|
||||
await powershell(coordinateScript(x, y, "hover"), 15000);
|
||||
ctx.log("CV_MouseHover hover at " + x + "," + y, "info", node.id);
|
||||
const phys = await toPhysical(x, y);
|
||||
await powershell(coordinateScript(phys.x, phys.y, "hover"), 15000);
|
||||
ctx.log("CV_MouseHover hover at " + phys.x + "," + phys.y, "info", node.id);
|
||||
return true;
|
||||
};
|
||||
|
||||
const mouseClickGround: CapabilityHandler = async (ctx, node, args) => {
|
||||
const { x, y } = await locateFor(args, node);
|
||||
const phys = await toPhysical(x, y);
|
||||
const clickway = String(args.clickway ?? "0");
|
||||
const action = clickway === "1" || /double/i.test(clickway) ? "doubleClick" as const : "click" as const;
|
||||
await powershell(coordinateScript(x, y, action), 15000);
|
||||
ctx.log("CV_MouseClick " + action + " at " + x + "," + y, "info", node.id);
|
||||
await powershell(coordinateScript(phys.x, phys.y, action), 15000);
|
||||
ctx.log("CV_MouseClick " + action + " at " + phys.x + "," + phys.y, "info", node.id);
|
||||
return true;
|
||||
};
|
||||
|
||||
|
||||
@ -696,17 +696,8 @@ export async function listVisibleWindows(): Promise<Array<{ title: string; proce
|
||||
return jsonLine<Array<{ title: string; process: string }>>(stdout, "[");
|
||||
}
|
||||
|
||||
export async function pickScreenRegion(): Promise<{ x: number; y: number; width: number; height: number; imagePath: string }> {
|
||||
const appWindows = BrowserWindow.getAllWindows().filter(w => !w.isDestroyed());
|
||||
for (const w of appWindows) w.minimize();
|
||||
const display = screen.getPrimaryDisplay();
|
||||
const sx = display.bounds.x, sy = display.bounds.y, sw = display.bounds.width, sh = display.bounds.height;
|
||||
const win = new BrowserWindow({
|
||||
x: sx, y: sy, width: sw, height: sh,
|
||||
frame: false, transparent: true, alwaysOnTop: true, skipTaskbar: true, resizable: false, movable: false,
|
||||
webPreferences: { contextIsolation: true, nodeIntegration: false, sandbox: true }
|
||||
});
|
||||
const htmlParts = [
|
||||
function regionOverlayHtml(): string {
|
||||
return [
|
||||
"<!DOCTYPE html><html><head><meta charset='utf-8'><style>",
|
||||
"html,body{margin:0;height:100%;background:rgba(0,30,80,0.25);cursor:crosshair;overflow:hidden}",
|
||||
"#sel{position:absolute;border:2px solid #2563eb;background:rgba(37,99,235,0.15);display:none}",
|
||||
@ -720,52 +711,82 @@ export async function pickScreenRegion(): Promise<{ x: number; y: number; width:
|
||||
"document.addEventListener('mouseup', function(e) { if (!start) return; window.__region = { x: Math.min(start[0], e.clientX), y: Math.min(start[1], e.clientY), width: Math.abs(e.clientX - start[0]), height: Math.abs(e.clientY - start[1]) }; window.__done = true; });",
|
||||
"document.addEventListener('keydown', function(e) { if (e.key === 'Escape') { window.__cancelled = true; window.__done = true; } });",
|
||||
"</script></body></html>"
|
||||
];
|
||||
void win.loadURL("data:text/html;charset=utf-8," + encodeURIComponent(htmlParts.join("")));
|
||||
win.show();
|
||||
].join("");
|
||||
}
|
||||
|
||||
export async function pickScreenRegion(): Promise<{ x: number; y: number; width: number; height: number; scale: number; imagePath: string }> {
|
||||
const appWindows = BrowserWindow.getAllWindows().filter(w => !w.isDestroyed());
|
||||
for (const w of appWindows) w.minimize();
|
||||
// 每个显示器各开一个覆盖层, 支持在任意副屏上框选
|
||||
const displays = screen.getAllDisplays();
|
||||
const url = "data:text/html;charset=utf-8," + encodeURIComponent(regionOverlayHtml());
|
||||
const overlays = displays.map(display => {
|
||||
const win = new BrowserWindow({
|
||||
x: display.bounds.x, y: display.bounds.y, width: display.bounds.width, height: display.bounds.height,
|
||||
frame: false, transparent: true, alwaysOnTop: true, skipTaskbar: true, resizable: false, movable: false,
|
||||
webPreferences: { contextIsolation: true, nodeIntegration: false, sandbox: true }
|
||||
});
|
||||
void win.loadURL(url);
|
||||
win.show();
|
||||
return { win, display };
|
||||
});
|
||||
const deadline = Date.now() + 60000;
|
||||
let region: { x: number; y: number; width: number; height: number } | null = null;
|
||||
let cancelled = false;
|
||||
let picked: { region: { x: number; y: number; width: number; height: number }; display: (typeof displays)[number] } | null = null;
|
||||
while (Date.now() < deadline) {
|
||||
await new Promise(r => setTimeout(r, 250));
|
||||
try {
|
||||
const state = await win.webContents.executeJavaScript("JSON.stringify({ done: window.__done === true, cancelled: window.__cancelled === true, region: window.__region || null })", true);
|
||||
const parsed = JSON.parse(state) as { done: boolean; cancelled: boolean; region: { x: number; y: number; width: number; height: number } | null };
|
||||
if (parsed.cancelled) { win.destroy(); throw new RuntimeError("pick_cancelled", "已取消框选"); }
|
||||
if (parsed.done && parsed.region && parsed.region.width > 2 && parsed.region.height > 2) { region = parsed.region; break; }
|
||||
if (parsed.done) break;
|
||||
} catch (error) {
|
||||
if ((error as Error).message.includes("已取消框选")) throw error;
|
||||
for (const { win, display } of overlays) {
|
||||
if (win.isDestroyed()) continue;
|
||||
try {
|
||||
const state = await win.webContents.executeJavaScript("JSON.stringify({ done: window.__done === true, cancelled: window.__cancelled === true, region: window.__region || null })", true);
|
||||
const parsed = JSON.parse(state) as { done: boolean; cancelled: boolean; region: { x: number; y: number; width: number; height: number } | null };
|
||||
if (parsed.cancelled) { cancelled = true; break; }
|
||||
if (parsed.done && parsed.region && parsed.region.width > 2 && parsed.region.height > 2) {
|
||||
picked = { region: parsed.region, display };
|
||||
break;
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
if (cancelled || picked) break;
|
||||
}
|
||||
for (const w of appWindows) { if (!w.isDestroyed()) { w.show(); w.restore(); w.focus(); } }
|
||||
win.destroy();
|
||||
if (!region) throw new RuntimeError("pick_cancelled", "未完成框选");
|
||||
for (const { win } of overlays) if (!win.isDestroyed()) win.destroy();
|
||||
if (cancelled) throw new RuntimeError("pick_cancelled", "已取消框选");
|
||||
if (!picked) throw new RuntimeError("pick_cancelled", "未完成框选");
|
||||
const { region, display } = picked;
|
||||
await new Promise(r => setTimeout(r, 300));
|
||||
const outDir = path.join(app.getPath("userData"), "region-picks");
|
||||
fs.mkdirSync(outDir, { recursive: true });
|
||||
const imagePath = path.join(outDir, `pick-region-${Date.now()}.png`);
|
||||
// GDI CopyFromScreen 截不到 GPU 合成窗口(Electron 自身界面), 会截出全黑;
|
||||
// 改用 desktopCapturer 走系统合成器截屏。覆盖层坐标是 DIP, 裁剪时按 scaleFactor 换算物理像素
|
||||
const dipX = sx + region.x;
|
||||
const dipY = sy + region.y;
|
||||
const target = displayForPoint(dipX + region.width / 2, dipY + region.height / 2);
|
||||
const scale = target.scaleFactor || 1;
|
||||
// 改用 desktopCapturer 走系统合成器截屏。覆盖层坐标是本屏 DIP, 裁剪时按 scaleFactor 换算物理像素
|
||||
const scale = display.scaleFactor || 1;
|
||||
const sources = await desktopCapturer.getSources({
|
||||
types: ["screen"],
|
||||
thumbnailSize: { width: Math.round(target.bounds.width * scale), height: Math.round(target.bounds.height * scale) }
|
||||
thumbnailSize: { width: Math.round(display.bounds.width * scale), height: Math.round(display.bounds.height * scale) }
|
||||
});
|
||||
const source = sources.find(item => item.display_id === String(target.id)) || sources[0];
|
||||
const source = sources.find(item => item.display_id === String(display.id))
|
||||
|| sources.find(item => item.thumbnail.getSize().width === Math.round(display.bounds.width * scale))
|
||||
|| sources[0];
|
||||
if (!source || source.thumbnail.isEmpty()) {
|
||||
throw new RuntimeError("pick_capture_failed", "框选截图失败：未获取到屏幕画面");
|
||||
}
|
||||
const thumbnail = source.thumbnail;
|
||||
const actualScale = thumbnail.getSize().width / Math.max(1, target.bounds.width);
|
||||
const cropX = Math.max(0, Math.min(thumbnail.getSize().width - 1, Math.round((dipX - target.bounds.x) * actualScale)));
|
||||
const cropY = Math.max(0, Math.min(thumbnail.getSize().height - 1, Math.round((dipY - target.bounds.y) * actualScale)));
|
||||
const actualScale = thumbnail.getSize().width / Math.max(1, display.bounds.width);
|
||||
const cropX = Math.max(0, Math.min(thumbnail.getSize().width - 1, Math.round(region.x * actualScale)));
|
||||
const cropY = Math.max(0, Math.min(thumbnail.getSize().height - 1, Math.round(region.y * actualScale)));
|
||||
const cropW = Math.max(1, Math.min(thumbnail.getSize().width - cropX, Math.round(region.width * actualScale)));
|
||||
const cropH = Math.max(1, Math.min(thumbnail.getSize().height - cropY, Math.round(region.height * actualScale)));
|
||||
fs.writeFileSync(imagePath, thumbnail.crop({ x: cropX, y: cropY, width: cropW, height: cropH }).toPNG());
|
||||
return { ...region, imagePath };
|
||||
// 返回屏幕绝对 DIP 坐标(含负值/副屏偏移)+该屏缩放比, 运行时由此换算点击位置
|
||||
return {
|
||||
x: display.bounds.x + region.x,
|
||||
y: display.bounds.y + region.y,
|
||||
width: region.width,
|
||||
height: region.height,
|
||||
scale,
|
||||
imagePath
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user