From 6b05a6566414a373cccde6ae5a65ba952a2b1fc7 Mon Sep 17 00:00:00 2001 From: yu <191972707@qq.com> Date: Tue, 3 Nov 2026 16:28:52 +0800 Subject: [PATCH] =?UTF-8?q?=E9=BC=A0=E6=A0=87=E7=BB=84=E4=BB=B6=E5=9D=90?= =?UTF-8?q?=E6=A0=87=E4=BF=AE=E6=AD=A3:=20DIP=20=E8=BD=AC=E7=89=A9?= =?UTF-8?q?=E7=90=86=E5=83=8F=E7=B4=A0=E4=BF=AE=E5=A4=8D=E7=BC=A9=E6=94=BE?= =?UTF-8?q?=E5=B1=8F=E5=81=8F=E7=A7=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/capabilities/window-extended.ts | 61 ++++++++++++++++++------ 1 file changed, 46 insertions(+), 15 deletions(-) diff --git a/src/main/capabilities/window-extended.ts b/src/main/capabilities/window-extended.ts index cb4e998..2e3ff9b 100644 --- a/src/main/capabilities/window-extended.ts +++ b/src/main/capabilities/window-extended.ts @@ -1,6 +1,7 @@ import { register } from "../engine/registry"; import { RuntimeError, type CapabilityHandler } from "../engine/types"; import { powershell, psString } from "./desktop"; +import { screen } from "electron"; const WINDOW_API = `Add-Type @' using System; @@ -133,61 +134,91 @@ function mousePath(from: [number, number], to: [number, number], mode: string, s return points; } +/** 界面(框选/元素拾取/截图)产出的是 DIP 逻辑坐标；DPI 感知进程的 SetCursorPos 用物理像素， + * 缩放屏(125%/150%)下直接混用会成比例偏移(用户实测 125% 下差约 25% 距离)。 + * 统一在执行前换算；非 Electron 环境(vitest)screen 不可用，回退原值保持旧行为 */ +function dipToPhysical(x: number, y: number): [number, number] { + try { + if (screen && typeof screen.dipToScreenPoint === "function") { + const physical = screen.dipToScreenPoint({ x, y }); + return [Math.round(physical.x), Math.round(physical.y)]; + } + } catch {} + return [Math.round(x), Math.round(y)]; +} + +/** 反向换算：物理像素(如 GetWindowRect 结果) → DIP，用于「相对激活窗口」偏移叠加 */ +function physicalToDip(x: number, y: number): [number, number] { + try { + if (screen && typeof screen.screenToDipPoint === "function") { + const dip = screen.screenToDipPoint({ x, y }); + return [Math.round(dip.x), Math.round(dip.y)]; + } + } catch {} + return [Math.round(x), Math.round(y)]; +} + const moveMouse: CapabilityHandler = async (_ctx, node, args) => { // 注意：position 字段是「坐标相对位置」(0=屏幕左上角/1=激活窗口左上角)，坐标本体在 location const [rawX, rawY] = parsePoint(args.location ?? args.point, "鼠标坐标", node.id, node.componentId); - const x = Math.round(rawX); - const y = Math.round(rawY); const relative = String(args.position ?? "0") === "1"; const moveType = String(args.move_type ?? "0"); const speed = Math.min(100, Math.max(1, Number(args.speed ?? 35) || 35)); + let dipTarget: [number, number]; if (relative) { - // 相对激活窗口左上角：取前台窗口位置做偏移 + // 相对激活窗口左上角：窗口矩形是物理像素，换回 DIP 再叠加偏移，最后统一转物理 const output = await powershell(`${WINDOW_API}; $fw = [RpaWindowExtended]::GetForegroundWindow(); $r = New-Object RpaWindowExtended+RECT; [void][RpaWindowExtended]::GetWindowRect($fw, [ref]$r); "$($r.Left),$($r.Top)"`); const [wx, wy] = output.split(",").map(Number); if (!Number.isFinite(wx) || !Number.isFinite(wy)) throw new RuntimeError("browser_error", "无法获取激活窗口位置", { nodeId: node.id, componentId: node.componentId }); - return moveToSmooth([x + wx, y + wy], moveType, speed, node.id, node.componentId); + const [dipWx, dipWy] = physicalToDip(wx, wy); + dipTarget = [dipWx + rawX, dipWy + rawY]; + } else { + dipTarget = [rawX, rawY]; } - return moveToSmooth([x, y], moveType, speed, node.id, node.componentId); + return moveToSmooth(dipTarget, moveType, speed); }; -async function moveToSmooth(target: [number, number], moveType: string, speed: number, nodeId: string, componentId: string): Promise<{ x: number; y: number }> { +async function moveToSmooth(dipTarget: [number, number], moveType: string, speed: number): Promise<{ x: number; y: number }> { + const [tx, ty] = dipToPhysical(dipTarget[0], dipTarget[1]); if (moveType === "2") { - await powershell(`${WINDOW_API}; [void][RpaWindowExtended]::SetCursorPos(${target[0]},${target[1]})`); - return { x: target[0], y: target[1] }; + await powershell(`${WINDOW_API}; [void][RpaWindowExtended]::SetCursorPos(${tx},${ty})`); + return { x: Math.round(dipTarget[0]), y: Math.round(dipTarget[1]) }; } - // 平滑移动：从当前光标位置出发，按速度插值 + // 平滑移动：从当前光标位置出发，按速度插值（路径全程物理像素，与目标同一坐标系） const delay = Math.min(80, Math.max(4, Math.round(1500 / speed))); const output = await powershell(`${WINDOW_API}; $cur = [RpaWindowExtended+POINT]::new(); [void][RpaWindowExtended]::GetCursorPos([ref]$cur); "$($cur.X),$($cur.Y)"`); const [cx, cy] = output.split(",").map(Number); const from: [number, number] = Number.isFinite(cx) && Number.isFinite(cy) ? [cx, cy] : [0, 0]; - const path = mousePath(from, target, moveType, speed); + const path = mousePath(from, [tx, ty], moveType, speed); const ptsLiteral = path.map(([px, py]) => `@(${px},${py})`).join(","); await powershell(`${WINDOW_API}; $pts = @(${ptsLiteral}); foreach ($pt in $pts) { [void][RpaWindowExtended]::SetCursorPos($pt[0], $pt[1]); Start-Sleep -Milliseconds ${delay} }`); - return { x: target[0], y: target[1] }; + return { x: Math.round(dipTarget[0]), y: Math.round(dipTarget[1]) }; } register(["Mouse_MoveToV1", "Mouse_MoveToV2"], moveMouse); const dragMouse: CapabilityHandler = async (_ctx, node, args) => { const [sx, sy] = parsePoint(args.Start ?? args.start, "起始坐标", node.id, node.componentId); const [ex, ey] = parsePoint(args.End ?? args.end, "结束坐标", node.id, node.componentId); + const [spx, spy] = dipToPhysical(sx, sy); + const [epx, epy] = dipToPhysical(ex, ey); const right = ["1", "right"].includes(value(args, ["side"], "0").toLowerCase()); const down = right ? 8 : 2; const up = right ? 16 : 4; - await powershell(`${WINDOW_API}; [RpaWindowExtended]::SetCursorPos(${sx},${sy}) | Out-Null; [RpaWindowExtended]::mouse_event(${down},0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 80; [RpaWindowExtended]::SetCursorPos(${ex},${ey}) | Out-Null; [RpaWindowExtended]::mouse_event(${up},0,0,0,[UIntPtr]::Zero)`); - return { start: [sx, sy], end: [ex, ey] }; + await powershell(`${WINDOW_API}; [RpaWindowExtended]::SetCursorPos(${spx},${spy}) | Out-Null; [RpaWindowExtended]::mouse_event(${down},0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 80; [RpaWindowExtended]::SetCursorPos(${epx},${epy}) | Out-Null; [RpaWindowExtended]::mouse_event(${up},0,0,0,[UIntPtr]::Zero)`); + return { start: [Math.round(sx), Math.round(sy)], end: [Math.round(ex), Math.round(ey)] }; }; register(["Mouse_DragToV1", "Mouse_DragToV2", "Mouse_DragToNew"], dragMouse); register(["Mouse_ClickCombine", "Mouse_ClickCombine_v1", "Mouse_ClickCombine_v2", "Mouse_ClickCombine_v3", "Mouse_ClickCombine_v4"], async (_ctx, node, args) => { if (value(args, ["target"], "0") !== "0") throw new RuntimeError("validation_error", "组合点击当前仅支持坐标目标", { nodeId: node.id, componentId: node.componentId }); const [x, y] = parsePoint(args.location, "点击坐标", node.id, node.componentId); + const [px, py] = dipToPhysical(x, y); const right = value(args, ["click_key"], "left").toLowerCase() === "right"; const doubleClick = value(args, ["click_time"], "click").toLowerCase().includes("double"); const down = right ? 8 : 2; const up = right ? 16 : 4; const repeats = doubleClick ? 2 : 1; - await powershell(`${WINDOW_API}; [RpaWindowExtended]::SetCursorPos(${x},${y}) | Out-Null; 1..${repeats} | ForEach-Object { [RpaWindowExtended]::mouse_event(${down},0,0,0,[UIntPtr]::Zero); [RpaWindowExtended]::mouse_event(${up},0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 80 }`); - return { x, y, button: right ? "right" : "left", clicks: repeats }; + await powershell(`${WINDOW_API}; [RpaWindowExtended]::SetCursorPos(${px},${py}) | Out-Null; 1..${repeats} | ForEach-Object { [RpaWindowExtended]::mouse_event(${down},0,0,0,[UIntPtr]::Zero); [RpaWindowExtended]::mouse_event(${up},0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 80 }`); + return { x: Math.round(x), y: Math.round(y), button: right ? "right" : "left", clicks: repeats }; });