鼠标组件坐标修正: DIP 转物理像素修复缩放屏偏移
This commit is contained in:
yu
parent
158104661c
commit
6b05a65664
@ -1,6 +1,7 @@
|
||||
import { register } from "../engine/registry";
|
||||
import { RuntimeError, type CapabilityHandler } from "../engine/types";
|
||||
import { powershell, psString } from "./desktop";
|
||||
import { screen } from "electron";
|
||||
|
||||
const WINDOW_API = `Add-Type @'
|
||||
using System;
|
||||
@ -133,61 +134,91 @@ function mousePath(from: [number, number], to: [number, number], mode: string, s
|
||||
return points;
|
||||
}
|
||||
|
||||
/** 界面(框选/元素拾取/截图)产出的是 DIP 逻辑坐标；DPI 感知进程的 SetCursorPos 用物理像素，
|
||||
* 缩放屏(125%/150%)下直接混用会成比例偏移(用户实测 125% 下差约 25% 距离)。
|
||||
* 统一在执行前换算；非 Electron 环境(vitest)screen 不可用，回退原值保持旧行为 */
|
||||
function dipToPhysical(x: number, y: number): [number, number] {
|
||||
try {
|
||||
if (screen && typeof screen.dipToScreenPoint === "function") {
|
||||
const physical = screen.dipToScreenPoint({ x, y });
|
||||
return [Math.round(physical.x), Math.round(physical.y)];
|
||||
}
|
||||
} catch {}
|
||||
return [Math.round(x), Math.round(y)];
|
||||
}
|
||||
|
||||
/** 反向换算：物理像素(如 GetWindowRect 结果) → DIP，用于「相对激活窗口」偏移叠加 */
|
||||
function physicalToDip(x: number, y: number): [number, number] {
|
||||
try {
|
||||
if (screen && typeof screen.screenToDipPoint === "function") {
|
||||
const dip = screen.screenToDipPoint({ x, y });
|
||||
return [Math.round(dip.x), Math.round(dip.y)];
|
||||
}
|
||||
} catch {}
|
||||
return [Math.round(x), Math.round(y)];
|
||||
}
|
||||
|
||||
const moveMouse: CapabilityHandler = async (_ctx, node, args) => {
|
||||
// 注意：position 字段是「坐标相对位置」(0=屏幕左上角/1=激活窗口左上角)，坐标本体在 location
|
||||
const [rawX, rawY] = parsePoint(args.location ?? args.point, "鼠标坐标", node.id, node.componentId);
|
||||
const x = Math.round(rawX);
|
||||
const y = Math.round(rawY);
|
||||
const relative = String(args.position ?? "0") === "1";
|
||||
const moveType = String(args.move_type ?? "0");
|
||||
const speed = Math.min(100, Math.max(1, Number(args.speed ?? 35) || 35));
|
||||
|
||||
let dipTarget: [number, number];
|
||||
if (relative) {
|
||||
// 相对激活窗口左上角：取前台窗口位置做偏移
|
||||
// 相对激活窗口左上角：窗口矩形是物理像素，换回 DIP 再叠加偏移，最后统一转物理
|
||||
const output = await powershell(`${WINDOW_API}; $fw = [RpaWindowExtended]::GetForegroundWindow(); $r = New-Object RpaWindowExtended+RECT; [void][RpaWindowExtended]::GetWindowRect($fw, [ref]$r); "$($r.Left),$($r.Top)"`);
|
||||
const [wx, wy] = output.split(",").map(Number);
|
||||
if (!Number.isFinite(wx) || !Number.isFinite(wy)) throw new RuntimeError("browser_error", "无法获取激活窗口位置", { nodeId: node.id, componentId: node.componentId });
|
||||
return moveToSmooth([x + wx, y + wy], moveType, speed, node.id, node.componentId);
|
||||
const [dipWx, dipWy] = physicalToDip(wx, wy);
|
||||
dipTarget = [dipWx + rawX, dipWy + rawY];
|
||||
} else {
|
||||
dipTarget = [rawX, rawY];
|
||||
}
|
||||
return moveToSmooth([x, y], moveType, speed, node.id, node.componentId);
|
||||
return moveToSmooth(dipTarget, moveType, speed);
|
||||
};
|
||||
|
||||
async function moveToSmooth(target: [number, number], moveType: string, speed: number, nodeId: string, componentId: string): Promise<{ x: number; y: number }> {
|
||||
async function moveToSmooth(dipTarget: [number, number], moveType: string, speed: number): Promise<{ x: number; y: number }> {
|
||||
const [tx, ty] = dipToPhysical(dipTarget[0], dipTarget[1]);
|
||||
if (moveType === "2") {
|
||||
await powershell(`${WINDOW_API}; [void][RpaWindowExtended]::SetCursorPos(${target[0]},${target[1]})`);
|
||||
return { x: target[0], y: target[1] };
|
||||
await powershell(`${WINDOW_API}; [void][RpaWindowExtended]::SetCursorPos(${tx},${ty})`);
|
||||
return { x: Math.round(dipTarget[0]), y: Math.round(dipTarget[1]) };
|
||||
}
|
||||
// 平滑移动：从当前光标位置出发，按速度插值
|
||||
// 平滑移动：从当前光标位置出发，按速度插值（路径全程物理像素，与目标同一坐标系）
|
||||
const delay = Math.min(80, Math.max(4, Math.round(1500 / speed)));
|
||||
const output = await powershell(`${WINDOW_API}; $cur = [RpaWindowExtended+POINT]::new(); [void][RpaWindowExtended]::GetCursorPos([ref]$cur); "$($cur.X),$($cur.Y)"`);
|
||||
const [cx, cy] = output.split(",").map(Number);
|
||||
const from: [number, number] = Number.isFinite(cx) && Number.isFinite(cy) ? [cx, cy] : [0, 0];
|
||||
const path = mousePath(from, target, moveType, speed);
|
||||
const path = mousePath(from, [tx, ty], moveType, speed);
|
||||
const ptsLiteral = path.map(([px, py]) => `@(${px},${py})`).join(",");
|
||||
await powershell(`${WINDOW_API}; $pts = @(${ptsLiteral}); foreach ($pt in $pts) { [void][RpaWindowExtended]::SetCursorPos($pt[0], $pt[1]); Start-Sleep -Milliseconds ${delay} }`);
|
||||
return { x: target[0], y: target[1] };
|
||||
return { x: Math.round(dipTarget[0]), y: Math.round(dipTarget[1]) };
|
||||
}
|
||||
register(["Mouse_MoveToV1", "Mouse_MoveToV2"], moveMouse);
|
||||
|
||||
const dragMouse: CapabilityHandler = async (_ctx, node, args) => {
|
||||
const [sx, sy] = parsePoint(args.Start ?? args.start, "起始坐标", node.id, node.componentId);
|
||||
const [ex, ey] = parsePoint(args.End ?? args.end, "结束坐标", node.id, node.componentId);
|
||||
const [spx, spy] = dipToPhysical(sx, sy);
|
||||
const [epx, epy] = dipToPhysical(ex, ey);
|
||||
const right = ["1", "right"].includes(value(args, ["side"], "0").toLowerCase());
|
||||
const down = right ? 8 : 2;
|
||||
const up = right ? 16 : 4;
|
||||
await powershell(`${WINDOW_API}; [RpaWindowExtended]::SetCursorPos(${sx},${sy}) | Out-Null; [RpaWindowExtended]::mouse_event(${down},0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 80; [RpaWindowExtended]::SetCursorPos(${ex},${ey}) | Out-Null; [RpaWindowExtended]::mouse_event(${up},0,0,0,[UIntPtr]::Zero)`);
|
||||
return { start: [sx, sy], end: [ex, ey] };
|
||||
await powershell(`${WINDOW_API}; [RpaWindowExtended]::SetCursorPos(${spx},${spy}) | Out-Null; [RpaWindowExtended]::mouse_event(${down},0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 80; [RpaWindowExtended]::SetCursorPos(${epx},${epy}) | Out-Null; [RpaWindowExtended]::mouse_event(${up},0,0,0,[UIntPtr]::Zero)`);
|
||||
return { start: [Math.round(sx), Math.round(sy)], end: [Math.round(ex), Math.round(ey)] };
|
||||
};
|
||||
register(["Mouse_DragToV1", "Mouse_DragToV2", "Mouse_DragToNew"], dragMouse);
|
||||
|
||||
register(["Mouse_ClickCombine", "Mouse_ClickCombine_v1", "Mouse_ClickCombine_v2", "Mouse_ClickCombine_v3", "Mouse_ClickCombine_v4"], async (_ctx, node, args) => {
|
||||
if (value(args, ["target"], "0") !== "0") throw new RuntimeError("validation_error", "组合点击当前仅支持坐标目标", { nodeId: node.id, componentId: node.componentId });
|
||||
const [x, y] = parsePoint(args.location, "点击坐标", node.id, node.componentId);
|
||||
const [px, py] = dipToPhysical(x, y);
|
||||
const right = value(args, ["click_key"], "left").toLowerCase() === "right";
|
||||
const doubleClick = value(args, ["click_time"], "click").toLowerCase().includes("double");
|
||||
const down = right ? 8 : 2;
|
||||
const up = right ? 16 : 4;
|
||||
const repeats = doubleClick ? 2 : 1;
|
||||
await powershell(`${WINDOW_API}; [RpaWindowExtended]::SetCursorPos(${x},${y}) | Out-Null; 1..${repeats} | ForEach-Object { [RpaWindowExtended]::mouse_event(${down},0,0,0,[UIntPtr]::Zero); [RpaWindowExtended]::mouse_event(${up},0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 80 }`);
|
||||
return { x, y, button: right ? "right" : "left", clicks: repeats };
|
||||
await powershell(`${WINDOW_API}; [RpaWindowExtended]::SetCursorPos(${px},${py}) | Out-Null; 1..${repeats} | ForEach-Object { [RpaWindowExtended]::mouse_event(${down},0,0,0,[UIntPtr]::Zero); [RpaWindowExtended]::mouse_event(${up},0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 80 }`);
|
||||
return { x: Math.round(x), y: Math.round(y), button: right ? "right" : "left", clicks: repeats };
|
||||
});
|
||||
|
||||
Loading…
Reference in New Issue
Block a user