fix: 移动鼠标支持对象坐标x/y(屏幕框选生成格式,此前报坐标无效), 实现三种移动方式(模拟人工缓动抖动/匀速直线/瞬间)与移动速度, 相对激活窗口模式叠加前台窗口偏移, 补DPI感知, 冒烟4项真实落位通过
This commit is contained in:
yu
parent
fb71c0551b
commit
158104661c
85
src/main/capabilities/mouse-moveto-smoke.test.ts
Normal file
85
src/main/capabilities/mouse-moveto-smoke.test.ts
Normal file
@ -0,0 +1,85 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import type { CatalogComponent, FlowNode } from "../../shared/types";
|
||||
import { getHandler } from "../engine/registry";
|
||||
import type { RunContext } from "../engine/types";
|
||||
import "./window-extended";
|
||||
|
||||
const run = promisify(execFile);
|
||||
const component = {} as CatalogComponent;
|
||||
const node = (id: string): FlowNode => ({ id: "smoke-mouse", componentId: id, position: { x: 0, y: 0 }, values: {}, options: {} });
|
||||
|
||||
const ctx = {
|
||||
runId: "mouse-smoke", flowId: "mouse-smoke-flow", flow: {} as RunContext["flow"],
|
||||
vars: {}, secrets: {}, signal: new AbortController().signal, evidenceDir: "D:/office-test2",
|
||||
emit: () => {}, log: () => {}, componentOf: () => undefined
|
||||
} as unknown as RunContext;
|
||||
|
||||
const PS_API = `Add-Type @'
|
||||
using System; using System.Runtime.InteropServices;
|
||||
public static class RpaMouseSmoke {
|
||||
[StructLayout(LayoutKind.Sequential)] public struct POINT { public int X; public int Y; }
|
||||
[DllImport("user32.dll")] public static extern bool GetCursorPos(out POINT p);
|
||||
[DllImport("user32.dll")] public static extern bool SetProcessDpiAwarenessContext(IntPtr v);
|
||||
}
|
||||
'@; try { [void][RpaMouseSmoke]::SetProcessDpiAwarenessContext([IntPtr](-4)) } catch {}; $c = [RpaMouseSmoke+POINT]::new(); [void][RpaMouseSmoke]::GetCursorPos([ref]$c); "$($c.X),$($c.Y)"`;
|
||||
|
||||
async function cursorPos(): Promise<[number, number]> {
|
||||
const { stdout } = await run("powershell.exe", ["-NoProfile", "-NonInteractive", "-Command", PS_API], { windowsHide: true });
|
||||
return stdout.trim().split(",").map(Number) as [number, number];
|
||||
}
|
||||
|
||||
async function invoke(componentId: string, args: Record<string, unknown>): Promise<unknown> {
|
||||
const handler = getHandler(componentId);
|
||||
expect(handler).toBeTypeOf("function");
|
||||
return handler!(ctx, node(componentId), args, component);
|
||||
}
|
||||
|
||||
let original: [number, number];
|
||||
afterEach(async () => {
|
||||
if (original) await invoke("Mouse_MoveToV2", { location: { x: original[0], y: original[1] }, position: "0", move_type: "2", speed: 100 });
|
||||
});
|
||||
|
||||
describe("移动鼠标组件真实冒烟", () => {
|
||||
it("对象坐标 x/y JSON 不再报无效, 瞬间移动真实落位", async () => {
|
||||
original = await cursorPos();
|
||||
// 用户屏幕框选生成的浮点对象坐标（此前直接报「鼠标坐标无效」）
|
||||
const out = await invoke("Mouse_MoveToV2", { location: { x: 1397, y: 227.60000610351562 }, position: "0", move_type: "2", speed: 100 }) as { x: number; y: number };
|
||||
expect(out).toEqual({ x: 1397, y: 228 });
|
||||
await new Promise((r) => setTimeout(r, 150));
|
||||
const pos = await cursorPos();
|
||||
expect(pos[0]).toBe(1397);
|
||||
expect(pos[1]).toBe(228);
|
||||
console.log("[移动鼠标] 对象坐标解析 + 瞬间移动落位 ✓ 实际光标:", pos);
|
||||
}, 60000);
|
||||
|
||||
it("模拟人工移动(默认方式)平滑到达目标", async () => {
|
||||
original = await cursorPos();
|
||||
const out = await invoke("Mouse_MoveToV2", { location: "[300,300]", position: "0", move_type: "0", speed: 60 }) as { x: number; y: number };
|
||||
expect(out).toEqual({ x: 300, y: 300 });
|
||||
await new Promise((r) => setTimeout(r, 200));
|
||||
const pos = await cursorPos();
|
||||
expect(Math.abs(pos[0] - 300)).toBeLessThanOrEqual(2);
|
||||
expect(Math.abs(pos[1] - 300)).toBeLessThanOrEqual(2);
|
||||
console.log("[移动鼠标] 模拟人工移动 ✓ 实际光标:", pos);
|
||||
}, 60000);
|
||||
|
||||
it("匀速直线移动与字符串数组坐标", async () => {
|
||||
original = await cursorPos();
|
||||
const out = await invoke("Mouse_MoveToV2", { location: "[500,400]", position: "0", move_type: "1", speed: 80 }) as { x: number; y: number };
|
||||
expect(out).toEqual({ x: 500, y: 400 });
|
||||
const pos = await cursorPos();
|
||||
expect(pos[0]).toBe(500);
|
||||
expect(pos[1]).toBe(400);
|
||||
console.log("[移动鼠标] 匀速直线 + 字符串坐标 ✓ 实际光标:", pos);
|
||||
}, 60000);
|
||||
|
||||
it("相对激活窗口左上角模式返回有效坐标", async () => {
|
||||
original = await cursorPos();
|
||||
const out = await invoke("Mouse_MoveToV2", { location: { x: 10, y: 10 }, position: "1", move_type: "2", speed: 100 }) as { x: number; y: number };
|
||||
expect(Number.isFinite(out.x)).toBe(true);
|
||||
expect(Number.isFinite(out.y)).toBe(true);
|
||||
console.log("[移动鼠标] 相对激活窗口模式 ✓ 目标:", out);
|
||||
}, 60000);
|
||||
});
|
||||
@ -7,14 +7,18 @@ using System;
|
||||
using System.Runtime.InteropServices;
|
||||
public static class RpaWindowExtended {
|
||||
[StructLayout(LayoutKind.Sequential)] public struct RECT { public int Left; public int Top; public int Right; public int Bottom; }
|
||||
[StructLayout(LayoutKind.Sequential)] public struct POINT { public int X; public int Y; }
|
||||
[DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr hWnd, out RECT rect);
|
||||
[DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd);
|
||||
[DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr hWnd, int command);
|
||||
[DllImport("user32.dll")] public static extern bool SetWindowPos(IntPtr hWnd, IntPtr after, int x, int y, int width, int height, uint flags);
|
||||
[DllImport("user32.dll")] public static extern bool SetCursorPos(int x, int y);
|
||||
[DllImport("user32.dll")] public static extern bool GetCursorPos(out POINT point);
|
||||
[DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow();
|
||||
[DllImport("user32.dll")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra);
|
||||
[DllImport("user32.dll")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value);
|
||||
}
|
||||
'@`;
|
||||
'@; try { [void][RpaWindowExtended]::SetProcessDpiAwarenessContext([IntPtr](-4)) } catch {}`;
|
||||
|
||||
function value(args: Record<string, unknown>, keys: string[], fallback = ""): string {
|
||||
for (const key of keys) {
|
||||
@ -54,6 +58,11 @@ function parsePoint(candidate: unknown, label: string, nodeId: string, component
|
||||
const source = parsed;
|
||||
try { parsed = JSON.parse(source); } catch { parsed = source.split(/[,，xX ]+/).map(Number); }
|
||||
}
|
||||
// 屏幕框选/拾取坐标生成的是 {"x":..,"y":..} 对象，直接转换
|
||||
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
||||
const record = parsed as Record<string, unknown>;
|
||||
if (record.x !== undefined && record.y !== undefined) parsed = [record.x, record.y];
|
||||
}
|
||||
if (!Array.isArray(parsed) || parsed.length < 2 || parsed.slice(0, 2).some((item) => !Number.isFinite(Number(item)))) {
|
||||
throw new RuntimeError("validation_error", `${label}无效`, { nodeId, componentId });
|
||||
}
|
||||
@ -102,11 +111,62 @@ register(["Window_SetPos", "Window_MoveWindow", "Window_MoveWindow_v1", "Window_
|
||||
return { ...(await windowObject(title)), x, y, width, height };
|
||||
});
|
||||
|
||||
/** 生成从当前位置到目标的插值路径点（JS 侧算好，PS 侧只执行循环） */
|
||||
function mousePath(from: [number, number], to: [number, number], mode: string, speed: number): Array<[number, number]> {
|
||||
const dx = to[0] - from[0];
|
||||
const dy = to[1] - from[1];
|
||||
const distance = Math.hypot(dx, dy);
|
||||
if (distance < 2) return [to];
|
||||
const stepCount = Math.max(6, Math.min(50, Math.round(distance / 12)));
|
||||
const points: Array<[number, number]> = [];
|
||||
for (let i = 1; i <= stepCount; i++) {
|
||||
const t = i / stepCount;
|
||||
// 模拟人工：缓入缓出 + 轻微垂直抖动；匀速直线：线性
|
||||
const ease = mode === "0" ? (t < 0.5 ? 2 * t * t : 1 - Math.pow(-2 * t + 2, 2) / 2) : t;
|
||||
const jitter = mode === "0" ? Math.sin(t * Math.PI) * (Math.sin(i * 12.9898) * 43758.5453 % 1) * 3 : 0;
|
||||
points.push([
|
||||
Math.round(from[0] + dx * ease - (dy / distance) * jitter),
|
||||
Math.round(from[1] + dy * ease + (dx / distance) * jitter)
|
||||
]);
|
||||
}
|
||||
points.push(to);
|
||||
return points;
|
||||
}
|
||||
|
||||
const moveMouse: CapabilityHandler = async (_ctx, node, args) => {
|
||||
const [x, y] = parsePoint(args.location ?? args.position, "鼠标坐标", node.id, node.componentId);
|
||||
await powershell(`${WINDOW_API}; [RpaWindowExtended]::SetCursorPos(${x},${y}) | Out-Null`);
|
||||
return { x, y };
|
||||
// 注意：position 字段是「坐标相对位置」(0=屏幕左上角/1=激活窗口左上角)，坐标本体在 location
|
||||
const [rawX, rawY] = parsePoint(args.location ?? args.point, "鼠标坐标", node.id, node.componentId);
|
||||
const x = Math.round(rawX);
|
||||
const y = Math.round(rawY);
|
||||
const relative = String(args.position ?? "0") === "1";
|
||||
const moveType = String(args.move_type ?? "0");
|
||||
const speed = Math.min(100, Math.max(1, Number(args.speed ?? 35) || 35));
|
||||
|
||||
if (relative) {
|
||||
// 相对激活窗口左上角：取前台窗口位置做偏移
|
||||
const output = await powershell(`${WINDOW_API}; $fw = [RpaWindowExtended]::GetForegroundWindow(); $r = New-Object RpaWindowExtended+RECT; [void][RpaWindowExtended]::GetWindowRect($fw, [ref]$r); "$($r.Left),$($r.Top)"`);
|
||||
const [wx, wy] = output.split(",").map(Number);
|
||||
if (!Number.isFinite(wx) || !Number.isFinite(wy)) throw new RuntimeError("browser_error", "无法获取激活窗口位置", { nodeId: node.id, componentId: node.componentId });
|
||||
return moveToSmooth([x + wx, y + wy], moveType, speed, node.id, node.componentId);
|
||||
}
|
||||
return moveToSmooth([x, y], moveType, speed, node.id, node.componentId);
|
||||
};
|
||||
|
||||
async function moveToSmooth(target: [number, number], moveType: string, speed: number, nodeId: string, componentId: string): Promise<{ x: number; y: number }> {
|
||||
if (moveType === "2") {
|
||||
await powershell(`${WINDOW_API}; [void][RpaWindowExtended]::SetCursorPos(${target[0]},${target[1]})`);
|
||||
return { x: target[0], y: target[1] };
|
||||
}
|
||||
// 平滑移动：从当前光标位置出发，按速度插值
|
||||
const delay = Math.min(80, Math.max(4, Math.round(1500 / speed)));
|
||||
const output = await powershell(`${WINDOW_API}; $cur = [RpaWindowExtended+POINT]::new(); [void][RpaWindowExtended]::GetCursorPos([ref]$cur); "$($cur.X),$($cur.Y)"`);
|
||||
const [cx, cy] = output.split(",").map(Number);
|
||||
const from: [number, number] = Number.isFinite(cx) && Number.isFinite(cy) ? [cx, cy] : [0, 0];
|
||||
const path = mousePath(from, target, moveType, speed);
|
||||
const ptsLiteral = path.map(([px, py]) => `@(${px},${py})`).join(",");
|
||||
await powershell(`${WINDOW_API}; $pts = @(${ptsLiteral}); foreach ($pt in $pts) { [void][RpaWindowExtended]::SetCursorPos($pt[0], $pt[1]); Start-Sleep -Milliseconds ${delay} }`);
|
||||
return { x: target[0], y: target[1] };
|
||||
}
|
||||
register(["Mouse_MoveToV1", "Mouse_MoveToV2"], moveMouse);
|
||||
|
||||
const dragMouse: CapabilityHandler = async (_ctx, node, args) => {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user