diff --git a/src/main/capabilities/mouse-moveto-smoke.test.ts b/src/main/capabilities/mouse-moveto-smoke.test.ts new file mode 100644 index 0000000..2495d6e --- /dev/null +++ b/src/main/capabilities/mouse-moveto-smoke.test.ts @@ -0,0 +1,85 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { afterEach, describe, expect, it } from "vitest"; +import type { CatalogComponent, FlowNode } from "../../shared/types"; +import { getHandler } from "../engine/registry"; +import type { RunContext } from "../engine/types"; +import "./window-extended"; + +const run = promisify(execFile); +const component = {} as CatalogComponent; +const node = (id: string): FlowNode => ({ id: "smoke-mouse", componentId: id, position: { x: 0, y: 0 }, values: {}, options: {} }); + +const ctx = { + runId: "mouse-smoke", flowId: "mouse-smoke-flow", flow: {} as RunContext["flow"], + vars: {}, secrets: {}, signal: new AbortController().signal, evidenceDir: "D:/office-test2", + emit: () => {}, log: () => {}, componentOf: () => undefined +} as unknown as RunContext; + +const PS_API = `Add-Type @' +using System; using System.Runtime.InteropServices; +public static class RpaMouseSmoke { + [StructLayout(LayoutKind.Sequential)] public struct POINT { public int X; public int Y; } + [DllImport("user32.dll")] public static extern bool GetCursorPos(out POINT p); + [DllImport("user32.dll")] public static extern bool SetProcessDpiAwarenessContext(IntPtr v); +} +'@; try { [void][RpaMouseSmoke]::SetProcessDpiAwarenessContext([IntPtr](-4)) } catch {}; $c = [RpaMouseSmoke+POINT]::new(); [void][RpaMouseSmoke]::GetCursorPos([ref]$c); "$($c.X),$($c.Y)"`; + +async function cursorPos(): Promise<[number, number]> { + const { stdout } = await run("powershell.exe", ["-NoProfile", "-NonInteractive", "-Command", PS_API], { windowsHide: true }); + return stdout.trim().split(",").map(Number) as [number, number]; +} + +async function invoke(componentId: string, args: Record): Promise { + const handler = getHandler(componentId); + expect(handler).toBeTypeOf("function"); + return handler!(ctx, node(componentId), args, component); +} + +let original: [number, number]; +afterEach(async () => { + if (original) await invoke("Mouse_MoveToV2", { location: { x: original[0], y: original[1] }, position: "0", move_type: "2", speed: 100 }); +}); + +describe("移动鼠标组件真实冒烟", () => { + it("对象坐标 x/y JSON 不再报无效, 瞬间移动真实落位", async () => { + original = await cursorPos(); + // 用户屏幕框选生成的浮点对象坐标(此前直接报「鼠标坐标无效」) + const out = await invoke("Mouse_MoveToV2", { location: { x: 1397, y: 227.60000610351562 }, position: "0", move_type: "2", speed: 100 }) as { x: number; y: number }; + expect(out).toEqual({ x: 1397, y: 228 }); + await new Promise((r) => setTimeout(r, 150)); + const pos = await cursorPos(); + expect(pos[0]).toBe(1397); + expect(pos[1]).toBe(228); + console.log("[移动鼠标] 对象坐标解析 + 瞬间移动落位 ✓ 实际光标:", pos); + }, 60000); + + it("模拟人工移动(默认方式)平滑到达目标", async () => { + original = await cursorPos(); + const out = await invoke("Mouse_MoveToV2", { location: "[300,300]", position: "0", move_type: "0", speed: 60 }) as { x: number; y: number }; + expect(out).toEqual({ x: 300, y: 300 }); + await new Promise((r) => setTimeout(r, 200)); + const pos = await cursorPos(); + expect(Math.abs(pos[0] - 300)).toBeLessThanOrEqual(2); + expect(Math.abs(pos[1] - 300)).toBeLessThanOrEqual(2); + console.log("[移动鼠标] 模拟人工移动 ✓ 实际光标:", pos); + }, 60000); + + it("匀速直线移动与字符串数组坐标", async () => { + original = await cursorPos(); + const out = await invoke("Mouse_MoveToV2", { location: "[500,400]", position: "0", move_type: "1", speed: 80 }) as { x: number; y: number }; + expect(out).toEqual({ x: 500, y: 400 }); + const pos = await cursorPos(); + expect(pos[0]).toBe(500); + expect(pos[1]).toBe(400); + console.log("[移动鼠标] 匀速直线 + 字符串坐标 ✓ 实际光标:", pos); + }, 60000); + + it("相对激活窗口左上角模式返回有效坐标", async () => { + original = await cursorPos(); + const out = await invoke("Mouse_MoveToV2", { location: { x: 10, y: 10 }, position: "1", move_type: "2", speed: 100 }) as { x: number; y: number }; + expect(Number.isFinite(out.x)).toBe(true); + expect(Number.isFinite(out.y)).toBe(true); + console.log("[移动鼠标] 相对激活窗口模式 ✓ 目标:", out); + }, 60000); +}); diff --git a/src/main/capabilities/window-extended.ts b/src/main/capabilities/window-extended.ts index a74084a..cb4e998 100644 --- a/src/main/capabilities/window-extended.ts +++ b/src/main/capabilities/window-extended.ts @@ -7,14 +7,18 @@ using System; using System.Runtime.InteropServices; public static class RpaWindowExtended { [StructLayout(LayoutKind.Sequential)] public struct RECT { public int Left; public int Top; public int Right; public int Bottom; } + [StructLayout(LayoutKind.Sequential)] public struct POINT { public int X; public int Y; } [DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr hWnd, out RECT rect); [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd); [DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr hWnd, int command); [DllImport("user32.dll")] public static extern bool SetWindowPos(IntPtr hWnd, IntPtr after, int x, int y, int width, int height, uint flags); [DllImport("user32.dll")] public static extern bool SetCursorPos(int x, int y); + [DllImport("user32.dll")] public static extern bool GetCursorPos(out POINT point); + [DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); [DllImport("user32.dll")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra); + [DllImport("user32.dll")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value); } -'@`; +'@; try { [void][RpaWindowExtended]::SetProcessDpiAwarenessContext([IntPtr](-4)) } catch {}`; function value(args: Record, keys: string[], fallback = ""): string { for (const key of keys) { @@ -54,6 +58,11 @@ function parsePoint(candidate: unknown, label: string, nodeId: string, component const source = parsed; try { parsed = JSON.parse(source); } catch { parsed = source.split(/[,,xX ]+/).map(Number); } } + // 屏幕框选/拾取坐标生成的是 {"x":..,"y":..} 对象,直接转换 + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { + const record = parsed as Record; + if (record.x !== undefined && record.y !== undefined) parsed = [record.x, record.y]; + } if (!Array.isArray(parsed) || parsed.length < 2 || parsed.slice(0, 2).some((item) => !Number.isFinite(Number(item)))) { throw new RuntimeError("validation_error", `${label}无效`, { nodeId, componentId }); } @@ -102,11 +111,62 @@ register(["Window_SetPos", "Window_MoveWindow", "Window_MoveWindow_v1", "Window_ return { ...(await windowObject(title)), x, y, width, height }; }); +/** 生成从当前位置到目标的插值路径点(JS 侧算好,PS 侧只执行循环) */ +function mousePath(from: [number, number], to: [number, number], mode: string, speed: number): Array<[number, number]> { + const dx = to[0] - from[0]; + const dy = to[1] - from[1]; + const distance = Math.hypot(dx, dy); + if (distance < 2) return [to]; + const stepCount = Math.max(6, Math.min(50, Math.round(distance / 12))); + const points: Array<[number, number]> = []; + for (let i = 1; i <= stepCount; i++) { + const t = i / stepCount; + // 模拟人工:缓入缓出 + 轻微垂直抖动;匀速直线:线性 + const ease = mode === "0" ? (t < 0.5 ? 2 * t * t : 1 - Math.pow(-2 * t + 2, 2) / 2) : t; + const jitter = mode === "0" ? Math.sin(t * Math.PI) * (Math.sin(i * 12.9898) * 43758.5453 % 1) * 3 : 0; + points.push([ + Math.round(from[0] + dx * ease - (dy / distance) * jitter), + Math.round(from[1] + dy * ease + (dx / distance) * jitter) + ]); + } + points.push(to); + return points; +} + const moveMouse: CapabilityHandler = async (_ctx, node, args) => { - const [x, y] = parsePoint(args.location ?? args.position, "鼠标坐标", node.id, node.componentId); - await powershell(`${WINDOW_API}; [RpaWindowExtended]::SetCursorPos(${x},${y}) | Out-Null`); - return { x, y }; + // 注意:position 字段是「坐标相对位置」(0=屏幕左上角/1=激活窗口左上角),坐标本体在 location + const [rawX, rawY] = parsePoint(args.location ?? args.point, "鼠标坐标", node.id, node.componentId); + const x = Math.round(rawX); + const y = Math.round(rawY); + const relative = String(args.position ?? "0") === "1"; + const moveType = String(args.move_type ?? "0"); + const speed = Math.min(100, Math.max(1, Number(args.speed ?? 35) || 35)); + + if (relative) { + // 相对激活窗口左上角:取前台窗口位置做偏移 + const output = await powershell(`${WINDOW_API}; $fw = [RpaWindowExtended]::GetForegroundWindow(); $r = New-Object RpaWindowExtended+RECT; [void][RpaWindowExtended]::GetWindowRect($fw, [ref]$r); "$($r.Left),$($r.Top)"`); + const [wx, wy] = output.split(",").map(Number); + if (!Number.isFinite(wx) || !Number.isFinite(wy)) throw new RuntimeError("browser_error", "无法获取激活窗口位置", { nodeId: node.id, componentId: node.componentId }); + return moveToSmooth([x + wx, y + wy], moveType, speed, node.id, node.componentId); + } + return moveToSmooth([x, y], moveType, speed, node.id, node.componentId); }; + +async function moveToSmooth(target: [number, number], moveType: string, speed: number, nodeId: string, componentId: string): Promise<{ x: number; y: number }> { + if (moveType === "2") { + await powershell(`${WINDOW_API}; [void][RpaWindowExtended]::SetCursorPos(${target[0]},${target[1]})`); + return { x: target[0], y: target[1] }; + } + // 平滑移动:从当前光标位置出发,按速度插值 + const delay = Math.min(80, Math.max(4, Math.round(1500 / speed))); + const output = await powershell(`${WINDOW_API}; $cur = [RpaWindowExtended+POINT]::new(); [void][RpaWindowExtended]::GetCursorPos([ref]$cur); "$($cur.X),$($cur.Y)"`); + const [cx, cy] = output.split(",").map(Number); + const from: [number, number] = Number.isFinite(cx) && Number.isFinite(cy) ? [cx, cy] : [0, 0]; + const path = mousePath(from, target, moveType, speed); + const ptsLiteral = path.map(([px, py]) => `@(${px},${py})`).join(","); + await powershell(`${WINDOW_API}; $pts = @(${ptsLiteral}); foreach ($pt in $pts) { [void][RpaWindowExtended]::SetCursorPos($pt[0], $pt[1]); Start-Sleep -Milliseconds ${delay} }`); + return { x: target[0], y: target[1] }; +} register(["Mouse_MoveToV1", "Mouse_MoveToV2"], moveMouse); const dragMouse: CapabilityHandler = async (_ctx, node, args) => {