阶段3-driver抽象: powershell助手拆至driver/windows, 五域AutomationDriver接口+WindowsDriver(PowerShell实现行为不变), desktop/window-extended/system-extended/common-final改走driver, catalog构建填充platformTag=win32
这个提交包含在：
yu 编写， trae-solo-agent [origin=hook_stop] 于
1 父节点
3decced268
当前提交
e691b9fa78
22 个文件已更改
+673
-338
没有与搜索条件相匹配的文件
@@ -1,25 +1,10 @@
|
||||
import { register } from "../engine/registry";
|
||||
import { RuntimeError, type CapabilityHandler } from "../engine/types";
|
||||
import { powershell, psString } from "./desktop";
|
||||
import { getHostBridge } from "../engine/host-bridge";
|
||||
import { defaultEngine } from "../engine/engine";
|
||||
|
||||
const WINDOW_API = `Add-Type @'
|
||||
using System;
|
||||
using System.Runtime.InteropServices;
|
||||
public static class RpaWindowExtended {
|
||||
[StructLayout(LayoutKind.Sequential)] public struct RECT { public int Left; public int Top; public int Right; public int Bottom; }
|
||||
[StructLayout(LayoutKind.Sequential)] public struct POINT { public int X; public int Y; }
|
||||
[DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr hWnd, out RECT rect);
|
||||
[DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd);
|
||||
[DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr hWnd, int command);
|
||||
[DllImport("user32.dll")] public static extern bool SetWindowPos(IntPtr hWnd, IntPtr after, int x, int y, int width, int height, uint flags);
|
||||
[DllImport("user32.dll")] public static extern bool SetCursorPos(int x, int y);
|
||||
[DllImport("user32.dll")] public static extern bool GetCursorPos(out POINT point);
|
||||
[DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow();
|
||||
[DllImport("user32.dll")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra);
|
||||
[DllImport("user32.dll")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value);
|
||||
}
|
||||
'@; try { [void][RpaWindowExtended]::SetProcessDpiAwarenessContext([IntPtr](-4)) } catch {}`;
|
||||
// 桌面自动化驱动（任务 3.3）：窗口矩形/鼠标注入原语统一走 WindowsDriver
|
||||
const driver = defaultEngine.driver;
|
||||
|
||||
function value(args: Record<string, unknown>, keys: string[], fallback = ""): string {
|
||||
for (const key of keys) {
|
||||
@@ -49,10 +34,6 @@ function requiredTitle(args: Record<string, unknown>, nodeId: string, componentI
|
||||
return title;
|
||||
}
|
||||
|
||||
function findWindow(title: string): string {
|
||||
return `$p = Get-Process | Where-Object { $_.MainWindowHandle -ne 0 -and $_.MainWindowTitle -like ${psString(`*${title}*`)} } | Select-Object -First 1; if (-not $p) { throw '找不到窗口：${title.replace(/'/g, "''")}' }`;
|
||||
}
|
||||
|
||||
function parsePoint(candidate: unknown, label: string, nodeId: string, componentId: string): [number, number] {
|
||||
let parsed: unknown = candidate;
|
||||
if (typeof parsed === "string") {
|
||||
@@ -70,70 +51,6 @@ function parsePoint(candidate: unknown, label: string, nodeId: string, component
|
||||
return [Number(parsed[0]), Number(parsed[1])];
|
||||
}
|
||||
|
||||
async function windowObject(title: string): Promise<Record<string, unknown>> {
|
||||
const output = await powershell(`${WINDOW_API}; ${findWindow(title)}; $r = New-Object RpaWindowExtended+RECT; [RpaWindowExtended]::GetWindowRect($p.MainWindowHandle,[ref]$r) | Out-Null; @{pid=$p.Id;title=$p.MainWindowTitle;hwnd=$p.MainWindowHandle.ToInt64();x=$r.Left;y=$r.Top;width=$r.Right-$r.Left;height=$r.Bottom-$r.Top} | ConvertTo-Json -Compress`);
|
||||
return JSON.parse(output);
|
||||
}
|
||||
|
||||
const getWindow: CapabilityHandler = async (_ctx, node, args) => windowObject(requiredTitle(args, node.id, node.componentId));
|
||||
register(["Window_Get", "Window_GetWindow", "Window_GetWindow_v1"], getWindow);
|
||||
|
||||
register("Window_ActiveWindow_v1", async (_ctx, node, args) => {
|
||||
const title = requiredTitle(args, node.id, node.componentId);
|
||||
const maximize = [true, 1, "1", "true", "True"].includes(args.isMax as never);
|
||||
await powershell(`${WINDOW_API}; ${findWindow(title)}; [RpaWindowExtended]::SetForegroundWindow($p.MainWindowHandle) | Out-Null; ${maximize ? "[RpaWindowExtended]::ShowWindow($p.MainWindowHandle,3) | Out-Null" : ""}`);
|
||||
return windowObject(title);
|
||||
});
|
||||
|
||||
register(["Window_GetWindowMessage", "Window_GetWindowMessage_v1"], async (_ctx, node, args) => {
|
||||
const object = await windowObject(requiredTitle(args, node.id, node.componentId));
|
||||
const type = value(args, ["switchType"], "all").toLowerCase();
|
||||
if (type === "title") return object.title;
|
||||
if (type === "pid") return object.pid;
|
||||
if (type === "position" || type === "location") return [object.x, object.y];
|
||||
if (type === "size") return [object.width, object.height];
|
||||
return object;
|
||||
});
|
||||
|
||||
register(["Window_SetPos", "Window_MoveWindow", "Window_MoveWindow_v1", "Window_SizeWindow"], async (_ctx, node, args) => {
|
||||
const title = requiredTitle(args, node.id, node.componentId);
|
||||
const current = await windowObject(title);
|
||||
const moveOnly = node.componentId.includes("MoveWindow");
|
||||
const sizeOnly = node.componentId === "Window_SizeWindow";
|
||||
const position = args.location !== undefined ? parsePoint(args.location, "窗口位置", node.id, node.componentId) : [Number(args.x ?? current.x), Number(args.y ?? current.y)] as [number, number];
|
||||
const sizeCandidate = args.size ?? args.window_size;
|
||||
const size = sizeCandidate !== undefined ? parsePoint(sizeCandidate, "窗口尺寸", node.id, node.componentId) : [Number(args.width ?? current.width), Number(args.height ?? current.height)] as [number, number];
|
||||
const x = sizeOnly ? Number(current.x) : position[0];
|
||||
const y = sizeOnly ? Number(current.y) : position[1];
|
||||
const width = moveOnly ? Number(current.width) : size[0];
|
||||
const height = moveOnly ? Number(current.height) : size[1];
|
||||
if (!(width > 0 && height > 0)) throw new RuntimeError("validation_error", "窗口尺寸必须大于 0", { nodeId: node.id, componentId: node.componentId });
|
||||
await powershell(`${WINDOW_API}; ${findWindow(title)}; [RpaWindowExtended]::SetWindowPos($p.MainWindowHandle,[IntPtr]::Zero,${x},${y},${width},${height},4) | Out-Null`);
|
||||
return { ...(await windowObject(title)), x, y, width, height };
|
||||
});
|
||||
|
||||
/** 生成从当前位置到目标的插值路径点（JS 侧算好，PS 侧只执行循环） */
|
||||
function mousePath(from: [number, number], to: [number, number], mode: string, speed: number): Array<[number, number]> {
|
||||
const dx = to[0] - from[0];
|
||||
const dy = to[1] - from[1];
|
||||
const distance = Math.hypot(dx, dy);
|
||||
if (distance < 2) return [to];
|
||||
const stepCount = Math.max(6, Math.min(50, Math.round(distance / 12)));
|
||||
const points: Array<[number, number]> = [];
|
||||
for (let i = 1; i <= stepCount; i++) {
|
||||
const t = i / stepCount;
|
||||
// 模拟人工：缓入缓出 + 轻微垂直抖动；匀速直线：线性
|
||||
const ease = mode === "0" ? (t < 0.5 ? 2 * t * t : 1 - Math.pow(-2 * t + 2, 2) / 2) : t;
|
||||
const jitter = mode === "0" ? Math.sin(t * Math.PI) * (Math.sin(i * 12.9898) * 43758.5453 % 1) * 3 : 0;
|
||||
points.push([
|
||||
Math.round(from[0] + dx * ease - (dy / distance) * jitter),
|
||||
Math.round(from[1] + dy * ease + (dx / distance) * jitter)
|
||||
]);
|
||||
}
|
||||
points.push(to);
|
||||
return points;
|
||||
}
|
||||
|
||||
/** 界面(框选/元素拾取/截图)产出的是 DIP 逻辑坐标；DPI 感知进程的 SetCursorPos 用物理像素，
|
||||
* 缩放屏(125%/150%)下直接混用会成比例偏移(用户实测 125% 下差约 25% 距离)。
|
||||
* 统一在执行前换算；宿主桥未注入时(vitest)回退原值保持旧行为 */
|
||||
@@ -160,6 +77,41 @@ function physicalToDip(x: number, y: number): [number, number] {
|
||||
return [Math.round(x), Math.round(y)];
|
||||
}
|
||||
|
||||
const getWindow: CapabilityHandler = async (_ctx, node, args) => driver.window.findByTitle(requiredTitle(args, node.id, node.componentId));
|
||||
register(["Window_Get", "Window_GetWindow", "Window_GetWindow_v1"], getWindow);
|
||||
|
||||
register("Window_ActiveWindow_v1", async (_ctx, node, args) => {
|
||||
const title = requiredTitle(args, node.id, node.componentId);
|
||||
const maximize = [true, 1, "1", "true", "True"].includes(args.isMax as never);
|
||||
return driver.window.activate(title, { maximize });
|
||||
});
|
||||
|
||||
register(["Window_GetWindowMessage", "Window_GetWindowMessage_v1"], async (_ctx, node, args) => {
|
||||
const object = await driver.window.findByTitle(requiredTitle(args, node.id, node.componentId));
|
||||
const type = value(args, ["switchType"], "all").toLowerCase();
|
||||
if (type === "title") return object.title;
|
||||
if (type === "pid") return object.pid;
|
||||
if (type === "position" || type === "location") return [object.x, object.y];
|
||||
if (type === "size") return [object.width, object.height];
|
||||
return object;
|
||||
});
|
||||
|
||||
register(["Window_SetPos", "Window_MoveWindow", "Window_MoveWindow_v1", "Window_SizeWindow"], async (_ctx, node, args) => {
|
||||
const title = requiredTitle(args, node.id, node.componentId);
|
||||
const current = await driver.window.findByTitle(title);
|
||||
const moveOnly = node.componentId.includes("MoveWindow");
|
||||
const sizeOnly = node.componentId === "Window_SizeWindow";
|
||||
const position = args.location !== undefined ? parsePoint(args.location, "窗口位置", node.id, node.componentId) : [Number(args.x ?? current.x), Number(args.y ?? current.y)] as [number, number];
|
||||
const sizeCandidate = args.size ?? args.window_size;
|
||||
const size = sizeCandidate !== undefined ? parsePoint(sizeCandidate, "窗口尺寸", node.id, node.componentId) : [Number(args.width ?? current.width), Number(args.height ?? current.height)] as [number, number];
|
||||
const x = sizeOnly ? Number(current.x) : position[0];
|
||||
const y = sizeOnly ? Number(current.y) : position[1];
|
||||
const width = moveOnly ? Number(current.width) : size[0];
|
||||
const height = moveOnly ? Number(current.height) : size[1];
|
||||
if (!(width > 0 && height > 0)) throw new RuntimeError("validation_error", "窗口尺寸必须大于 0", { nodeId: node.id, componentId: node.componentId });
|
||||
return driver.window.setPos(title, { x, y, width, height });
|
||||
});
|
||||
|
||||
const moveMouse: CapabilityHandler = async (_ctx, node, args) => {
|
||||
// 注意：position 字段是「坐标相对位置」(0=屏幕左上角/1=激活窗口左上角)，坐标本体在 location
|
||||
const [rawX, rawY] = parsePoint(args.location ?? args.point, "鼠标坐标", node.id, node.componentId);
|
||||
@@ -170,33 +122,22 @@ const moveMouse: CapabilityHandler = async (_ctx, node, args) => {
|
||||
let dipTarget: [number, number];
|
||||
if (relative) {
|
||||
// 相对激活窗口左上角：窗口矩形是物理像素，换回 DIP 再叠加偏移，最后统一转物理
|
||||
const output = await powershell(`${WINDOW_API}; $fw = [RpaWindowExtended]::GetForegroundWindow(); $r = New-Object RpaWindowExtended+RECT; [void][RpaWindowExtended]::GetWindowRect($fw, [ref]$r); "$($r.Left),$($r.Top)"`);
|
||||
const [wx, wy] = output.split(",").map(Number);
|
||||
const { x: wx, y: wy } = await driver.window.foregroundPosition();
|
||||
if (!Number.isFinite(wx) || !Number.isFinite(wy)) throw new RuntimeError("browser_error", "无法获取激活窗口位置", { nodeId: node.id, componentId: node.componentId });
|
||||
const [dipWx, dipWy] = physicalToDip(wx, wy);
|
||||
dipTarget = [dipWx + rawX, dipWy + rawY];
|
||||
} else {
|
||||
dipTarget = [rawX, rawY];
|
||||
}
|
||||
return moveToSmooth(dipTarget, moveType, speed);
|
||||
};
|
||||
|
||||
async function moveToSmooth(dipTarget: [number, number], moveType: string, speed: number): Promise<{ x: number; y: number }> {
|
||||
const [tx, ty] = dipToPhysical(dipTarget[0], dipTarget[1]);
|
||||
if (moveType === "2") {
|
||||
await powershell(`${WINDOW_API}; [void][RpaWindowExtended]::SetCursorPos(${tx},${ty})`);
|
||||
return { x: Math.round(dipTarget[0]), y: Math.round(dipTarget[1]) };
|
||||
await driver.mouse.moveTo({ x: tx, y: ty });
|
||||
} else {
|
||||
// 平滑移动：从当前光标位置出发，按速度插值（路径全程物理像素，与目标同一坐标系）
|
||||
await driver.mouse.moveSmooth({ x: tx, y: ty }, { ease: moveType === "0", speed });
|
||||
}
|
||||
// 平滑移动：从当前光标位置出发，按速度插值（路径全程物理像素，与目标同一坐标系）
|
||||
const delay = Math.min(80, Math.max(4, Math.round(1500 / speed)));
|
||||
const output = await powershell(`${WINDOW_API}; $cur = [RpaWindowExtended+POINT]::new(); [void][RpaWindowExtended]::GetCursorPos([ref]$cur); "$($cur.X),$($cur.Y)"`);
|
||||
const [cx, cy] = output.split(",").map(Number);
|
||||
const from: [number, number] = Number.isFinite(cx) && Number.isFinite(cy) ? [cx, cy] : [0, 0];
|
||||
const path = mousePath(from, [tx, ty], moveType, speed);
|
||||
const ptsLiteral = path.map(([px, py]) => `@(${px},${py})`).join(",");
|
||||
await powershell(`${WINDOW_API}; $pts = @(${ptsLiteral}); foreach ($pt in $pts) { [void][RpaWindowExtended]::SetCursorPos($pt[0], $pt[1]); Start-Sleep -Milliseconds ${delay} }`);
|
||||
return { x: Math.round(dipTarget[0]), y: Math.round(dipTarget[1]) };
|
||||
}
|
||||
};
|
||||
register(["Mouse_MoveToV1", "Mouse_MoveToV2"], moveMouse);
|
||||
|
||||
const dragMouse: CapabilityHandler = async (_ctx, node, args) => {
|
||||
@@ -205,9 +146,8 @@ const dragMouse: CapabilityHandler = async (_ctx, node, args) => {
|
||||
const [spx, spy] = dipToPhysical(sx, sy);
|
||||
const [epx, epy] = dipToPhysical(ex, ey);
|
||||
const right = ["1", "right"].includes(value(args, ["side"], "0").toLowerCase());
|
||||
const down = right ? 8 : 2;
|
||||
const up = right ? 16 : 4;
|
||||
await powershell(`${WINDOW_API}; [RpaWindowExtended]::SetCursorPos(${spx},${spy}) | Out-Null; [RpaWindowExtended]::mouse_event(${down},0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 80; [RpaWindowExtended]::SetCursorPos(${epx},${epy}) | Out-Null; [RpaWindowExtended]::mouse_event(${up},0,0,0,[UIntPtr]::Zero)`);
|
||||
// 旧实现按下后停 80ms、拖到终点立即松开（与桌面版 Window_Mouse 不同，松开前不停顿）
|
||||
await driver.mouse.drag({ x: spx, y: spy }, { x: epx, y: epy }, { button: right ? "right" : "left", holdMs: 80, releaseDelayMs: 0 });
|
||||
return { start: [Math.round(sx), Math.round(sy)], end: [Math.round(ex), Math.round(ey)] };
|
||||
};
|
||||
register(["Mouse_DragToV1", "Mouse_DragToV2", "Mouse_DragToNew"], dragMouse);
|
||||
@@ -218,9 +158,7 @@ register(["Mouse_ClickCombine", "Mouse_ClickCombine_v1", "Mouse_ClickCombine_v2"
|
||||
const [px, py] = dipToPhysical(x, y);
|
||||
const right = value(args, ["click_key"], "left").toLowerCase() === "right";
|
||||
const doubleClick = value(args, ["click_time"], "click").toLowerCase().includes("double");
|
||||
const down = right ? 8 : 2;
|
||||
const up = right ? 16 : 4;
|
||||
const repeats = doubleClick ? 2 : 1;
|
||||
await powershell(`${WINDOW_API}; [RpaWindowExtended]::SetCursorPos(${px},${py}) | Out-Null; 1..${repeats} | ForEach-Object { [RpaWindowExtended]::mouse_event(${down},0,0,0,[UIntPtr]::Zero); [RpaWindowExtended]::mouse_event(${up},0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 80 }`);
|
||||
return { x: Math.round(x), y: Math.round(y), button: right ? "right" : "left", clicks: repeats };
|
||||
// 旧实现每次 down-up 之后停 80ms
|
||||
await driver.mouse.click({ x: px, y: py }, { button: right ? "right" : "left", double: doubleClick, settleMs: 80 });
|
||||
return { x: Math.round(x), y: Math.round(y), button: right ? "right" : "left", clicks: doubleClick ? 2 : 1 };
|
||||
});
|
||||
在新工单中引用
屏蔽一个用户