From e691b9fa7839e03a115293b7de6df70ffc662b0b Mon Sep 17 00:00:00 2001 From: yu <1919727607@qq.com> Date: Wed, 4 Nov 2026 13:28:32 +0800 Subject: [PATCH] =?UTF-8?q?=E9=98=B6=E6=AE=B53-driver=E6=8A=BD=E8=B1=A1:?= =?UTF-8?q?=20powershell=E5=8A=A9=E6=89=8B=E6=8B=86=E8=87=B3driver/windows?= =?UTF-8?q?,=20=E4=BA=94=E5=9F=9FAutomationDriver=E6=8E=A5=E5=8F=A3+Window?= =?UTF-8?q?sDriver(PowerShell=E5=AE=9E=E7=8E=B0=E8=A1=8C=E4=B8=BA=E4=B8=8D?= =?UTF-8?q?=E5=8F=98),=20desktop/window-extended/system-extended/common-fi?= =?UTF-8?q?nal=E6=94=B9=E8=B5=B0driver,=20catalog=E6=9E=84=E5=BB=BA?= =?UTF-8?q?=E5=A1=AB=E5=85=85platformTag=3Dwin32?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/rpa-core/README.md | 14 + .../app-system-regression.test.ts | 2 +- .../rpa-core/src/capabilities/common-final.ts | 18 +- .../src/capabilities/cv-vision.test.ts | 2 +- .../rpa-core/src/capabilities/cv-vision.ts | 2 +- .../src/capabilities/desktop-uia.test.ts | 2 +- .../rpa-core/src/capabilities/desktop-uia.ts | 51 ++-- packages/rpa-core/src/capabilities/desktop.ts | 175 +---------- .../rpa-core/src/capabilities/excel-com.ts | 2 +- packages/rpa-core/src/capabilities/message.ts | 2 +- .../src/capabilities/remaining-core.ts | 2 +- .../src/capabilities/system-extended.ts | 26 +- packages/rpa-core/src/capabilities/vision.ts | 2 +- .../src/capabilities/window-extended.ts | 160 ++++------- .../src/capabilities/word-extended.ts | 2 +- packages/rpa-core/src/capabilities/word.ts | 2 +- packages/rpa-core/src/driver/types.ts | 165 +++++++++++ .../rpa-core/src/driver/windows/powershell.ts | 74 +++++ .../src/driver/windows/windows-driver.ts | 272 ++++++++++++++++++ packages/rpa-core/src/engine/engine.ts | 4 + packages/rpa-core/src/index.ts | 26 +- scripts/build-catalog.mjs | 6 +- 22 files changed, 673 insertions(+), 338 deletions(-) create mode 100644 packages/rpa-core/src/driver/types.ts create mode 100644 packages/rpa-core/src/driver/windows/powershell.ts create mode 100644 packages/rpa-core/src/driver/windows/windows-driver.ts diff --git a/packages/rpa-core/README.md b/packages/rpa-core/README.md index f2fb330..4e013d6 100644 --- a/packages/rpa-core/README.md +++ b/packages/rpa-core/README.md @@ -8,6 +8,7 @@ RPA 能力层包：零 Electron 依赖的流程引擎、能力注册表与能力 | 模块 | 说明 | |---|---| | `src/engine/` | 注册表、运行时、表达式、`createEngine()` 工厂、`host-bridge.ts` 宿主桥 | +| `src/driver/` | `types.ts` 五域 AutomationDriver 接口（mouse/keyboard/window/screen/element）+ `windows/powershell.ts` 进程助手 + `windows/windows-driver.ts` PowerShell 实现 | | `src/capabilities/` | 能力实现（副作用注册，经 `@rpa/core/capabilities` 拉起） | | `src/browser/` | 外部浏览器 CDP：`external.ts`（Edge/Chrome 进程管理）+ `transport.ts`（CdpTransport 接口、ExternalTransport） | @@ -45,3 +46,16 @@ human-dialog 抛 `host_unavailable`。 `npm run check:deps`（根目录亦可）扫描 `src/**`：禁止 `import "electron"`、 相对引用逃出包 `src/`、引用宿主 `src/main`。阶段 2 起为提交前必跑项。 + +## Driver（阶段 3） + +- 五域接口 `AutomationDriver`（决策点 6）：`RpaEngine.driver` 默认 `new WindowsDriver()`， + `createEngine()` 可注入自定义实现（CLI/Linux driver 等未来形态）。 +- 驱动只承载**原始注入原语**（单次 SetCursorPos / SendKeys / 窗口操作）；拟真抖动、 + DIP 坐标换算、多步组合等策略留在能力文件，保证机械替换不改时序行为。 +- element 域实现承载在 `capabilities/desktop-uia.ts`（经 `setElementDriverImplementation` + 注册到 WindowsDriver），handler 层面向 driver 的收口在阶段 5/6 按需进行。 +- `powershell()` 助手统一位于 `driver/windows/powershell.ts`，包内能力禁止自行 + `execFile("powershell.exe")`（check:deps 与 review 双重把关）。 +- 平台标记：`npm run catalog` 构建时按组填充 `platformTag: "win32"`（desktop/app/ + office/vision 组），渲染层置灰延后（决策点 7）。 diff --git a/packages/rpa-core/src/capabilities/app-system-regression.test.ts b/packages/rpa-core/src/capabilities/app-system-regression.test.ts index 135b084..b6b84a4 100644 --- a/packages/rpa-core/src/capabilities/app-system-regression.test.ts +++ b/packages/rpa-core/src/capabilities/app-system-regression.test.ts @@ -4,7 +4,7 @@ import type { RunContext } from "../engine/types"; import { getHandler } from "../engine/registry"; const mocks = vi.hoisted(() => ({ powershell: vi.fn(), execFile: vi.fn() })); -vi.mock("./desktop", () => ({ powershell: mocks.powershell, psString: (value: unknown) => `'${String(value ?? "").replace(/'/g, "''")}'` })); +vi.mock("../driver/windows/powershell", () => ({ powershell: mocks.powershell, psString: (value: unknown) => `'${String(value ?? "").replace(/'/g, "''")}'` })); vi.mock("node:child_process", () => ({ execFile: mocks.execFile })); import "./data"; import "./control"; diff --git a/packages/rpa-core/src/capabilities/common-final.ts b/packages/rpa-core/src/capabilities/common-final.ts index 36d2036..1d67788 100644 --- a/packages/rpa-core/src/capabilities/common-final.ts +++ b/packages/rpa-core/src/capabilities/common-final.ts @@ -4,7 +4,11 @@ import path from "node:path"; import { PDFDocument, StandardFonts, degrees, rgb } from "pdf-lib"; import { getHandler, register } from "../engine/registry"; import { RuntimeError, type CapabilityHandler } from "../engine/types"; -import { powershell, psString } from "./desktop"; +import { powershell, psString } from "../driver/windows/powershell"; +import { defaultEngine } from "../engine/engine"; + +// 窗口/屏幕查询统一走驱动（任务 3.3） +const driver = defaultEngine.driver; function text(args: Record, keys: string[], fallback = ""): string { for (const key of keys) { const value = args[key]; if (value !== undefined && value !== null && value !== "") return String(value); } @@ -20,25 +24,21 @@ register("Window_GetRect", delegate("Window_GetWindowRect")); register("Window_MaximumSize", (ctx, node, args, component) => delegate("Window_SetWindowState")(ctx, node, { ...args, WindowType: "maximize" }, component)); register(["Keyboard_DriveSend", "Keyboard_DriveSend_V1", "Keyboard_DriveSend_V2"], delegate("Keyboard_Send_V1")); -register("Window_GetScreenSize", async () => { - const output = await powershell("Add-Type -AssemblyName System.Windows.Forms; $b=[System.Windows.Forms.Screen]::PrimaryScreen.Bounds; @{width=$b.Width;height=$b.Height;x=$b.X;y=$b.Y} | ConvertTo-Json -Compress"); - return JSON.parse(output); -}); +register("Window_GetScreenSize", async () => driver.screen.size()); register("Window_WindowList", async (_ctx, _node, args) => { const title = text(args, ["match_title", "select_title"]); const regex = String(args.is_regex ?? "0") === "1"; - const output = await powershell(`Get-Process | Where-Object { $_.MainWindowHandle -ne 0 ${title ? (regex ? `-and $_.MainWindowTitle -match ${psString(title)}` : `-and $_.MainWindowTitle -like ${psString(`*${title}*`)}`) : ""} } | ForEach-Object { @{pid=$_.Id;title=$_.MainWindowTitle;hwnd=$_.MainWindowHandle.ToInt64()} } | ConvertTo-Json -Compress`); - if (!output) return []; const parsed = JSON.parse(output); return Array.isArray(parsed) ? parsed : [parsed]; + return driver.window.list({ title, regex }); }); register("Window_SetWindowShow", async (_ctx, node, args) => { const title = text(args, ["窗口对象", "window_obj", "title"]); if (!title || title === "窗口对象") throw new RuntimeError("validation_error", "窗口标题为空", { nodeId: node.id, componentId: node.componentId }); const show = !text(args, ["WindowType"], "show").toLowerCase().includes("hide"); - await powershell(`$p=Get-Process|Where-Object{$_.MainWindowTitle -like ${psString(`*${title}*`)}}|Select-Object -First 1;if(!$p){throw '找不到窗口'};Add-Type @'\nusing System;using System.Runtime.InteropServices;public static class W{[DllImport("user32.dll")]public static extern bool ShowWindow(IntPtr h,int n);}\n'@;[W]::ShowWindow($p.MainWindowHandle,${show ? 5 : 0})|Out-Null`); + await driver.window.setState(title, show ? 5 : 0); return show; }); register("Window_WaitWindow", async (_ctx, node, args) => { const title = text(args, ["match_title", "select_title", "window_obj", "target_window"]); if (!title) throw new RuntimeError("validation_error", "窗口标题为空", { nodeId: node.id, componentId: node.componentId }); const expected = text(args, ["switchType"], "0") !== "1"; const deadline = Date.now() + Math.max(0, Number(args.second ?? 5)) * 1000; - do { const exists = (await powershell(`$p=Get-Process|Where-Object{$_.MainWindowTitle -like ${psString(`*${title}*`)}}|Select-Object -First 1;[bool]$p`)).toLowerCase() === "true"; if (exists === expected) return true; await new Promise((resolve) => setTimeout(resolve, 200)); } while (Date.now() <= deadline); + do { const exists = await driver.window.exists(title); if (exists === expected) return true; await new Promise((resolve) => setTimeout(resolve, 200)); } while (Date.now() <= deadline); return false; }); diff --git a/packages/rpa-core/src/capabilities/cv-vision.test.ts b/packages/rpa-core/src/capabilities/cv-vision.test.ts index 31c52c4..b765d6c 100644 --- a/packages/rpa-core/src/capabilities/cv-vision.test.ts +++ b/packages/rpa-core/src/capabilities/cv-vision.test.ts @@ -13,7 +13,7 @@ const detectMock = vi.hoisted(() => vi.fn(async () => ({ // desktopCapturer 模拟: 默认无画面(走 GDI 回退), 多屏用例按请求宽度提供对应显示器画面 const electronState = vi.hoisted(() => ({ sources: [] as Array<{ display_id: string; thumbnail: any }> })); -vi.mock("./desktop", () => ({ powershell: powershellMock })); +vi.mock("../driver/windows/powershell", () => ({ powershell: powershellMock })); vi.mock("@gutenye/ocr-node", () => ({ default: { create: vi.fn(async () => ({ detect: detectMock })) } })); diff --git a/packages/rpa-core/src/capabilities/cv-vision.ts b/packages/rpa-core/src/capabilities/cv-vision.ts index 50ad6ce..e290fdb 100644 --- a/packages/rpa-core/src/capabilities/cv-vision.ts +++ b/packages/rpa-core/src/capabilities/cv-vision.ts @@ -1,6 +1,6 @@ import { register } from "../engine/registry"; import { RuntimeError, type CapabilityHandler } from "../engine/types"; -import { powershell } from "./desktop"; +import { powershell } from "../driver/windows/powershell"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; diff --git a/packages/rpa-core/src/capabilities/desktop-uia.test.ts b/packages/rpa-core/src/capabilities/desktop-uia.test.ts index 3bf714b..a605328 100644 --- a/packages/rpa-core/src/capabilities/desktop-uia.test.ts +++ b/packages/rpa-core/src/capabilities/desktop-uia.test.ts @@ -2,7 +2,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const powershellMock = vi.hoisted(() => vi.fn(async () => '{"clicked":true,"value":"cached"}')); -vi.mock("./desktop", () => ({ +vi.mock("../driver/windows/powershell", () => ({ powershell: powershellMock, psString: (value: unknown) => JSON.stringify(String(value ?? "")) })); diff --git a/packages/rpa-core/src/capabilities/desktop-uia.ts b/packages/rpa-core/src/capabilities/desktop-uia.ts index 04db745..d476cc6 100644 --- a/packages/rpa-core/src/capabilities/desktop-uia.ts +++ b/packages/rpa-core/src/capabilities/desktop-uia.ts @@ -2,30 +2,11 @@ import { register } from "../engine/registry"; import fs from "node:fs"; import path from "node:path"; import { RuntimeError, type CapabilityHandler } from "../engine/types"; -import { powershell, psString } from "./desktop"; +import { powershell, psString } from "../driver/windows/powershell"; +import type { UiaSelector } from "../driver/types"; +import { setElementDriverImplementation } from "../driver/windows/windows-driver"; -interface UiaSelector { - window?: string; - name?: string; - id?: string; - /** .NET ControlType 字段名（已映射，如 ComboBox/ListItem/DataGrid） */ - type?: string; - /** 用户输入的原始类型字符串 */ - typeRaw?: string; - className?: string; - screen?: { x: number; y: number; w: number; h: number }; - /** Relative physical rect inside the top-level window. */ - rel?: { x: number; y: number; w: number; h: number }; - process?: string; - winClass?: string; - enriched?: boolean; - aiType?: string; - aiText?: string; - ai?: boolean; - /** 从窗口（或桌面根）到目标元素的祖先路径，逐段用稳定属性定位 */ - path?: Array<{ type?: string; name?: string; id?: string; className?: string; index?: number }>; - index: number; -} +// UiaSelector 类型已上移 driver/types.ts（任务 3.2 element 域） /** ControlType 显式映射：type= 值 → .NET ControlType 字段名；复合名称不得用首字母大写推导 */ const UIA_CONTROL_TYPE_MAP: Record = { @@ -1045,3 +1026,27 @@ register("Element_DesktopChildren", childrenGroundHandler(true)); register("Element_RestoreWindowGround", restoreWindowGround); register("Element_ActivateWindowGround", activateWindowGround); register("Element_MoveWindowGround", moveWindowGround); + +/** ElementDriver 的 PowerShell 实现（任务 3.3，决策点 6）：ground 原语委托本模块内部函数。 + * handler 层面向 driver 接口的收口放到阶段 5/6 按需进行，避免大范围重写引入行为回归。 */ +setElementDriverImplementation({ + parseSelector: (raw) => parseUiaSelector(raw), + async find(selector, options) { + return findElement({ selector, args: {}, node: { id: "driver", componentId: "driver" } }, options?.timeoutMs ?? 5000, options?.gapMs ?? 200); + }, + async exists(selector) { + return (await findElement({ selector, args: {}, node: { id: "driver", componentId: "driver" } }, 2000, 200)) !== null; + }, + async click(selector, options) { + const t = { selector, args: {}, node: { id: "driver", componentId: "driver" } }; + await runScript(clickScript(t), t, options?.timeoutMs ?? 60000); + }, + async setValue(selector, value) { + const t = { selector, args: {}, node: { id: "driver", componentId: "driver" } }; + await runScript(setValueScript(t, value), t); + }, + async getValue(selector) { + const t = { selector, args: {}, node: { id: "driver", componentId: "driver" } }; + return runScript(getValueScript(t), t); + } +}); diff --git a/packages/rpa-core/src/capabilities/desktop.ts b/packages/rpa-core/src/capabilities/desktop.ts index 8d0e26e..4c14823 100644 --- a/packages/rpa-core/src/capabilities/desktop.ts +++ b/packages/rpa-core/src/capabilities/desktop.ts @@ -1,13 +1,12 @@ -import { execFile } from "node:child_process"; -import { promisify } from "node:util"; import { register } from "../engine/registry"; import { RuntimeError, type CapabilityHandler } from "../engine/types"; import { type ManualOperationSettings, defaultEngine } from "../engine/engine"; - -const execFileAsync = promisify(execFile); +import { powershell, psString } from "../driver/windows/powershell"; // 人为拟真设置本体挂在引擎实例上（任务 1.3），此处委托 defaultEngine。 const manualOperation: ManualOperationSettings = defaultEngine.manualOperation; +// 桌面自动化驱动（任务 3.3）：user32 注入原语统一走 WindowsDriver +const driver = defaultEngine.driver; function jitter(value: number, amount = 3): number { return Math.max(0, Math.round(value + (Math.random() * amount * 2 - amount))); @@ -26,18 +25,6 @@ async function manualDelay(): Promise { const timeout = low + Math.random() * (high - low); if (timeout > 0) await new Promise((resolve) => setTimeout(resolve, timeout)); } -const USER32 = `Add-Type @' -using System; -using System.Runtime.InteropServices; -public static class RpaUser32 { - [DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y); - [DllImport("user32.dll")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra); - [DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); - [DllImport("user32.dll", CharSet=CharSet.Unicode)] public static extern int GetWindowText(IntPtr hWnd, System.Text.StringBuilder text, int count); - [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd); -} -'@`; - function text(args: Record, keys: string[], fallback = ""): string { for (const key of keys) { if (args[key] !== undefined && args[key] !== null && args[key] !== "") return String(args[key]); @@ -65,82 +52,6 @@ function point(value: unknown): [number, number] { } } -function encoded(command: string): string { - return Buffer.from(command, "utf16le").toString("base64"); -} - -/** 从 PowerShell 错误输出里提取 CLIXML 段的真实错误文本（否则显示为乱码转义序列） */ -function decodeCliXml(detail: string): string { - const lines = [...detail.matchAll(/(.*?)<\/S>/gs)].map((m) => m[1]); - if (lines.length === 0) return detail; - // _xHHHH_ 是 CLIXML 转义 - const decoded = lines.join("\n").replace(/_x([0-9A-Fa-f]{4})_/g, (_, h) => String.fromCharCode(parseInt(h, 16))); - const parts = decoded.split("\n").map((l) => l.trim()); - const position = parts.filter((l) => l.startsWith("所在位置")).slice(0, 1); - const message = parts.filter( - (l) => l && !l.startsWith("所在位置") && !l.startsWith("+") && !l.startsWith("~~") && !l.includes("CategoryInfo") && !l.includes("FullyQualifiedErrorId") && !l.startsWith(" { - try { - // Windows PowerShell 的默认输出编码依机器代码页变化；Node 按 UTF-8 解码管道。 - const utf8 = "$ProgressPreference='SilentlyContinue'; [Console]::OutputEncoding=[System.Text.UTF8Encoding]::new($false); $OutputEncoding=[Console]::OutputEncoding; "; - const result = await execFileAsync("powershell.exe", ["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-EncodedCommand", encoded(utf8 + command)], { - windowsHide: true, - timeout, - maxBuffer: 4 * 1024 * 1024, - encoding: "buffer" - } as never); - return decodePipe(result.stdout as unknown as Buffer).trim(); - } catch (error) { - const err = error as Error & { stdout?: unknown; stderr?: unknown }; - // 脚本主动失败（RpaFail）把完整错误写在 stdout 的 RPA_ERROR:: 标记后，不经 CLIXML 渲染，无截断 - const stdoutText = Buffer.isBuffer(err.stdout) ? decodePipe(err.stdout) : String(err.stdout ?? ""); - const marked = stdoutText.match(/RPA_ERROR::([\s\S]*)/); - if (marked) { - const detail = marked[1].trim(); - throw new RuntimeError("desktop_command_failed", `桌面操作失败：${detail}`, { detail }); - } - const stderrBuf = Buffer.isBuffer(err.stderr) ? err.stderr : Buffer.from(String(err.stderr ?? ""), "utf8"); - const stderrText = stderrBuf.length > 0 ? decodePipe(stderrBuf) : err.message; - const detail = decodeCliXml(stderrText); - throw new RuntimeError("desktop_command_failed", `桌面操作失败：${detail}`, { detail }); - } -} - -export function psString(value: unknown): string { - return `'${String(value ?? "").replace(/'/g, "''")}'`; -} - -/** 把 SendKeys 特殊字符按字面转义（+^%~(){}[] → {+} 等）；SendKeys 用花括号表示字面量 */ -function sendKeysEscape(ch: string): string { - return "+^%~(){}[]".includes(ch) ? `{${ch}}` : ch; -} - -/** - * 把字符包装成 PowerShell 安全字面量。 - * ASCII 直接用单引号字符串（撇号内部 '' 转义）；非 ASCII 用 [char]0xXXXX 数值构造—— - * Windows PowerShell 5.1 会把 ‘ ’ “ ” 等智能引号当作引号变体解析，直接放进字符串字面量会报「字符串缺少终止符」。 - */ -function psCharLiteral(text: string): string { - if (/^[\x20-\x7E]*$/.test(text)) return psString(text); - return Array.from(text).map((c) => `[char]0x${c.codePointAt(0)!.toString(16).padStart(4, "0")}`).join("+") || psString(""); -} - const keyboardSend: CapabilityHandler = async (_ctx, node, args) => { // 字段: words=输入内容(键盘输入), keys/key=按键组合(按键系列), interval=输入间隔(V1毫秒/基础版秒), fuctype=输入方式(0后台接口/1人工模拟/2剪贴板粘贴), window_title=输入目标窗口(留空=当前焦点窗口) const keys = text(args, ["words", "keys", "key", "text", "value"]); @@ -152,27 +63,7 @@ const keyboardSend: CapabilityHandler = async (_ctx, node, args) => { const isTextSend = node.componentId === "Keyboard_Send" || node.componentId === "Keyboard_Send_V1"; // 指定目标窗口时先激活再输入，内容才会进到目标窗口而不是运行时的焦点窗口 const targetTitle = text(args, ["window_title", "target_window", "window_obj", "match_title", "select_title"]); - const activate = targetTitle ? `${activateWindowScript(targetTitle)}; Start-Sleep -Milliseconds 300; ` : ""; - - // 三种输入方式：脚本尽量简单，转义逻辑放 JS 侧，避免 PowerShell 字符串拼接的语法陷阱 - let send: string; - if (mode === "2") { - // 剪贴板粘贴：原文进剪贴板 + Ctrl+V，不做 SendKeys 转义；内容含智能引号等非 ASCII 时同样走 [char] 构造 - const contentExpr = /^[\x20-\x7E]*$/.test(keys) ? psString(keys) : `(-join @(${Array.from(keys).map((c) => psCharLiteral(c)).join(",")}))`; - send = `Set-Clipboard -Value ${contentExpr}; Start-Sleep -Milliseconds 120; [System.Windows.Forms.SendKeys]::SendWait('^v')`; - } else if (mode === "1" && isTextSend) { - // 人工模拟：逐字符；JS 侧把每个字符转义成 SendKeys 字面量，PS 侧只做数组循环 - // 数组元素必须用 psCharLiteral：ASCII 走引号字符串，非 ASCII 走 [char]0xXXXX（避开 PS5.1 智能引号解析陷阱） - const escapedChars = Array.from(keys).map((ch) => sendKeysEscape(ch)); - const arr = escapedChars.map((c) => psCharLiteral(c)).join(","); - send = `$chars = @(${arr}); foreach ($c in $chars) { [System.Windows.Forms.SendKeys]::SendWait([string]$c); Start-Sleep -Milliseconds ${intervalMs} }`; - } else { - // 后台接口 / 按键组合 / 人工模拟(按键组合组件)：整段发送，文本特殊字符转义 - const escaped = isTextSend ? Array.from(keys).map(sendKeysEscape).join("") : keys; - const contentExpr = /^[\x20-\x7E]*$/.test(escaped) ? psString(escaped) : `(-join @(${Array.from(escaped).map((c) => psCharLiteral(c)).join(",")}))`; - send = `[System.Windows.Forms.SendKeys]::SendWait(${contentExpr})`; - } - await powershell(`${USER32}; ${activate}Add-Type -AssemblyName System.Windows.Forms; ${send}`); + await driver.keyboard.send({ keys, intervalMs, mode: mode === "2" ? "clipboard" : mode === "1" ? "manual" : "api", textMode: isTextSend, activateTitle: targetTitle || undefined }); await manualDelay(); return keys; }; @@ -197,89 +88,47 @@ const disableManualOperation: CapabilityHandler = async () => { const mouseClick: CapabilityHandler = async (_ctx, node, args) => { const [x, y] = manualJitter(point(args.position ?? args.point ?? args.location ?? "[0,0]"), false, true); const button = text(args, ["side", "button"], "0").toLowerCase(); - const down = button.includes("right") || button === "1" ? 0x0008 : 0x0002; - const up = button.includes("right") || button === "1" ? 0x0010 : 0x0004; - await powershell(`${USER32}; [RpaUser32]::SetCursorPos(${x},${y}); [RpaUser32]::mouse_event(${down},0,0,0,[UIntPtr]::Zero); [RpaUser32]::mouse_event(${up},0,0,0,[UIntPtr]::Zero)`); + await driver.mouse.click({ x, y }, { button: button.includes("right") || button === "1" ? "right" : "left" }); return { x, y }; }; const mouseMove: CapabilityHandler = async (_ctx, node, args) => { const [x, y] = manualJitter(point(args.position ?? args.point ?? args.location ?? "[0,0]"), true, false); - await powershell(`${USER32}; [RpaUser32]::SetCursorPos(${x},${y})`); + await driver.mouse.moveTo({ x, y }); return { x, y }; }; const mouseDrag: CapabilityHandler = async (_ctx, node, args) => { const [sx, sy] = point(args.Start ?? args.start ?? "[0,0]"); const [ex, ey] = point(args.End ?? args.end ?? "[0,0]"); - await powershell(`${USER32}; [RpaUser32]::SetCursorPos(${sx},${sy}); [RpaUser32]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 80; [RpaUser32]::SetCursorPos(${ex},${ey}); Start-Sleep -Milliseconds 80; [RpaUser32]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)`); + await driver.mouse.drag({ x: sx, y: sy }, { x: ex, y: ey }, { holdMs: 80, releaseDelayMs: 80 }); return { start: [sx, sy], end: [ex, ey] }; }; const mouseRoll: CapabilityHandler = async (_ctx, node, args) => { const amount = number(args, ["RollNum", "rollNum", "amount"], 1) * 120; - await powershell(`${USER32}; [RpaUser32]::mouse_event(0x0800,0,0,${amount},[UIntPtr]::Zero)`); + await driver.mouse.scroll(amount); return amount; }; -/** 把匹配标题的窗口带到前台；多招式依次兜底（应用点击「运行」后自身是前台， - * Windows 前台锁定策略会拒绝普通 SetForegroundWindow，最小化再恢复可绕过）。 - * 失败用 RPA_ERROR 标记写 stdout 后 exit 1：不走 throw/CLIXML，避免控制台宽度截断和编码歧义 */ -function activateWindowScript(selector: string): string { - const safeTitle = selector.replace(/'/g, "''"); - return `function RpaFail($m) { [Console]::Out.WriteLine('RPA_ERROR::' + $m); exit 1 } -$p = Get-Process | Where-Object { $_.MainWindowTitle -and $_.MainWindowTitle -like ${psString(`*${selector}*`)} } | Select-Object -First 1 -if (-not $p) { $vis = (Get-Process | Where-Object { $_.MainWindowTitle } | Select-Object -First 12 | ForEach-Object { '『' + $_.MainWindowTitle + '』' }) -join ' '; RpaFail ('找不到窗口：${safeTitle}。当前可见窗口标题: ' + $vis) } -Add-Type -MemberDefinition '[DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); [DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr h, int cmd); [DllImport("user32.dll")] public static extern bool IsIconic(IntPtr h); [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr h); [DllImport("user32.dll")] public static extern void keybd_event(byte bVk, byte bScan, uint dwFlags, UIntPtr dwExtraInfo); [DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr hWnd, IntPtr pid); [DllImport("user32.dll")] public static extern bool AttachThreadInput(uint a, uint b, bool attach); [DllImport("user32.dll")] public static extern void SwitchToThisWindow(IntPtr hWnd, bool fAltTab); [DllImport("kernel32.dll")] public static extern uint GetCurrentThreadId();' -Name RpaFocus2 -Namespace WinRpaFocus2 | Out-Null -$h = $p.MainWindowHandle -if ([WinRpaFocus2.RpaFocus2]::IsIconic($h)) { [void][WinRpaFocus2.RpaFocus2]::ShowWindow($h, 9); Start-Sleep -Milliseconds 150 } -if ([WinRpaFocus2.RpaFocus2]::GetForegroundWindow() -ne $h) { - [void][WinRpaFocus2.RpaFocus2]::SwitchToThisWindow($h, $true) - Start-Sleep -Milliseconds 150 -} -if ([WinRpaFocus2.RpaFocus2]::GetForegroundWindow() -ne $h) { - [void][WinRpaFocus2.RpaFocus2]::keybd_event(0x12, 0, 0, [UIntPtr]::Zero) - [void][WinRpaFocus2.RpaFocus2]::SetForegroundWindow($h) - [void][WinRpaFocus2.RpaFocus2]::keybd_event(0x12, 0, 2, [UIntPtr]::Zero) - Start-Sleep -Milliseconds 200 -} -if ([WinRpaFocus2.RpaFocus2]::GetForegroundWindow() -ne $h) { - $fgThread = [WinRpaFocus2.RpaFocus2]::GetWindowThreadProcessId([WinRpaFocus2.RpaFocus2]::GetForegroundWindow(), [IntPtr]::Zero) - $thisThread = [WinRpaFocus2.RpaFocus2]::GetCurrentThreadId() - $attached = [WinRpaFocus2.RpaFocus2]::AttachThreadInput($thisThread, $fgThread, $true) - try { [void][WinRpaFocus2.RpaFocus2]::SetForegroundWindow($h) } finally { if ($attached) { [void][WinRpaFocus2.RpaFocus2]::AttachThreadInput($thisThread, $fgThread, $false) } } - Start-Sleep -Milliseconds 200 -} -if ([WinRpaFocus2.RpaFocus2]::GetForegroundWindow() -ne $h) { - [void][WinRpaFocus2.RpaFocus2]::ShowWindow($h, 6) - Start-Sleep -Milliseconds 150 - [void][WinRpaFocus2.RpaFocus2]::ShowWindow($h, 9) - Start-Sleep -Milliseconds 300 -} -if ([WinRpaFocus2.RpaFocus2]::GetForegroundWindow() -ne $h) { RpaFail '无法把窗口带到前台，请确认窗口未被关闭或全屏占用' }`; -} - const setActiveWindow: CapabilityHandler = async (_ctx, node, args) => { const selector = text(args, ["window_obj", "match_title", "select_title", "title", "target_window", "window_title"]); if (!selector) throw new RuntimeError("validation_error", "窗口标题为空", { nodeId: node.id, componentId: node.componentId }); - const result = await powershell(`${USER32} -${activateWindowScript(selector)} -$p.MainWindowTitle`); - return result; + return driver.window.focus(selector); }; -const getActiveWindow: CapabilityHandler = async () => powershell(`${USER32}; $sb = New-Object System.Text.StringBuilder 512; [RpaUser32]::GetWindowText([RpaUser32]::GetForegroundWindow(),$sb,$sb.Capacity) | Out-Null; $sb.ToString()`); +const getActiveWindow: CapabilityHandler = async () => driver.window.foregroundTitle(); const windowExists: CapabilityHandler = async (_ctx, node, args) => { const selector = text(args, ["window_obj", "match_title", "select_title", "title", "target_window", "window_title"]); if (!selector) throw new RuntimeError("validation_error", "窗口标题为空", { nodeId: node.id, componentId: node.componentId }); - return (await powershell(`$p = Get-Process | Where-Object { $_.MainWindowTitle -and $_.MainWindowTitle -like ${psString(`*${selector}*`)} } | Select-Object -First 1; [bool]$p`)).toLowerCase() === "true"; + return driver.window.exists(selector); }; const closeWindow: CapabilityHandler = async (_ctx, node, args) => { const selector = text(args, ["window_obj", "match_title", "select_title", "title", "target_window", "window_title"]); if (!selector) throw new RuntimeError("validation_error", "窗口标题为空", { nodeId: node.id, componentId: node.componentId }); - return powershell(`$p = Get-Process | Where-Object { $_.MainWindowTitle -and $_.MainWindowTitle -like ${psString(`*${selector}*`)} } | Select-Object -First 1; if (-not $p) { throw '找不到窗口' }; $p.CloseMainWindow() | Out-Null; $true`); + return driver.window.close(selector); }; const runApp: CapabilityHandler = async (_ctx, node, args) => { diff --git a/packages/rpa-core/src/capabilities/excel-com.ts b/packages/rpa-core/src/capabilities/excel-com.ts index eea9c5e..e87c93c 100644 --- a/packages/rpa-core/src/capabilities/excel-com.ts +++ b/packages/rpa-core/src/capabilities/excel-com.ts @@ -3,7 +3,7 @@ import path from "node:path"; import ExcelJS from "exceljs"; import { register } from "../engine/registry"; import { RuntimeError, type CapabilityHandler } from "../engine/types"; -import { powershell, psString } from "./desktop"; +import { powershell, psString } from "../driver/windows/powershell"; import { releaseWorkbook } from "./excel-store"; function text(args: Record, keys: string[], fallback = ""): string { diff --git a/packages/rpa-core/src/capabilities/message.ts b/packages/rpa-core/src/capabilities/message.ts index ae30530..c46c3f3 100644 --- a/packages/rpa-core/src/capabilities/message.ts +++ b/packages/rpa-core/src/capabilities/message.ts @@ -1,7 +1,7 @@ import nodemailer from "nodemailer"; import { register } from "../engine/registry"; import { RuntimeError } from "../engine/types"; -import { powershell } from "./desktop"; +import { powershell } from "../driver/windows/powershell"; function text(args: Record, keys: string[], fallback = ""): string { for (const key of keys) if (args[key] !== undefined && args[key] !== null && args[key] !== "") return String(args[key]); diff --git a/packages/rpa-core/src/capabilities/remaining-core.ts b/packages/rpa-core/src/capabilities/remaining-core.ts index 9ef3f88..bcb6858 100644 --- a/packages/rpa-core/src/capabilities/remaining-core.ts +++ b/packages/rpa-core/src/capabilities/remaining-core.ts @@ -5,7 +5,7 @@ import { runFlow } from "../engine/runtime"; import { RuntimeError, type CapabilityHandler, type RunContext } from "../engine/types"; import { defaultEngine } from "../engine/engine"; import { getSubflowSource } from "../engine/host-bridge"; -import { powershell, psString } from "./desktop"; +import { powershell, psString } from "../driver/windows/powershell"; import { findImage } from "./vision"; function text(args: Record, keys: string[], fallback = ""): string { diff --git a/packages/rpa-core/src/capabilities/system-extended.ts b/packages/rpa-core/src/capabilities/system-extended.ts index 3beffc8..e09610c 100644 --- a/packages/rpa-core/src/capabilities/system-extended.ts +++ b/packages/rpa-core/src/capabilities/system-extended.ts @@ -2,7 +2,8 @@ import os from "node:os"; import path from "node:path"; import { register } from "../engine/registry"; import { RuntimeError, type CapabilityHandler } from "../engine/types"; -import { powershell, psString } from "./desktop"; +import { powershell, psString } from "../driver/windows/powershell"; +import { defaultEngine } from "../engine/engine"; function value(args: Record, keys: string[], fallback = ""): string { for (const key of keys) { @@ -31,20 +32,8 @@ function selector(args: Record): string { return String(raw ?? ""); } -const WINDOW_API = `Add-Type @' -using System; -using System.Runtime.InteropServices; -public static class RpaWindowApi { - [StructLayout(LayoutKind.Sequential)] public struct RECT { public int Left; public int Top; public int Right; public int Bottom; } - [DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr hWnd, out RECT rect); - [DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr hWnd, int command); - [DllImport("user32.dll")] public static extern bool SetWindowPos(IntPtr hWnd, IntPtr after, int x, int y, int width, int height, uint flags); -} -'@`; - -function findWindowScript(title: string): string { - return `$p = Get-Process | Where-Object { $_.MainWindowHandle -ne 0 -and $_.MainWindowTitle -like ${psString(`*${title}*`)} } | Select-Object -First 1; if (-not $p) { throw '找不到窗口：${title.replace(/'/g, "''")}' }`; -} +// 窗口矩形/状态操作统一走驱动（任务 3.3） +const driver = defaultEngine.driver; register("System_GetEnv", (_ctx, node, args) => { const name = required(args, ["name"], "环境变量名称", node.id, node.componentId); @@ -115,15 +104,14 @@ register("Clipboard_ClearClipboard", async () => { register("Window_GetWindowRect", async (_ctx, node, args) => { const title = required({ title: selector(args) }, ["title"], "窗口标题", node.id, node.componentId); - const output = await powershell(`${WINDOW_API}; ${findWindowScript(title)}; $r = New-Object RpaWindowApi+RECT; [RpaWindowApi]::GetWindowRect($p.MainWindowHandle,[ref]$r) | Out-Null; @{x=$r.Left;y=$r.Top;width=$r.Right-$r.Left;height=$r.Bottom-$r.Top;title=$p.MainWindowTitle;pid=$p.Id} | ConvertTo-Json -Compress`); - return JSON.parse(output); + return driver.window.findByTitle(title); }); register(["Window_SetWindowState", "Window_SetWindowState_v1"], async (_ctx, node, args) => { const title = required({ title: selector(args) }, ["title"], "窗口标题", node.id, node.componentId); const state = value(args, ["WindowType", "switchType"], "maximize").toLowerCase(); const command = state.includes("min") ? 6 : state.includes("max") ? 3 : 9; - await powershell(`${WINDOW_API}; ${findWindowScript(title)}; [RpaWindowApi]::ShowWindow($p.MainWindowHandle,${command}) | Out-Null`); + await driver.window.setState(title, command); return state; }); @@ -136,6 +124,6 @@ register("Window_SizeWindow_v1", async (_ctx, node, args) => { } const [width, height] = Array.isArray(size) ? size.map(Number) : [0, 0]; if (!(width > 0 && height > 0)) throw new RuntimeError("validation_error", "窗口尺寸无效", { nodeId: node.id, componentId: node.componentId }); - await powershell(`${WINDOW_API}; ${findWindowScript(title)}; $r = New-Object RpaWindowApi+RECT; [RpaWindowApi]::GetWindowRect($p.MainWindowHandle,[ref]$r) | Out-Null; [RpaWindowApi]::SetWindowPos($p.MainWindowHandle,[IntPtr]::Zero,$r.Left,$r.Top,${width},${height},4) | Out-Null`); + await driver.window.setSize(title, width, height); return { width, height }; }); diff --git a/packages/rpa-core/src/capabilities/vision.ts b/packages/rpa-core/src/capabilities/vision.ts index 4199afd..62036da 100644 --- a/packages/rpa-core/src/capabilities/vision.ts +++ b/packages/rpa-core/src/capabilities/vision.ts @@ -3,7 +3,7 @@ import { promises as fs } from "node:fs"; import { PNG } from "pngjs"; import { register } from "../engine/registry"; import { RuntimeError } from "../engine/types"; -import { powershell, psString } from "./desktop"; +import { powershell, psString } from "../driver/windows/powershell"; function text(args: Record, keys: string[], fallback = ""): string { for (const key of keys) if (args[key] !== undefined && args[key] !== null && args[key] !== "") return String(args[key]); diff --git a/packages/rpa-core/src/capabilities/window-extended.ts b/packages/rpa-core/src/capabilities/window-extended.ts index ec45e8d..baf9efa 100644 --- a/packages/rpa-core/src/capabilities/window-extended.ts +++ b/packages/rpa-core/src/capabilities/window-extended.ts @@ -1,25 +1,10 @@ import { register } from "../engine/registry"; import { RuntimeError, type CapabilityHandler } from "../engine/types"; -import { powershell, psString } from "./desktop"; import { getHostBridge } from "../engine/host-bridge"; +import { defaultEngine } from "../engine/engine"; -const WINDOW_API = `Add-Type @' -using System; -using System.Runtime.InteropServices; -public static class RpaWindowExtended { - [StructLayout(LayoutKind.Sequential)] public struct RECT { public int Left; public int Top; public int Right; public int Bottom; } - [StructLayout(LayoutKind.Sequential)] public struct POINT { public int X; public int Y; } - [DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr hWnd, out RECT rect); - [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd); - [DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr hWnd, int command); - [DllImport("user32.dll")] public static extern bool SetWindowPos(IntPtr hWnd, IntPtr after, int x, int y, int width, int height, uint flags); - [DllImport("user32.dll")] public static extern bool SetCursorPos(int x, int y); - [DllImport("user32.dll")] public static extern bool GetCursorPos(out POINT point); - [DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); - [DllImport("user32.dll")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra); - [DllImport("user32.dll")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value); -} -'@; try { [void][RpaWindowExtended]::SetProcessDpiAwarenessContext([IntPtr](-4)) } catch {}`; +// 桌面自动化驱动（任务 3.3）：窗口矩形/鼠标注入原语统一走 WindowsDriver +const driver = defaultEngine.driver; function value(args: Record, keys: string[], fallback = ""): string { for (const key of keys) { @@ -49,10 +34,6 @@ function requiredTitle(args: Record, nodeId: string, componentI return title; } -function findWindow(title: string): string { - return `$p = Get-Process | Where-Object { $_.MainWindowHandle -ne 0 -and $_.MainWindowTitle -like ${psString(`*${title}*`)} } | Select-Object -First 1; if (-not $p) { throw '找不到窗口：${title.replace(/'/g, "''")}' }`; -} - function parsePoint(candidate: unknown, label: string, nodeId: string, componentId: string): [number, number] { let parsed: unknown = candidate; if (typeof parsed === "string") { @@ -70,70 +51,6 @@ function parsePoint(candidate: unknown, label: string, nodeId: string, component return [Number(parsed[0]), Number(parsed[1])]; } -async function windowObject(title: string): Promise> { - const output = await powershell(`${WINDOW_API}; ${findWindow(title)}; $r = New-Object RpaWindowExtended+RECT; [RpaWindowExtended]::GetWindowRect($p.MainWindowHandle,[ref]$r) | Out-Null; @{pid=$p.Id;title=$p.MainWindowTitle;hwnd=$p.MainWindowHandle.ToInt64();x=$r.Left;y=$r.Top;width=$r.Right-$r.Left;height=$r.Bottom-$r.Top} | ConvertTo-Json -Compress`); - return JSON.parse(output); -} - -const getWindow: CapabilityHandler = async (_ctx, node, args) => windowObject(requiredTitle(args, node.id, node.componentId)); -register(["Window_Get", "Window_GetWindow", "Window_GetWindow_v1"], getWindow); - -register("Window_ActiveWindow_v1", async (_ctx, node, args) => { - const title = requiredTitle(args, node.id, node.componentId); - const maximize = [true, 1, "1", "true", "True"].includes(args.isMax as never); - await powershell(`${WINDOW_API}; ${findWindow(title)}; [RpaWindowExtended]::SetForegroundWindow($p.MainWindowHandle) | Out-Null; ${maximize ? "[RpaWindowExtended]::ShowWindow($p.MainWindowHandle,3) | Out-Null" : ""}`); - return windowObject(title); -}); - -register(["Window_GetWindowMessage", "Window_GetWindowMessage_v1"], async (_ctx, node, args) => { - const object = await windowObject(requiredTitle(args, node.id, node.componentId)); - const type = value(args, ["switchType"], "all").toLowerCase(); - if (type === "title") return object.title; - if (type === "pid") return object.pid; - if (type === "position" || type === "location") return [object.x, object.y]; - if (type === "size") return [object.width, object.height]; - return object; -}); - -register(["Window_SetPos", "Window_MoveWindow", "Window_MoveWindow_v1", "Window_SizeWindow"], async (_ctx, node, args) => { - const title = requiredTitle(args, node.id, node.componentId); - const current = await windowObject(title); - const moveOnly = node.componentId.includes("MoveWindow"); - const sizeOnly = node.componentId === "Window_SizeWindow"; - const position = args.location !== undefined ? parsePoint(args.location, "窗口位置", node.id, node.componentId) : [Number(args.x ?? current.x), Number(args.y ?? current.y)] as [number, number]; - const sizeCandidate = args.size ?? args.window_size; - const size = sizeCandidate !== undefined ? parsePoint(sizeCandidate, "窗口尺寸", node.id, node.componentId) : [Number(args.width ?? current.width), Number(args.height ?? current.height)] as [number, number]; - const x = sizeOnly ? Number(current.x) : position[0]; - const y = sizeOnly ? Number(current.y) : position[1]; - const width = moveOnly ? Number(current.width) : size[0]; - const height = moveOnly ? Number(current.height) : size[1]; - if (!(width > 0 && height > 0)) throw new RuntimeError("validation_error", "窗口尺寸必须大于 0", { nodeId: node.id, componentId: node.componentId }); - await powershell(`${WINDOW_API}; ${findWindow(title)}; [RpaWindowExtended]::SetWindowPos($p.MainWindowHandle,[IntPtr]::Zero,${x},${y},${width},${height},4) | Out-Null`); - return { ...(await windowObject(title)), x, y, width, height }; -}); - -/** 生成从当前位置到目标的插值路径点（JS 侧算好，PS 侧只执行循环） */ -function mousePath(from: [number, number], to: [number, number], mode: string, speed: number): Array<[number, number]> { - const dx = to[0] - from[0]; - const dy = to[1] - from[1]; - const distance = Math.hypot(dx, dy); - if (distance < 2) return [to]; - const stepCount = Math.max(6, Math.min(50, Math.round(distance / 12))); - const points: Array<[number, number]> = []; - for (let i = 1; i <= stepCount; i++) { - const t = i / stepCount; - // 模拟人工：缓入缓出 + 轻微垂直抖动；匀速直线：线性 - const ease = mode === "0" ? (t < 0.5 ? 2 * t * t : 1 - Math.pow(-2 * t + 2, 2) / 2) : t; - const jitter = mode === "0" ? Math.sin(t * Math.PI) * (Math.sin(i * 12.9898) * 43758.5453 % 1) * 3 : 0; - points.push([ - Math.round(from[0] + dx * ease - (dy / distance) * jitter), - Math.round(from[1] + dy * ease + (dx / distance) * jitter) - ]); - } - points.push(to); - return points; -} - /** 界面(框选/元素拾取/截图)产出的是 DIP 逻辑坐标；DPI 感知进程的 SetCursorPos 用物理像素， * 缩放屏(125%/150%)下直接混用会成比例偏移(用户实测 125% 下差约 25% 距离)。 * 统一在执行前换算；宿主桥未注入时(vitest)回退原值保持旧行为 */ @@ -160,6 +77,41 @@ function physicalToDip(x: number, y: number): [number, number] { return [Math.round(x), Math.round(y)]; } +const getWindow: CapabilityHandler = async (_ctx, node, args) => driver.window.findByTitle(requiredTitle(args, node.id, node.componentId)); +register(["Window_Get", "Window_GetWindow", "Window_GetWindow_v1"], getWindow); + +register("Window_ActiveWindow_v1", async (_ctx, node, args) => { + const title = requiredTitle(args, node.id, node.componentId); + const maximize = [true, 1, "1", "true", "True"].includes(args.isMax as never); + return driver.window.activate(title, { maximize }); +}); + +register(["Window_GetWindowMessage", "Window_GetWindowMessage_v1"], async (_ctx, node, args) => { + const object = await driver.window.findByTitle(requiredTitle(args, node.id, node.componentId)); + const type = value(args, ["switchType"], "all").toLowerCase(); + if (type === "title") return object.title; + if (type === "pid") return object.pid; + if (type === "position" || type === "location") return [object.x, object.y]; + if (type === "size") return [object.width, object.height]; + return object; +}); + +register(["Window_SetPos", "Window_MoveWindow", "Window_MoveWindow_v1", "Window_SizeWindow"], async (_ctx, node, args) => { + const title = requiredTitle(args, node.id, node.componentId); + const current = await driver.window.findByTitle(title); + const moveOnly = node.componentId.includes("MoveWindow"); + const sizeOnly = node.componentId === "Window_SizeWindow"; + const position = args.location !== undefined ? parsePoint(args.location, "窗口位置", node.id, node.componentId) : [Number(args.x ?? current.x), Number(args.y ?? current.y)] as [number, number]; + const sizeCandidate = args.size ?? args.window_size; + const size = sizeCandidate !== undefined ? parsePoint(sizeCandidate, "窗口尺寸", node.id, node.componentId) : [Number(args.width ?? current.width), Number(args.height ?? current.height)] as [number, number]; + const x = sizeOnly ? Number(current.x) : position[0]; + const y = sizeOnly ? Number(current.y) : position[1]; + const width = moveOnly ? Number(current.width) : size[0]; + const height = moveOnly ? Number(current.height) : size[1]; + if (!(width > 0 && height > 0)) throw new RuntimeError("validation_error", "窗口尺寸必须大于 0", { nodeId: node.id, componentId: node.componentId }); + return driver.window.setPos(title, { x, y, width, height }); +}); + const moveMouse: CapabilityHandler = async (_ctx, node, args) => { // 注意：position 字段是「坐标相对位置」(0=屏幕左上角/1=激活窗口左上角)，坐标本体在 location const [rawX, rawY] = parsePoint(args.location ?? args.point, "鼠标坐标", node.id, node.componentId); @@ -170,33 +122,22 @@ const moveMouse: CapabilityHandler = async (_ctx, node, args) => { let dipTarget: [number, number]; if (relative) { // 相对激活窗口左上角：窗口矩形是物理像素，换回 DIP 再叠加偏移，最后统一转物理 - const output = await powershell(`${WINDOW_API}; $fw = [RpaWindowExtended]::GetForegroundWindow(); $r = New-Object RpaWindowExtended+RECT; [void][RpaWindowExtended]::GetWindowRect($fw, [ref]$r); "$($r.Left),$($r.Top)"`); - const [wx, wy] = output.split(",").map(Number); + const { x: wx, y: wy } = await driver.window.foregroundPosition(); if (!Number.isFinite(wx) || !Number.isFinite(wy)) throw new RuntimeError("browser_error", "无法获取激活窗口位置", { nodeId: node.id, componentId: node.componentId }); const [dipWx, dipWy] = physicalToDip(wx, wy); dipTarget = [dipWx + rawX, dipWy + rawY]; } else { dipTarget = [rawX, rawY]; } - return moveToSmooth(dipTarget, moveType, speed); -}; - -async function moveToSmooth(dipTarget: [number, number], moveType: string, speed: number): Promise<{ x: number; y: number }> { const [tx, ty] = dipToPhysical(dipTarget[0], dipTarget[1]); if (moveType === "2") { - await powershell(`${WINDOW_API}; [void][RpaWindowExtended]::SetCursorPos(${tx},${ty})`); - return { x: Math.round(dipTarget[0]), y: Math.round(dipTarget[1]) }; + await driver.mouse.moveTo({ x: tx, y: ty }); + } else { + // 平滑移动：从当前光标位置出发，按速度插值（路径全程物理像素，与目标同一坐标系） + await driver.mouse.moveSmooth({ x: tx, y: ty }, { ease: moveType === "0", speed }); } - // 平滑移动：从当前光标位置出发，按速度插值（路径全程物理像素，与目标同一坐标系） - const delay = Math.min(80, Math.max(4, Math.round(1500 / speed))); - const output = await powershell(`${WINDOW_API}; $cur = [RpaWindowExtended+POINT]::new(); [void][RpaWindowExtended]::GetCursorPos([ref]$cur); "$($cur.X),$($cur.Y)"`); - const [cx, cy] = output.split(",").map(Number); - const from: [number, number] = Number.isFinite(cx) && Number.isFinite(cy) ? [cx, cy] : [0, 0]; - const path = mousePath(from, [tx, ty], moveType, speed); - const ptsLiteral = path.map(([px, py]) => `@(${px},${py})`).join(","); - await powershell(`${WINDOW_API}; $pts = @(${ptsLiteral}); foreach ($pt in $pts) { [void][RpaWindowExtended]::SetCursorPos($pt[0], $pt[1]); Start-Sleep -Milliseconds ${delay} }`); return { x: Math.round(dipTarget[0]), y: Math.round(dipTarget[1]) }; -} +}; register(["Mouse_MoveToV1", "Mouse_MoveToV2"], moveMouse); const dragMouse: CapabilityHandler = async (_ctx, node, args) => { @@ -205,9 +146,8 @@ const dragMouse: CapabilityHandler = async (_ctx, node, args) => { const [spx, spy] = dipToPhysical(sx, sy); const [epx, epy] = dipToPhysical(ex, ey); const right = ["1", "right"].includes(value(args, ["side"], "0").toLowerCase()); - const down = right ? 8 : 2; - const up = right ? 16 : 4; - await powershell(`${WINDOW_API}; [RpaWindowExtended]::SetCursorPos(${spx},${spy}) | Out-Null; [RpaWindowExtended]::mouse_event(${down},0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 80; [RpaWindowExtended]::SetCursorPos(${epx},${epy}) | Out-Null; [RpaWindowExtended]::mouse_event(${up},0,0,0,[UIntPtr]::Zero)`); + // 旧实现按下后停 80ms、拖到终点立即松开（与桌面版 Window_Mouse 不同，松开前不停顿） + await driver.mouse.drag({ x: spx, y: spy }, { x: epx, y: epy }, { button: right ? "right" : "left", holdMs: 80, releaseDelayMs: 0 }); return { start: [Math.round(sx), Math.round(sy)], end: [Math.round(ex), Math.round(ey)] }; }; register(["Mouse_DragToV1", "Mouse_DragToV2", "Mouse_DragToNew"], dragMouse); @@ -218,9 +158,7 @@ register(["Mouse_ClickCombine", "Mouse_ClickCombine_v1", "Mouse_ClickCombine_v2" const [px, py] = dipToPhysical(x, y); const right = value(args, ["click_key"], "left").toLowerCase() === "right"; const doubleClick = value(args, ["click_time"], "click").toLowerCase().includes("double"); - const down = right ? 8 : 2; - const up = right ? 16 : 4; - const repeats = doubleClick ? 2 : 1; - await powershell(`${WINDOW_API}; [RpaWindowExtended]::SetCursorPos(${px},${py}) | Out-Null; 1..${repeats} | ForEach-Object { [RpaWindowExtended]::mouse_event(${down},0,0,0,[UIntPtr]::Zero); [RpaWindowExtended]::mouse_event(${up},0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 80 }`); - return { x: Math.round(x), y: Math.round(y), button: right ? "right" : "left", clicks: repeats }; + // 旧实现每次 down-up 之后停 80ms + await driver.mouse.click({ x: px, y: py }, { button: right ? "right" : "left", double: doubleClick, settleMs: 80 }); + return { x: Math.round(x), y: Math.round(y), button: right ? "right" : "left", clicks: doubleClick ? 2 : 1 }; }); diff --git a/packages/rpa-core/src/capabilities/word-extended.ts b/packages/rpa-core/src/capabilities/word-extended.ts index 6c5f1c3..6954b19 100644 --- a/packages/rpa-core/src/capabilities/word-extended.ts +++ b/packages/rpa-core/src/capabilities/word-extended.ts @@ -2,7 +2,7 @@ import { promises as fs } from "node:fs"; import path from "node:path"; import { getHandler, register } from "../engine/registry"; import { RuntimeError } from "../engine/types"; -import { powershell, psString } from "./desktop"; +import { powershell, psString } from "../driver/windows/powershell"; interface WordHandle { path: string; } diff --git a/packages/rpa-core/src/capabilities/word.ts b/packages/rpa-core/src/capabilities/word.ts index 6da20c6..5a5a086 100644 --- a/packages/rpa-core/src/capabilities/word.ts +++ b/packages/rpa-core/src/capabilities/word.ts @@ -1,6 +1,6 @@ import path from "node:path"; import { register } from "../engine/registry"; -import { powershell, psString } from "./desktop"; +import { powershell, psString } from "../driver/windows/powershell"; interface WordHandle { path: string; } diff --git a/packages/rpa-core/src/driver/types.ts b/packages/rpa-core/src/driver/types.ts new file mode 100644 index 0000000..8e58474 --- /dev/null +++ b/packages/rpa-core/src/driver/types.ts @@ -0,0 +1,165 @@ +/** + * AutomationDriver 驱动抽象（任务 3.2，决策点 6：五域接口 + 机械包一层）。 + * + * 接口签名以现有 PowerShell 能力的输入输出为准，不重新发明： + * WindowsDriver（driver/windows/windows-driver.ts）是 PowerShell 实现，行为与 + * 改造前完全一致；引擎创建时注入 driver（createEngine 可注入自定义实现）。 + * + * 约定：driver 只承载「原始注入原语」（单个 SendKeys/一次 SetCursorPos/一次 + * 窗口操作）；带拟真抖动、坐标换算、多步组合的「策略」仍留在能力文件， + * 保证机械替换不改任何时序行为。 + */ + +/** UIA 选择器（element 域类型，能力与 driver 共用） */ +export interface UiaSelector { + window?: string; + name?: string; + id?: string; + /** .NET ControlType 字段名（已映射，如 ComboBox/ListItem/DataGrid） */ + type?: string; + /** 用户输入的原始类型字符串 */ + typeRaw?: string; + className?: string; + screen?: { x: number; y: number; w: number; h: number }; + /** Relative physical rect inside the top-level window. */ + rel?: { x: number; y: number; w: number; h: number }; + process?: string; + winClass?: string; + enriched?: boolean; + aiType?: string; + aiText?: string; + ai?: boolean; + /** 从窗口（或桌面根）到目标元素的祖先路径，逐段用稳定属性定位 */ + path?: Array<{ type?: string; name?: string; id?: string; className?: string; index?: number }>; + index: number; +} + +export interface DriverPoint { + x: number; + y: number; +} + +export interface DriverRect { + x: number; + y: number; + width: number; + height: number; +} + +export interface WindowInfo extends DriverRect { + pid: number; + title: string; + hwnd: number; +} + +export interface WindowSummary { + pid: number; + title: string; + hwnd: number; +} + +export type MouseButton = "left" | "right"; + +export interface MouseClickOptions { + button?: MouseButton; + double?: boolean; + /** 每次 down-up 后的停顿毫秒（0 = 不停顿，保持旧桌面能力的紧凑时序） */ + settleMs?: number; +} + +export interface MouseDragOptions { + button?: MouseButton; + /** 按下后的按住毫秒 */ + holdMs?: number; + /** 拖到终点后、松开前的停顿毫秒 */ + releaseDelayMs?: number; +} + +export interface MouseSmoothOptions { + /** true=缓入缓出+轻微抖动（拟真），false=匀速直线 */ + ease?: boolean; + /** 1~100，越大越快 */ + speed?: number; +} + +/** mouse 域：坐标一律为物理像素（DIP 换算属于能力层策略） */ +export interface MouseDriver { + moveTo(point: DriverPoint): Promise; + click(point: DriverPoint, options?: MouseClickOptions): Promise; + drag(from: DriverPoint, to: DriverPoint, options?: MouseDragOptions): Promise; + scroll(amount: number): Promise; + /** 从当前光标位置出发按速度插值移动到目标 */ + moveSmooth(target: DriverPoint, options?: MouseSmoothOptions): Promise; +} + +export interface KeyboardSendOptions { + keys: string; + intervalMs: number; + /** api=后台接口整段发送 / manual=逐字符拟真 / clipboard=剪贴板粘贴 */ + mode: "api" | "manual" | "clipboard"; + /** 文本类组件做 SendKeys 字面转义；按键组合组件原样发送 */ + textMode: boolean; + /** 目标窗口标题（非空时先多招式激活再输入，与旧行为一致） */ + activateTitle?: string; +} + +/** keyboard 域 */ +export interface KeyboardDriver { + send(options: KeyboardSendOptions): Promise; +} + +export interface WindowDriver { + /** 按标题模糊匹配窗口并返回矩形信息（GetWindowRect，物理像素） */ + findByTitle(title: string): Promise; + /** 简单激活（SetForegroundWindow + 可选最大化），返回窗口信息 */ + activate(title: string, options?: { maximize?: boolean }): Promise; + /** 多招式前台激活（失败抛错），返回窗口主标题 */ + focus(title: string): Promise; + /** 前台窗口标题 */ + foregroundTitle(): Promise; + /** 前台窗口矩形左上角（物理像素） */ + foregroundPosition(): Promise; + exists(title: string): Promise; + /** CloseMainWindow 结果（"True"/"False"） */ + close(title: string): Promise; + /** ShowWindow 命令（6=最小化 3=最大化 9=恢复 5=显示 0=隐藏） */ + setState(title: string, showCommand: number): Promise; + /** 只改尺寸，保持左上角不动 */ + setSize(title: string, width: number, height: number): Promise; + setPos(title: string, rect: DriverRect): Promise; + list(filter?: { title?: string; regex?: boolean }): Promise; +} + +/** screen 域（截图等 System.Drawing 组合操作不在接口面内，能力层可继续用 powershell()） */ +export interface ScreenDriver { + /** 主屏边界（System.Windows.Forms.Screen.PrimaryScreen.Bounds） */ + size(): Promise<{ x: number; y: number; width: number; height: number }>; +} + +export interface ElementFindOptions { + timeoutMs?: number; + gapMs?: number; +} + +/** + * element 域（UIA）。PowerShell 实现承载在 capabilities/desktop-uia.ts + * （ground 原语集），WindowsDriver.element 委托该实现；能力 handler 层 + * 到 driver 形式的收口在阶段 5/6 按需进行，避免行为回归。 + */ +export interface ElementDriver { + parseSelector(raw: unknown): UiaSelector; + find(selector: UiaSelector, options?: ElementFindOptions): Promise | null>; + exists(selector: UiaSelector): Promise; + click(selector: UiaSelector, options?: ElementFindOptions): Promise; + setValue(selector: UiaSelector, value: string): Promise; + getValue(selector: UiaSelector): Promise | null>; +} + +/** 五域聚合，注入点（RpaEngine.driver） */ +export interface AutomationDriver { + readonly mouse: MouseDriver; + readonly keyboard: KeyboardDriver; + readonly window: WindowDriver; + readonly screen: ScreenDriver; + readonly element: ElementDriver; +} diff --git a/packages/rpa-core/src/driver/windows/powershell.ts b/packages/rpa-core/src/driver/windows/powershell.ts new file mode 100644 index 0000000..94af0b4 --- /dev/null +++ b/packages/rpa-core/src/driver/windows/powershell.ts @@ -0,0 +1,74 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { RuntimeError } from "../../engine/types"; + +/** + * PowerShell 进程助手（任务 3.1 从 capabilities/desktop.ts 拆出）。 + * + * 包内所有 PowerShell 调用必须经此助手（开发规范 v1 第二节第 3 条）， + * 禁止各能力文件自行 execFile("powershell.exe", ...)； + * 阶段 3.5（可选）将统一切换为常驻 worker，接口签名不变。 + */ + +const execFileAsync = promisify(execFile); + +function encoded(command: string): string { + return Buffer.from(command, "utf16le").toString("base64"); +} + +/** 从 PowerShell 错误输出里提取 CLIXML 段的真实错误文本（否则显示为乱码转义序列） */ +function decodeCliXml(detail: string): string { + const lines = [...detail.matchAll(/(.*?)<\/S>/gs)].map((m) => m[1]); + if (lines.length === 0) return detail; + // _xHHHH_ 是 CLIXML 转义 + const decoded = lines.join("\n").replace(/_x([0-9A-Fa-f]{4})_/g, (_, h) => String.fromCharCode(parseInt(h, 16))); + const parts = decoded.split("\n").map((l) => l.trim()); + const position = parts.filter((l) => l.startsWith("所在位置")).slice(0, 1); + const message = parts.filter( + (l) => l && !l.startsWith("所在位置") && !l.startsWith("+") && !l.startsWith("~~") && !l.includes("CategoryInfo") && !l.includes("FullyQualifiedErrorId") && !l.startsWith(" { + try { + // Windows PowerShell 的默认输出编码依机器代码页变化；Node 按 UTF-8 解码管道。 + const utf8 = "$ProgressPreference='SilentlyContinue'; [Console]::OutputEncoding=[System.Text.UTF8Encoding]::new($false); $OutputEncoding=[Console]::OutputEncoding; "; + const result = await execFileAsync("powershell.exe", ["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-EncodedCommand", encoded(utf8 + command)], { + windowsHide: true, + timeout, + maxBuffer: 4 * 1024 * 1024, + encoding: "buffer" + } as never); + return decodePipe(result.stdout as unknown as Buffer).trim(); + } catch (error) { + const err = error as Error & { stdout?: unknown; stderr?: unknown }; + // 脚本主动失败（RpaFail）把完整错误写在 stdout 的 RPA_ERROR:: 标记后，不经 CLIXML 渲染，无截断 + const stdoutText = Buffer.isBuffer(err.stdout) ? decodePipe(err.stdout) : String(err.stdout ?? ""); + const marked = stdoutText.match(/RPA_ERROR::([\s\S]*)/); + if (marked) { + const detail = marked[1].trim(); + throw new RuntimeError("desktop_command_failed", `桌面操作失败：${detail}`, { detail }); + } + const stderrBuf = Buffer.isBuffer(err.stderr) ? err.stderr : Buffer.from(String(err.stderr ?? ""), "utf8"); + const stderrText = stderrBuf.length > 0 ? decodePipe(stderrBuf) : err.message; + const detail = decodeCliXml(stderrText); + throw new RuntimeError("desktop_command_failed", `桌面操作失败：${detail}`, { detail }); + } +} + +export function psString(value: unknown): string { + return `'${String(value ?? "").replace(/'/g, "''")}'`; +} diff --git a/packages/rpa-core/src/driver/windows/windows-driver.ts b/packages/rpa-core/src/driver/windows/windows-driver.ts new file mode 100644 index 0000000..43b70f5 --- /dev/null +++ b/packages/rpa-core/src/driver/windows/windows-driver.ts @@ -0,0 +1,272 @@ +import type { + AutomationDriver, + DriverPoint, + DriverRect, + ElementDriver, + KeyboardSendOptions, + MouseClickOptions, + MouseDragOptions, + MouseSmoothOptions, + MouseDriver, + KeyboardDriver, + ScreenDriver, + WindowDriver, + WindowInfo, + WindowSummary +} from "../types"; +import { powershell, psString } from "./powershell"; + +/** + * WindowsDriver（任务 3.3）：AutomationDriver 的 PowerShell 实现。 + * 脚本与改造前各能力文件内的片段逐字一致（仅 Add-Type 块收拢到本文件）， + * 行为不变；坐标一律物理像素，DIP 换算等策略留在能力层。 + */ + +/** 基础注入原语（原 capabilities/desktop.ts 的 USER32 块，补 SetProcessDpiAwarenessContext： + * 原 window-extended.ts 的 WINDOW_API 在同一进程内已设置 per-monitor DPI 感知， + * 驱动统一携带该声明，保证 SetCursorPos 全程物理像素语义一致） */ +const USER32 = `Add-Type @' +using System; +using System.Runtime.InteropServices; +public static class RpaUser32 { + [DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y); + [DllImport("user32.dll")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra); + [DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); + [DllImport("user32.dll", CharSet=CharSet.Unicode)] public static extern int GetWindowText(IntPtr hWnd, System.Text.StringBuilder text, int count); + [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd); + [DllImport("user32.dll")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value); +} +'@; try { [void][RpaUser32]::SetProcessDpiAwarenessContext([IntPtr](-4)) } catch {}`; + +/** 窗口矩形/鼠标原语（原 capabilities/window-extended.ts 的 WINDOW_API 块） */ +const WINDOW_API = `Add-Type @' +using System; +using System.Runtime.InteropServices; +public static class RpaWindowExtended { + [StructLayout(LayoutKind.Sequential)] public struct RECT { public int Left; public int Top; public int Right; public int Bottom; } + [StructLayout(LayoutKind.Sequential)] public struct POINT { public int X; public int Y; } + [DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr hWnd, out RECT rect); + [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd); + [DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr hWnd, int command); + [DllImport("user32.dll")] public static extern bool SetWindowPos(IntPtr hWnd, IntPtr after, int x, int y, int width, int height, uint flags); + [DllImport("user32.dll")] public static extern bool SetCursorPos(int x, int y); + [DllImport("user32.dll")] public static extern bool GetCursorPos(out POINT point); + [DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); + [DllImport("user32.dll")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra); + [DllImport("user32.dll")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value); +} +'@; try { [void][RpaWindowExtended]::SetProcessDpiAwarenessContext([IntPtr](-4)) } catch {}`; + +function findWindowScript(title: string): string { + return `$p = Get-Process | Where-Object { $_.MainWindowHandle -ne 0 -and $_.MainWindowTitle -like ${psString(`*${title}*`)} } | Select-Object -First 1; if (-not $p) { throw '找不到窗口：${title.replace(/'/g, "''")}' }`; +} + +async function windowObject(title: string): Promise { + const output = await powershell(`${WINDOW_API}; ${findWindowScript(title)}; $r = New-Object RpaWindowExtended+RECT; [RpaWindowExtended]::GetWindowRect($p.MainWindowHandle,[ref]$r) | Out-Null; @{pid=$p.Id;title=$p.MainWindowTitle;hwnd=$p.MainWindowHandle.ToInt64();x=$r.Left;y=$r.Top;width=$r.Right-$r.Left;height=$r.Bottom-$r.Top} | ConvertTo-Json -Compress`); + return JSON.parse(output); +} + +/** 把匹配标题的窗口带到前台；多招式依次兜底（应用点击「运行」后自身是前台， + * Windows 前台锁定策略会拒绝普通 SetForegroundWindow，最小化再恢复可绕过）。 + * 失败用 RPA_ERROR 标记写 stdout 后 exit 1：不走 throw/CLIXML，避免控制台宽度截断和编码歧义 */ +function activateWindowScript(selector: string): string { + const safeTitle = selector.replace(/'/g, "''"); + return `function RpaFail($m) { [Console]::Out.WriteLine('RPA_ERROR::' + $m); exit 1 } +$p = Get-Process | Where-Object { $_.MainWindowTitle -and $_.MainWindowTitle -like ${psString(`*${selector}*`)} } | Select-Object -First 1 +if (-not $p) { $vis = (Get-Process | Where-Object { $_.MainWindowTitle } | Select-Object -First 12 | ForEach-Object { '『' + $_.MainWindowTitle + '』' }) -join ' '; RpaFail ('找不到窗口：${safeTitle}。当前可见窗口标题: ' + $vis) } +Add-Type -MemberDefinition '[DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); [DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr h, int cmd); [DllImport("user32.dll")] public static extern bool IsIconic(IntPtr h); [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr h); [DllImport("user32.dll")] public static extern void keybd_event(byte bVk, byte bScan, uint dwFlags, UIntPtr dwExtraInfo); [DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr hWnd, IntPtr pid); [DllImport("user32.dll")] public static extern bool AttachThreadInput(uint a, uint b, bool attach); [DllImport("user32.dll")] public static extern void SwitchToThisWindow(IntPtr hWnd, bool fAltTab); [DllImport("kernel32.dll")] public static extern uint GetCurrentThreadId();' -Name RpaFocus2 -Namespace WinRpaFocus2 | Out-Null +$h = $p.MainWindowHandle +if ([WinRpaFocus2.RpaFocus2]::IsIconic($h)) { [void][WinRpaFocus2.RpaFocus2]::ShowWindow($h, 9); Start-Sleep -Milliseconds 150 } +if ([WinRpaFocus2.RpaFocus2]::GetForegroundWindow() -ne $h) { + [void][WinRpaFocus2.RpaFocus2]::SwitchToThisWindow($h, $true) + Start-Sleep -Milliseconds 150 +} +if ([WinRpaFocus2.RpaFocus2]::GetForegroundWindow() -ne $h) { + [void][WinRpaFocus2.RpaFocus2]::keybd_event(0x12, 0, 0, [UIntPtr]::Zero) + [void][WinRpaFocus2.RpaFocus2]::SetForegroundWindow($h) + [void][WinRpaFocus2.RpaFocus2]::keybd_event(0x12, 0, 2, [UIntPtr]::Zero) + Start-Sleep -Milliseconds 200 +} +if ([WinRpaFocus2.RpaFocus2]::GetForegroundWindow() -ne $h) { + $fgThread = [WinRpaFocus2.RpaFocus2]::GetWindowThreadProcessId([WinRpaFocus2.RpaFocus2]::GetForegroundWindow(), [IntPtr]::Zero) + $thisThread = [WinRpaFocus2.RpaFocus2]::GetCurrentThreadId() + $attached = [WinRpaFocus2.RpaFocus2]::AttachThreadInput($thisThread, $fgThread, $true) + try { [void][WinRpaFocus2.RpaFocus2]::SetForegroundWindow($h) } finally { if ($attached) { [void][WinRpaFocus2.RpaFocus2]::AttachThreadInput($thisThread, $fgThread, $false) } } + Start-Sleep -Milliseconds 200 +} +if ([WinRpaFocus2.RpaFocus2]::GetForegroundWindow() -ne $h) { + [void][WinRpaFocus2.RpaFocus2]::ShowWindow($h, 6) + Start-Sleep -Milliseconds 150 + [void][WinRpaFocus2.RpaFocus2]::ShowWindow($h, 9) + Start-Sleep -Milliseconds 300 +} +if ([WinRpaFocus2.RpaFocus2]::GetForegroundWindow() -ne $h) { RpaFail '无法把窗口带到前台，请确认窗口未被关闭或全屏占用' }`; +} + +/** 把 SendKeys 特殊字符按字面转义（+^%~(){}[] → {+} 等）；SendKeys 用花括号表示字面量 */ +function sendKeysEscape(ch: string): string { + return "+^%~(){}[]".includes(ch) ? `{${ch}}` : ch; +} + +/** 把字符包装成 PowerShell 安全字面量。 + * ASCII 直接用单引号字符串（撇号内部 '' 转义）；非 ASCII 用 [char]0xXXXX 数值构造—— + * Windows PowerShell 5.1 会把 ‘ ’ “ ” 等智能引号当作引号变体解析，直接放进字符串字面量会报「字符串缺少终止符」。 */ +function psCharLiteral(text: string): string { + if (/^[\x20-\x7E]*$/.test(text)) return psString(text); + return Array.from(text).map((c) => `[char]0x${c.codePointAt(0)!.toString(16).padStart(4, "0")}`).join("+") || psString(""); +} + +/** 生成从当前位置到目标的插值路径点（JS 侧算好，PS 侧只执行循环）（原 window-extended.ts 的 mousePath） */ +function mousePath(from: DriverPoint, to: DriverPoint, mode: string, speed: number): Array<[number, number]> { + const dx = to.x - from.x; + const dy = to.y - from.y; + const distance = Math.hypot(dx, dy); + if (distance < 2) return [[to.x, to.y]]; + const stepCount = Math.max(6, Math.min(50, Math.round(distance / 12))); + const points: Array<[number, number]> = []; + for (let i = 1; i <= stepCount; i++) { + const t = i / stepCount; + // 模拟人工：缓入缓出 + 轻微垂直抖动；匀速直线：线性 + const ease = mode === "0" ? (t < 0.5 ? 2 * t * t : 1 - Math.pow(-2 * t + 2, 2) / 2) : t; + const jitter = mode === "0" ? Math.sin(t * Math.PI) * (Math.sin(i * 12.9898) * 43758.5453 % 1) * 3 : 0; + points.push([ + Math.round(from.x + dx * ease - (dy / distance) * jitter), + Math.round(from.y + dy * ease + (dx / distance) * jitter) + ]); + } + points.push([to.x, to.y]); + return points; +} + +const windowsMouse: MouseDriver = { + async moveTo(point) { + await powershell(`${USER32}; [RpaUser32]::SetCursorPos(${point.x},${point.y})`); + }, + async click(point, options: MouseClickOptions = {}) { + const right = options.button === "right"; + const down = right ? 0x0008 : 0x0002; + const up = right ? 0x0010 : 0x0004; + const repeats = options.double ? 2 : 1; + const settle = options.settleMs && options.settleMs > 0 ? `; Start-Sleep -Milliseconds ${Math.round(options.settleMs)}` : ""; + await powershell(`${USER32}; [RpaUser32]::SetCursorPos(${point.x},${point.y});${Array.from({ length: repeats }, () => ` [RpaUser32]::mouse_event(${down},0,0,0,[UIntPtr]::Zero); [RpaUser32]::mouse_event(${up},0,0,0,[UIntPtr]::Zero)${settle}`).join(";")}`); + }, + async drag(from, to, options: MouseDragOptions = {}) { + const right = options.button === "right"; + const down = right ? 0x0008 : 0x0002; + const up = right ? 0x0010 : 0x0004; + const hold = Math.max(0, Math.round(options.holdMs ?? 80)); + const release = Math.max(0, Math.round(options.releaseDelayMs ?? 80)); + await powershell(`${USER32}; [RpaUser32]::SetCursorPos(${from.x},${from.y}); [RpaUser32]::mouse_event(${down},0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds ${hold}; [RpaUser32]::SetCursorPos(${to.x},${to.y});${release > 0 ? ` Start-Sleep -Milliseconds ${release};` : ""} [RpaUser32]::mouse_event(${up},0,0,0,[UIntPtr]::Zero)`); + }, + async scroll(amount) { + await powershell(`${USER32}; [RpaUser32]::mouse_event(0x0800,0,0,${amount},[UIntPtr]::Zero)`); + }, + async moveSmooth(target, options: MouseSmoothOptions = {}) { + const ease = options.ease !== false; + const speed = Math.min(100, Math.max(1, options.speed ?? 35)); + const delay = Math.min(80, Math.max(4, Math.round(1500 / speed))); + // 从当前光标位置出发，按速度插值（路径全程物理像素，与目标同一坐标系） + const output = await powershell(`${WINDOW_API}; $cur = [RpaWindowExtended+POINT]::new(); [void][RpaWindowExtended]::GetCursorPos([ref]$cur); "$($cur.X),$($cur.Y)"`); + const [cx, cy] = output.split(",").map(Number); + const from: DriverPoint = Number.isFinite(cx) && Number.isFinite(cy) ? { x: cx, y: cy } : { x: 0, y: 0 }; + const path = mousePath(from, target, ease ? "0" : "1", speed); + const ptsLiteral = path.map(([px, py]) => `@(${px},${py})`).join(","); + await powershell(`${WINDOW_API}; $pts = @(${ptsLiteral}); foreach ($pt in $pts) { [void][RpaWindowExtended]::SetCursorPos($pt[0], $pt[1]); Start-Sleep -Milliseconds ${delay} }`); + } +}; + +const windowsKeyboard: KeyboardDriver = { + async send(options: KeyboardSendOptions) { + // 三种输入方式：脚本尽量简单，转义逻辑放 JS 侧，避免 PowerShell 字符串拼接的语法陷阱 + let send: string; + if (options.mode === "clipboard") { + // 剪贴板粘贴：原文进剪贴板 + Ctrl+V，不做 SendKeys 转义；内容含智能引号等非 ASCII 时同样走 [char] 构造 + const contentExpr = /^[\x20-\x7E]*$/.test(options.keys) ? psString(options.keys) : `(-join @(${Array.from(options.keys).map((c) => psCharLiteral(c)).join(",")}))`; + send = `Set-Clipboard -Value ${contentExpr}; Start-Sleep -Milliseconds 120; [System.Windows.Forms.SendKeys]::SendWait('^v')`; + } else if (options.mode === "manual" && options.textMode) { + // 人工模拟：逐字符；JS 侧把每个字符转义成 SendKeys 字面量，PS 侧只做数组循环 + const escapedChars = Array.from(options.keys).map((ch) => sendKeysEscape(ch)); + const arr = escapedChars.map((c) => psCharLiteral(c)).join(","); + send = `$chars = @(${arr}); foreach ($c in $chars) { [System.Windows.Forms.SendKeys]::SendWait([string]$c); Start-Sleep -Milliseconds ${options.intervalMs} }`; + } else { + // 后台接口 / 按键组合 / 人工模拟(按键组合组件)：整段发送，文本特殊字符转义 + const escaped = options.textMode ? Array.from(options.keys).map(sendKeysEscape).join("") : options.keys; + const contentExpr = /^[\x20-\x7E]*$/.test(escaped) ? psString(escaped) : `(-join @(${Array.from(escaped).map((c) => psCharLiteral(c)).join(",")}))`; + send = `[System.Windows.Forms.SendKeys]::SendWait(${contentExpr})`; + } + const activate = options.activateTitle ? `${activateWindowScript(options.activateTitle)}; Start-Sleep -Milliseconds 300; ` : ""; + await powershell(`${USER32}; ${activate}Add-Type -AssemblyName System.Windows.Forms; ${send}`); + } +}; + +const windowsWindow: WindowDriver = { + findByTitle: (title) => windowObject(title), + async activate(title, options: { maximize?: boolean } = {}) { + const maximize = options.maximize === true; + await powershell(`${WINDOW_API}; ${findWindowScript(title)}; [RpaWindowExtended]::SetForegroundWindow($p.MainWindowHandle) | Out-Null; ${maximize ? "[RpaWindowExtended]::ShowWindow($p.MainWindowHandle,3) | Out-Null" : ""}`); + return windowObject(title); + }, + async focus(title) { + return powershell(`${USER32}\n${activateWindowScript(title)}\n$p.MainWindowTitle`); + }, + async foregroundTitle() { + return powershell(`${USER32}; $sb = New-Object System.Text.StringBuilder 512; [RpaUser32]::GetWindowText([RpaUser32]::GetForegroundWindow(),$sb,$sb.Capacity) | Out-Null; $sb.ToString()`); + }, + async foregroundPosition(): Promise { + const output = await powershell(`${WINDOW_API}; $fw = [RpaWindowExtended]::GetForegroundWindow(); $r = New-Object RpaWindowExtended+RECT; [void][RpaWindowExtended]::GetWindowRect($fw, [ref]$r); "$($r.Left),$($r.Top)"`); + const [x, y] = output.split(",").map(Number); + return { x, y }; + }, + async exists(title) { + return (await powershell(`$p = Get-Process | Where-Object { $_.MainWindowTitle -and $_.MainWindowTitle -like ${psString(`*${title}*`)} } | Select-Object -First 1; [bool]$p`)).toLowerCase() === "true"; + }, + async close(title): Promise { + return powershell(`$p = Get-Process | Where-Object { $_.MainWindowTitle -and $_.MainWindowTitle -like ${psString(`*${title}*`)} } | Select-Object -First 1; if (-not $p) { throw '找不到窗口' }; $p.CloseMainWindow() | Out-Null; $true`); + }, + async setState(title, showCommand): Promise { + await powershell(`${WINDOW_API}; ${findWindowScript(title)}; [RpaWindowExtended]::ShowWindow($p.MainWindowHandle,${showCommand}) | Out-Null`); + }, + async setSize(title, width, height) { + await powershell(`${WINDOW_API}; ${findWindowScript(title)}; $r = New-Object RpaWindowExtended+RECT; [RpaWindowExtended]::GetWindowRect($p.MainWindowHandle,[ref]$r) | Out-Null; [RpaWindowExtended]::SetWindowPos($p.MainWindowHandle,[IntPtr]::Zero,$r.Left,$r.Top,${width},${height},4) | Out-Null`); + }, + async setPos(title, rect: DriverRect): Promise { + if (!(rect.width > 0 && rect.height > 0)) throw new Error("窗口尺寸必须大于 0"); + await powershell(`${WINDOW_API}; ${findWindowScript(title)}; [RpaWindowExtended]::SetWindowPos($p.MainWindowHandle,[IntPtr]::Zero,${rect.x},${rect.y},${rect.width},${rect.height},4) | Out-Null`); + return { ...(await windowObject(title)), x: rect.x, y: rect.y, width: rect.width, height: rect.height }; + }, + async list(filter: { title?: string; regex?: boolean } = {}) { + const title = filter.title ?? ""; + const regex = filter.regex === true; + const output = await powershell(`Get-Process | Where-Object { $_.MainWindowHandle -ne 0 ${title ? (regex ? `-and $_.MainWindowTitle -match ${psString(title)}` : `-and $_.MainWindowTitle -like ${psString(`*${title}*`)}`) : ""} } | ForEach-Object { @{pid=$_.Id;title=$_.MainWindowTitle;hwnd=$_.MainWindowHandle.ToInt64()} } | ConvertTo-Json -Compress`); + if (!output) return [] as WindowSummary[]; + const parsed = JSON.parse(output); + return (Array.isArray(parsed) ? parsed : [parsed]) as WindowSummary[]; + } +}; + +const windowsScreen: ScreenDriver = { + async size() { + const output = await powershell("Add-Type -AssemblyName System.Windows.Forms; $b=[System.Windows.Forms.Screen]::PrimaryScreen.Bounds; @{width=$b.Width;height=$b.Height;x=$b.X;y=$b.Y} | ConvertTo-Json -Compress"); + return JSON.parse(output); + } +}; + +/** element 域实现承载在 capabilities/desktop-uia.ts（阶段 3.3 说明），此处运行时注入，避免 driver → capabilities 的模块依赖 */ +let elementImplementation: ElementDriver | undefined; +export function setElementDriverImplementation(impl: ElementDriver): void { + elementImplementation = impl; +} + +export class WindowsDriver implements AutomationDriver { + readonly mouse = windowsMouse; + readonly keyboard = windowsKeyboard; + readonly window = windowsWindow; + readonly screen = windowsScreen; + + get element(): ElementDriver { + if (!elementImplementation) { + throw new Error("ElementDriver 实现未注册：请 import @rpa/core/capabilities（desktop-uia 模块自动注册）"); + } + return elementImplementation; + } +} diff --git a/packages/rpa-core/src/engine/engine.ts b/packages/rpa-core/src/engine/engine.ts index 70fe406..9f55a1e 100644 --- a/packages/rpa-core/src/engine/engine.ts +++ b/packages/rpa-core/src/engine/engine.ts @@ -1,5 +1,7 @@ import type { Client } from "basic-ftp"; import type { Workbook as ExcelJSWorkbook } from "exceljs"; +import { WindowsDriver } from "../driver/windows/windows-driver"; +import type { AutomationDriver } from "../driver/types"; /** 人为拟真操作的开关与随机参数（desktop 能力的进程级设置）。 */ export interface ManualOperationSettings { @@ -39,6 +41,8 @@ export class RpaEngine { minMs: 1000, maxMs: 3000 }; + /** 桌面自动化驱动（任务 3.4 注入点，默认 PowerShell 版 WindowsDriver） */ + readonly driver: AutomationDriver = new WindowsDriver(); } export function createEngine(): RpaEngine { diff --git a/packages/rpa-core/src/index.ts b/packages/rpa-core/src/index.ts index 1dc1f4e..739b5e2 100644 --- a/packages/rpa-core/src/index.ts +++ b/packages/rpa-core/src/index.ts @@ -44,8 +44,30 @@ export { type ExternalRemoteOptions, type ExternalConnection } from "./browser/external"; -// PowerShell 助手（阶段 3 将移入 driver/windows/）：宿主留守能力（window-extended 等）经此引用 -export { powershell, psString } from "./capabilities/desktop"; +// 桌面自动化驱动（阶段 3）：五域接口 + PowerShell 实现，引擎创建时注入 +export type { + AutomationDriver, + DriverPoint, + DriverRect, + ElementDriver, + ElementFindOptions, + KeyboardDriver, + KeyboardSendOptions, + MouseClickOptions, + MouseDragOptions, + MouseDriver, + MouseSmoothOptions, + MouseButton, + ScreenDriver, + UiaSelector, + WindowDriver, + WindowInfo, + WindowSummary +} from "./driver/types"; +export { WindowsDriver, setElementDriverImplementation } from "./driver/windows/windows-driver"; + +// PowerShell 助手（任务 3.1 已移入 driver/windows/） +export { powershell, psString } from "./driver/windows/powershell"; export type { BrowserLike, BrowserTabInfo, diff --git a/scripts/build-catalog.mjs b/scripts/build-catalog.mjs index bc0b3d4..d3cc6df 100644 --- a/scripts/build-catalog.mjs +++ b/scripts/build-catalog.mjs @@ -228,6 +228,10 @@ function normalizeVendorComponent(component, vendorMap) { const fields = (component.inputFields || []).map((f) => enrichFromVendor(normalizeVendorField(f), vendor)); const output = outputOf(component, vendor); const risk = ["web", "app", "desktop", "message", "mobile"].includes(group) ? "high" : "low"; + // 任务 3.4（决策点 7）：平台标记只填 catalog 数据，渲染层置灰延后。 + // Windows 专用组：desktop(user32/键鼠)、app(COM/进程)、office(Excel/Word COM)、vision(PowerShell 截屏)； + // data/file/message 纯 Node，web 走 CDP，均视为跨平台（null）。 + const WIN32_GROUPS = new Set(["desktop", "app", "office", "vision"]); return { id: component.id, name: component.name, @@ -242,7 +246,7 @@ function normalizeVendorComponent(component, vendorMap) { functionCalls: (component.functions || []).map((f) => ({ function: f.function, args: f.arguments || [], assignTo: f.assignTo || "" })), version: component.version || "", tags: component.tags || [], - platformTag: component.platformTag || null, + platformTag: component.platformTag || (WIN32_GROUPS.has(group) ? "win32" : null), icon: group, risk, fields,