From 3decced26897c80741e801574fb13131fb009e32 Mon Sep 17 00:00:00 2001 From: yu <1919727607@qq.com> Date: Wed, 4 Nov 2026 12:57:41 +0800 Subject: [PATCH] =?UTF-8?q?=E9=98=B6=E6=AE=B52-=E6=96=ADElectron=E4=BE=9D?= =?UTF-8?q?=E8=B5=96:=20human-dialog/window-extended/ai-service=E7=BB=8FHo?= =?UTF-8?q?stBridge=E8=BF=81=E5=85=A5=E5=8C=85,=20external+transport?= =?UTF-8?q?=E8=BF=81=E5=8C=85(session=E7=95=99=E5=AE=BF=E4=B8=BB),=20?= =?UTF-8?q?=E6=96=B0=E5=A2=9Echeck:deps=E4=BE=9D=E8=B5=96=E6=8A=A4?= =?UTF-8?q?=E6=A0=8F=E4=B8=8E=E5=8C=85README=E7=95=99=E5=AE=88=E6=B8=85?= =?UTF-8?q?=E5=8D=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package-lock.json | 39 +++- package.json | 3 +- packages/rpa-core/README.md | 47 +++++ packages/rpa-core/package.json | 7 +- packages/rpa-core/scripts/check-deps.mjs | 63 ++++++ .../rpa-core/src}/browser/external.ts | 2 +- packages/rpa-core/src/browser/transport.ts | 180 +++++++++++++++++ .../rpa-core/src}/capabilities/ai-service.ts | 8 +- .../src}/capabilities/common-final.test.ts | 8 +- .../src}/capabilities/human-dialog.ts | 22 ++- packages/rpa-core/src/capabilities/index.ts | 13 +- .../capabilities/mouse-moveto-smoke.test.ts | 6 +- .../src}/capabilities/window-extended.test.ts | 6 +- .../src}/capabilities/window-extended.ts | 20 +- packages/rpa-core/src/engine/host-bridge.ts | 53 ++++- packages/rpa-core/src/index.ts | 34 ++++ src/main/browser/session.ts | 7 +- src/main/browser/transport.ts | 182 +----------------- src/main/capabilities/ai-caps.ts | 2 +- src/main/capabilities/index.ts | 8 +- src/main/index.ts | 31 ++- 21 files changed, 505 insertions(+), 236 deletions(-) create mode 100644 packages/rpa-core/README.md create mode 100644 packages/rpa-core/scripts/check-deps.mjs rename {src/main => packages/rpa-core/src}/browser/external.ts (99%) create mode 100644 packages/rpa-core/src/browser/transport.ts rename {src/main => packages/rpa-core/src}/capabilities/ai-service.ts (91%) rename {src/main => packages/rpa-core/src}/capabilities/common-final.test.ts (94%) rename {src/main => packages/rpa-core/src}/capabilities/human-dialog.ts (88%) rename {src/main => packages/rpa-core/src}/capabilities/mouse-moveto-smoke.test.ts (96%) rename {src/main => packages/rpa-core/src}/capabilities/window-extended.test.ts (92%) rename {src/main => packages/rpa-core/src}/capabilities/window-extended.ts (96%) diff --git a/package-lock.json b/package-lock.json index d7b6aca..244ba80 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,6 +7,9 @@ "": { "name": "rpa-studio", "version": "0.1.0", + "workspaces": [ + "packages/*" + ], "dependencies": { "@cantoo/pdf-lib": "^2.11.1", "@gutenye/ocr-models": "^1.4.2", @@ -31,7 +34,6 @@ "react-dom": "^18.3.1", "sm-crypto": "^0.5.8", "tesseract.js": "^7.0.0", - "ws": "^8.22.0", "zustand": "^5.0.2" }, "devDependencies": { @@ -43,7 +45,6 @@ "@types/pngjs": "^6.0.5", "@types/react": "^18.3.12", "@types/react-dom": "^18.3.1", - "@types/ws": "^8.18.2", "@vitejs/plugin-react": "^4.7.0", "electron": "31.7.7", "electron-builder": "^26.15.7", @@ -3140,6 +3141,10 @@ "win32" ] }, + "node_modules/@rpa/core": { + "resolved": "packages/rpa-core", + "link": true + }, "node_modules/@selderee/plugin-htmlparser2": { "version": "0.12.0", "resolved": "https://registry.npmmirror.com/@selderee/plugin-htmlparser2/-/plugin-htmlparser2-0.12.0.tgz", @@ -12645,6 +12650,36 @@ "optional": true } } + }, + "packages/rpa-core": { + "name": "@rpa/core", + "version": "0.1.0", + "dependencies": { + "@gutenye/ocr-node": "^1.4.9", + "adm-zip": "^0.5.16", + "basic-ftp": "^6.2.3", + "exceljs": "^4.4.0", + "imapflow": "^2.10.0", + "mailparser": "^3.14.1", + "mssql": "^12.7.13", + "mysql2": "^3.24.6", + "nodemailer": "^6.9.16", + "pdf-lib": "^1.17.1", + "pg": "^8.23.1", + "pngjs": "^7.0.0", + "sm-crypto": "^0.5.8", + "ws": "^8.22.0" + }, + "devDependencies": { + "@types/adm-zip": "^0.5.7", + "@types/mssql": "^12.3.0", + "@types/nodemailer": "^6.4.17", + "@types/pg": "^8.23.1", + "@types/pngjs": "^6.0.5", + "@types/ws": "^8.18.2", + "typescript": "^5.7.2", + "vitest": "^2.1.8" + } } } } diff --git a/package.json b/package.json index ade7d55..c5b92a1 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "build": "electron-vite build", "preview": "electron-vite preview", "typecheck": "tsc --noEmit -p tsconfig.json", + "check:deps": "node packages/rpa-core/scripts/check-deps.mjs", "catalog": "node scripts/build-catalog.mjs", "snapshot": "vitest run scripts/capability-snapshot.test.ts", "package": "node scripts/package.mjs", @@ -44,7 +45,6 @@ "react-dom": "^18.3.1", "sm-crypto": "^0.5.8", "tesseract.js": "^7.0.0", - "ws": "^8.22.0", "zustand": "^5.0.2" }, "devDependencies": { @@ -56,7 +56,6 @@ "@types/pngjs": "^6.0.5", "@types/react": "^18.3.12", "@types/react-dom": "^18.3.1", - "@types/ws": "^8.18.2", "@vitejs/plugin-react": "^4.7.0", "electron": "31.7.7", "electron-builder": "^26.15.7", diff --git a/packages/rpa-core/README.md b/packages/rpa-core/README.md new file mode 100644 index 0000000..f2fb330 --- /dev/null +++ b/packages/rpa-core/README.md @@ -0,0 +1,47 @@ +# @rpa/core + +RPA 能力层包：零 Electron 依赖的流程引擎、能力注册表与能力实现。 +抽离路线见 `docs/rpa-core-refactor-tasks.md`，开发规范见 `docs/dev-conventions.md`。 + +## 包边界 + +| 模块 | 说明 | +|---|---| +| `src/engine/` | 注册表、运行时、表达式、`createEngine()` 工厂、`host-bridge.ts` 宿主桥 | +| `src/capabilities/` | 能力实现（副作用注册，经 `@rpa/core/capabilities` 拉起） | +| `src/browser/` | 外部浏览器 CDP：`external.ts`（Edge/Chrome 进程管理）+ `transport.ts`（CdpTransport 接口、ExternalTransport） | + +## HostBridge 契约（决策点 4 最小面） + +宿主（Electron 主进程 / 未来 Agent-Employee product-shell）启动时注入： + +```ts +import { setHostBridge, setSubflowSource } from "@rpa/core"; + +setHostBridge({ + createFormWindow({ title }) { /* 返回 HostDialogWindow 适配（BrowserWindow） */ }, + dipToScreenPoint(p) { /* screen.dipToScreenPoint */ }, + screenToDipPoint(p) { /* screen.screenToDipPoint */ }, + loadAiConfig() { /* settings.aiConfig */ } +}); +setSubflowSource({ listFlows, getFlow }); // 子流程仓库 +``` + +未注入时的行为：window-extended 的 DPI 换算回退恒等（与旧 vitest 行为一致）； +ai-service 返回空配置（requireAiConfig 统一抛 `ai_not_configured`）； +human-dialog 抛 `host_unavailable`。 + +## Electron 相关能力留守清单（宿主适配层，不入包） + +- `src/main/browser/session.ts`：BrowserSession 编排 + 内置运行浏览器（依赖 `BrowserWindow` 与应用生命周期） +- `src/main/browser/transport.ts`：`EmbeddedTransport`（包装 `webContents.debugger`） +- `src/main/capabilities/browser*.ts` / `web-batch2.ts` / `web-crawl.ts`：经 BrowserSession 操作浏览器窗口 +- `src/main/capabilities/ai-caps.ts`：AI 编排能力（依赖宿主注入的 chatCompletion 配置面，可随后续阶段评估迁入） +- `src/main/overlay.ts` / `desktop-pick.ts` / `vision-lab.ts` / `fixture/`：Electron 窗口强耦合 +- `src/main/local-vision/`：onnxruntime-node 原生依赖，Bun 兼容性单独立项验证 +- `src/main/flows.ts` / `run/` / `settings.ts`：应用级编排与持久化 + +## 依赖护栏 + +`npm run check:deps`（根目录亦可）扫描 `src/**`：禁止 `import "electron"`、 +相对引用逃出包 `src/`、引用宿主 `src/main`。阶段 2 起为提交前必跑项。 diff --git a/packages/rpa-core/package.json b/packages/rpa-core/package.json index bbf675c..732e68d 100644 --- a/packages/rpa-core/package.json +++ b/packages/rpa-core/package.json @@ -11,7 +11,8 @@ }, "scripts": { "test": "vitest run", - "typecheck": "tsc --noEmit -p tsconfig.json" + "typecheck": "tsc --noEmit -p tsconfig.json", + "check:deps": "node scripts/check-deps.mjs" }, "dependencies": { "@gutenye/ocr-node": "^1.4.9", @@ -26,7 +27,8 @@ "pdf-lib": "^1.17.1", "pg": "^8.23.1", "pngjs": "^7.0.0", - "sm-crypto": "^0.5.8" + "sm-crypto": "^0.5.8", + "ws": "^8.22.0" }, "devDependencies": { "@types/adm-zip": "^0.5.7", @@ -34,6 +36,7 @@ "@types/nodemailer": "^6.4.17", "@types/pg": "^8.23.1", "@types/pngjs": "^6.0.5", + "@types/ws": "^8.18.2", "typescript": "^5.7.2", "vitest": "^2.1.8" } diff --git a/packages/rpa-core/scripts/check-deps.mjs b/packages/rpa-core/scripts/check-deps.mjs new file mode 100644 index 0000000..3f3ccdc --- /dev/null +++ b/packages/rpa-core/scripts/check-deps.mjs @@ -0,0 +1,63 @@ +#!/usr/bin/env node +/** + * rpa-core 依赖护栏（任务 2.4，规范 v1 第二节的机器强制版）。 + * + * 禁止： + * 1. import "electron"（含 "electron/..." 子路径）； + * 2. 相对 import 逃出包 src/（如指向宿主 src/main 的 ../../../src/main/...）； + * 3. 任何 import 路径中出现 src/main（宿主模块横向依赖）。 + * + * 用法：node scripts/check-deps.mjs（退出码非 0 表示违规） + */ +import { readdir, readFile } from "node:fs/promises"; +import path from "node:path"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; + +const pkgRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const srcRoot = path.join(pkgRoot, "src"); + +const IMPORT_RE = /(?:import|export)\s+(?:[\s\S]*?from\s*)?["']([^"']+)["']/g; + +async function walk(dir) { + const entries = await readdir(dir, { withFileTypes: true }); + const files = []; + for (const entry of entries) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) files.push(...(await walk(full))); + else if (/\.tsx?$/.test(entry.name)) files.push(full); + } + return files; +} + +const offenders = []; +for (const file of await walk(srcRoot)) { + const raw = await readFile(file, "utf8"); + // 剥掉注释，避免文档示例里的 "electron" 字样误报 + const source = raw.replace(/\/\*[\s\S]*?\*\//g, "").replace(/^\s*\/\/.*$/gm, ""); + for (const match of source.matchAll(IMPORT_RE)) { + const spec = match[1]; + const rel = path.relative(pkgRoot, file).replaceAll("\\", "/"); + if (spec === "electron" || spec.startsWith("electron/")) { + offenders.push(`${rel}: 禁止 import "${spec}"（宿主能力走 HostBridge 注入）`); + continue; + } + if (spec.includes("src/main")) { + offenders.push(`${rel}: 禁止引用宿主模块 "${spec}"`); + continue; + } + if (spec.startsWith(".")) { + const resolved = path.resolve(path.dirname(file), spec); + if (!resolved.startsWith(srcRoot + path.sep) && resolved !== srcRoot) { + offenders.push(`${rel}: 相对引用逃出包 src/ → "${spec}"`); + } + } + } +} + +if (offenders.length) { + console.error("rpa-core 依赖护栏检查失败："); + for (const line of offenders) console.error(" - " + line); + process.exit(1); +} +console.log(`rpa-core 依赖护栏检查通过（src/ 下 ${offenders.length === 0 ? "无" : ""}违规 import）`); diff --git a/src/main/browser/external.ts b/packages/rpa-core/src/browser/external.ts similarity index 99% rename from src/main/browser/external.ts rename to packages/rpa-core/src/browser/external.ts index 7685edd..900adb1 100644 --- a/src/main/browser/external.ts +++ b/packages/rpa-core/src/browser/external.ts @@ -2,7 +2,7 @@ import { execFile, spawn, type ChildProcess } from "node:child_process"; import { promises as fs } from "node:fs"; import os from "node:os"; import path from "node:path"; -import { RuntimeError } from "@rpa/core"; +import { RuntimeError } from "../engine/types"; import { ExternalTransport } from "./transport"; /** diff --git a/packages/rpa-core/src/browser/transport.ts b/packages/rpa-core/src/browser/transport.ts new file mode 100644 index 0000000..1dfcb1c --- /dev/null +++ b/packages/rpa-core/src/browser/transport.ts @@ -0,0 +1,180 @@ +import WebSocket from "ws"; +import { RuntimeError } from "../engine/types"; + +/** + * CDP 传输层抽象。 + * + * 上层 BrowserSession（宿主适配层）的全部 CDP 通信都经由本接口： + * - EmbeddedTransport：内置运行浏览器（Electron BrowserWindow.webContents.debugger），留宿主 + * - ExternalTransport：外部 Edge/Chrome 浏览器（WebSocket 直连调试端口），在本包内 + * + * 两者只在传输方式上不同，CDP 协议域（DOM/Input/Page/Network/Runtime）完全一致。 + */ +export type CdpEventHandler = (params: Record) => void; + +export interface CdpTransport { + readonly kind: "embedded" | "external"; + /** 连接是否已失效（窗口销毁 / ws 断开） */ + readonly closed: boolean; + send(method: string, params?: Record): Promise; + /** 订阅 CDP 事件（如 Page.loadEventFired、Network.responseReceived），handler 只收到对应 method 的 params */ + on(method: string, handler: CdpEventHandler): void; + off(method: string, handler: CdpEventHandler): void; + close(): Promise; +} + +export function mapCdpError(error: unknown, kind: "embedded" | "external"): RuntimeError { + const message = (error as Error)?.message || String(error); + if (/destroyed|Target closed|not attached|connection.*(closed|lost)|websocket/i.test(message)) { + if (kind === "external") { + return new RuntimeError("browser_disconnected", "外部浏览器连接已断开", { + detail: message, + recommendation: "确认浏览器仍处于打开状态；若被手动关闭请重新运行「打开网页」" + }); + } + return new RuntimeError("tab_unavailable", "浏览器连接已失效", { detail: message }); + } + return new RuntimeError("browser_error", message, { recommendation: "重试或检查页面状态" }); +} + +interface CdpMessage { + id?: number; + method?: string; + params?: Record; + result?: unknown; + error?: { message?: string; data?: unknown }; +} + +/** 外部 Edge/Chrome：WebSocket 直连页面级 CDP 端点（webSocketDebuggerUrl） */ +export class ExternalTransport implements CdpTransport { + readonly kind = "external" as const; + private ws: WebSocket | null = null; + private nextId = 1; + private pending = new Map void; reject: (error: Error) => void }>(); + private handlers = new Map>(); + private closedFlag = false; + private readonly openPromise: Promise; + + private constructor( + wsUrl: string, + readonly description: string + ) { + this.openPromise = new Promise((resolve, reject) => { + const ws = new WebSocket(wsUrl, { maxPayload: 256 * 1024 * 1024 }); + this.ws = ws; + ws.on("open", () => resolve()); + ws.on("message", (data: unknown) => this.handleMessage(String(data))); + ws.on("error", (error: Error) => { + this.closedFlag = true; + this.failAllPending(new Error(`外部浏览器连接异常：${error.message}`)); + reject(error); + }); + ws.on("close", () => { + this.closedFlag = true; + this.failAllPending(new Error("外部浏览器连接已关闭")); + }); + }); + } + + static async connect(wsUrl: string, description: string, timeoutMs = 10000): Promise { + const transport = new ExternalTransport(wsUrl, description); + const timeout = new Promise((_, reject) => + setTimeout(() => reject(new RuntimeError("browser_disconnected", `连接外部浏览器超时：${description}`)), timeoutMs) + ); + await Promise.race([transport.openPromise, timeout]); + return transport; + } + + get closed(): boolean { + return this.closedFlag || !this.ws || this.ws.readyState !== this.ws.OPEN; + } + + async send(method: string, params: Record = {}): Promise { + if (this.closed) { + throw new RuntimeError("browser_disconnected", "外部浏览器连接已断开", { + recommendation: "确认浏览器未被手动关闭，然后重新运行「打开网页」" + }); + } + const id = this.nextId++; + const ws = this.ws as WebSocket; + const reply = new Promise((resolve, reject) => { + this.pending.set(id, { resolve, reject }); + try { + ws.send(JSON.stringify({ id, method, params })); + } catch (error) { + this.pending.delete(id); + reject(error as Error); + } + }); + try { + return (await reply) as T; + } catch (error) { + throw mapCdpError(error, "external"); + } + } + + on(method: string, handler: CdpEventHandler): void { + if (!this.handlers.has(method)) this.handlers.set(method, new Set()); + this.handlers.get(method)?.add(handler); + } + + off(method: string, handler: CdpEventHandler): void { + this.handlers.get(method)?.delete(handler); + } + + async close(): Promise { + const ws = this.ws; + this.closedFlag = true; + this.failAllPending(new Error("外部浏览器连接已主动关闭")); + if (!ws) return; + await new Promise((resolve) => { + if (ws.readyState === ws.CLOSED) return resolve(); + ws.once("close", () => resolve()); + try { + ws.close(); + } catch { + resolve(); + } + setTimeout(() => resolve(), 1500); + }); + } + + private handleMessage(raw: string): void { + let message: CdpMessage; + try { + message = JSON.parse(raw) as CdpMessage; + } catch { + return; + } + if (typeof message.id === "number") { + const entry = this.pending.get(message.id); + if (!entry) return; + this.pending.delete(message.id); + if (message.error) { + entry.reject(new Error(message.error.message || "CDP 调用失败")); + } else { + entry.resolve(message.result); + } + return; + } + if (message.method) { + const handlers = this.handlers.get(message.method); + if (!handlers) return; + const params = message.params ?? {}; + for (const handler of handlers) { + try { + handler(params); + } catch { + // 单个监听器异常不影响其他监听器 + } + } + } + } + + private failAllPending(error: Error): void { + for (const [id, entry] of this.pending) { + entry.reject(error); + this.pending.delete(id); + } + } +} diff --git a/src/main/capabilities/ai-service.ts b/packages/rpa-core/src/capabilities/ai-service.ts similarity index 91% rename from src/main/capabilities/ai-service.ts rename to packages/rpa-core/src/capabilities/ai-service.ts index 9d7a84a..2d02875 100644 --- a/src/main/capabilities/ai-service.ts +++ b/packages/rpa-core/src/capabilities/ai-service.ts @@ -1,5 +1,5 @@ -import { loadSettings } from "../settings"; -import { RuntimeError } from "@rpa/core"; +import { getHostBridge } from "../engine/host-bridge"; +import { RuntimeError } from "../engine/types"; export type ChatContentPart = | { type: "text"; text: string } @@ -18,7 +18,9 @@ export interface ResolvedAiConfig { } export function getAiConfig(): ResolvedAiConfig { - const config = loadSettings().aiConfig; + const bridge = getHostBridge(); + // 宿主桥未注入（vitest / CLI 未配置）时返回空配置，requireAiConfig 统一抛 ai_not_configured + const config = bridge ? bridge.loadAiConfig() : { baseUrl: "", apiKey: "", model: "", visionModel: "" }; return { ...config, visionModel: config.visionModel || config.model }; } diff --git a/src/main/capabilities/common-final.test.ts b/packages/rpa-core/src/capabilities/common-final.test.ts similarity index 94% rename from src/main/capabilities/common-final.test.ts rename to packages/rpa-core/src/capabilities/common-final.test.ts index 993844e..4cf2f44 100644 --- a/src/main/capabilities/common-final.test.ts +++ b/packages/rpa-core/src/capabilities/common-final.test.ts @@ -4,10 +4,10 @@ import os from "node:os"; import path from "node:path"; import { PDFDocument } from "pdf-lib"; import { afterEach, describe, expect, it } from "vitest"; -import type { CatalogComponent, FlowNode } from "../../shared/types"; -import type { RunContext } from "@rpa/core"; -import { getHandler } from "@rpa/core"; -import "@rpa/core/capabilities"; import "./window-extended"; +import type { CatalogComponent, FlowNode } from "../types"; +import type { RunContext } from "../engine/types"; +import { getHandler } from "../engine/registry"; +import "./index"; const dirs: string[] = []; const context = { vars: {} } as RunContext; const component = {} as CatalogComponent; const node = (id: string): FlowNode => ({ id, componentId: id, position: { x: 0, y: 0 }, values: {}, options: {} }); diff --git a/src/main/capabilities/human-dialog.ts b/packages/rpa-core/src/capabilities/human-dialog.ts similarity index 88% rename from src/main/capabilities/human-dialog.ts rename to packages/rpa-core/src/capabilities/human-dialog.ts index 3dbbd6a..f4112d7 100644 --- a/src/main/capabilities/human-dialog.ts +++ b/packages/rpa-core/src/capabilities/human-dialog.ts @@ -1,6 +1,6 @@ -import { BrowserWindow } from "electron"; -import { register } from "@rpa/core"; -import { RuntimeError } from "@rpa/core"; +import { getHostBridge, type HostDialogWindow } from "../engine/host-bridge"; +import { register } from "../engine/registry"; +import { RuntimeError } from "../engine/types"; interface FormFieldSpec { key?: string; name?: string; label?: string; type?: string; value?: string; options?: unknown[] } @@ -28,6 +28,12 @@ function escapeHtml(s: string): string { } async function showForm(options: { title: string; fields: FormFieldSpec[]; timeoutSec: number }): Promise> { + const bridge = getHostBridge(); + if (!bridge) { + throw new RuntimeError("host_unavailable", "宿主桥未注入，无法打开人机交互窗口", { + recommendation: "宿主（Electron 主进程）启动时需调用 setHostBridge() 注入窗口工厂" + }); + } return new Promise((resolve) => { const rows = options.fields.map((f, i) => { const key = escapeHtml(fieldKey(f, i)); @@ -53,11 +59,7 @@ async function showForm(options: { title: string; fields: FormFieldSpec[]; timeo return "
'; }).join("\n"); const html = '

' + escapeHtml(options.title) + "

" + rows + ''; - const win = new BrowserWindow({ - width: 480, height: 0, useContentSize: true, autoHideMenuBar: true, - title: options.title, alwaysOnTop: true, - webPreferences: { contextIsolation: true, nodeIntegration: false, sandbox: true } - }); + const win: HostDialogWindow = bridge.createFormWindow({ title: options.title }); let settled = false; let timer: NodeJS.Timeout | undefined; let timeout: NodeJS.Timeout | undefined; @@ -71,14 +73,14 @@ async function showForm(options: { title: string; fields: FormFieldSpec[]; timeo }; timer = setInterval(async () => { try { - const values = await win.webContents.executeJavaScript("window.__formValues || null", true); + const values = await win.executeJavaScript("window.__formValues || null"); if (values) finish(values as Record); } catch { finish({}); } }, 250); timeout = setTimeout(() => { - try { void win.webContents.executeJavaScript("document.getElementById(\"ok\").click()"); } catch { + try { void win.executeJavaScript("document.getElementById(\"ok\").click()"); } catch { finish({}); } }, Math.max(5, options.timeoutSec) * 1000); diff --git a/packages/rpa-core/src/capabilities/index.ts b/packages/rpa-core/src/capabilities/index.ts index d14b9ff..b3be3f9 100644 --- a/packages/rpa-core/src/capabilities/index.ts +++ b/packages/rpa-core/src/capabilities/index.ts @@ -1,11 +1,11 @@ /** - * rpa-core 纯 Node 能力注册入口。 + * rpa-core 能力注册入口。 * - * import 本模块即完成全部纯 Node 能力的副作用注册（与改造前 + * import 本模块即完成全部包内能力的副作用注册（与改造前 * src/main/capabilities/index.ts 的加载顺序保持一致，别名/重复 id 的 - * last-wins 语义不变）。依赖 Electron 或宿主模块的能力（human-dialog / - * window-extended / ai-service / ai-caps / browser* / web-batch2 / web-crawl） - * 留在宿主 src/main/capabilities，阶段 2 通过 HostBridge 迁入。 + * last-wins 语义不变）。依赖 Electron 窗口的浏览器能力（ai-caps / browser* / + * web-batch2 / web-crawl）留在宿主 src/main/capabilities； + * human-dialog / ai-service / window-extended 已于阶段 2 经 HostBridge 迁入。 */ import "./control"; import "./files"; @@ -37,3 +37,6 @@ import "./basic-encryption"; import "./mail-remaining"; import "./remaining-core"; import "./common-final"; +import "./human-dialog"; +import "./ai-service"; +import "./window-extended"; diff --git a/src/main/capabilities/mouse-moveto-smoke.test.ts b/packages/rpa-core/src/capabilities/mouse-moveto-smoke.test.ts similarity index 96% rename from src/main/capabilities/mouse-moveto-smoke.test.ts rename to packages/rpa-core/src/capabilities/mouse-moveto-smoke.test.ts index cd11077..c4ef1b3 100644 --- a/src/main/capabilities/mouse-moveto-smoke.test.ts +++ b/packages/rpa-core/src/capabilities/mouse-moveto-smoke.test.ts @@ -1,9 +1,9 @@ import { execFile } from "node:child_process"; import { promisify } from "node:util"; import { afterEach, describe, expect, it } from "vitest"; -import type { CatalogComponent, FlowNode } from "../../shared/types"; -import { getHandler } from "@rpa/core"; -import type { RunContext } from "@rpa/core"; +import type { CatalogComponent, FlowNode } from "../types"; +import { getHandler } from "../engine/registry"; +import type { RunContext } from "../engine/types"; import "./window-extended"; const run = promisify(execFile); diff --git a/src/main/capabilities/window-extended.test.ts b/packages/rpa-core/src/capabilities/window-extended.test.ts similarity index 92% rename from src/main/capabilities/window-extended.test.ts rename to packages/rpa-core/src/capabilities/window-extended.test.ts index 344c8d6..9cd84e1 100644 --- a/src/main/capabilities/window-extended.test.ts +++ b/packages/rpa-core/src/capabilities/window-extended.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from "vitest"; -import type { CatalogComponent, FlowNode } from "../../shared/types"; -import type { RunContext } from "@rpa/core"; -import { getHandler } from "@rpa/core"; +import type { CatalogComponent, FlowNode } from "../types"; +import type { RunContext } from "../engine/types"; +import { getHandler } from "../engine/registry"; import "./window-extended"; const context = {} as RunContext; diff --git a/src/main/capabilities/window-extended.ts b/packages/rpa-core/src/capabilities/window-extended.ts similarity index 96% rename from src/main/capabilities/window-extended.ts rename to packages/rpa-core/src/capabilities/window-extended.ts index d503a12..ec45e8d 100644 --- a/src/main/capabilities/window-extended.ts +++ b/packages/rpa-core/src/capabilities/window-extended.ts @@ -1,7 +1,7 @@ -import { register } from "@rpa/core"; -import { RuntimeError, type CapabilityHandler } from "@rpa/core"; -import { powershell, psString } from "@rpa/core"; -import { screen } from "electron"; +import { register } from "../engine/registry"; +import { RuntimeError, type CapabilityHandler } from "../engine/types"; +import { powershell, psString } from "./desktop"; +import { getHostBridge } from "../engine/host-bridge"; const WINDOW_API = `Add-Type @' using System; @@ -136,11 +136,12 @@ function mousePath(from: [number, number], to: [number, number], mode: string, s /** 界面(框选/元素拾取/截图)产出的是 DIP 逻辑坐标；DPI 感知进程的 SetCursorPos 用物理像素， * 缩放屏(125%/150%)下直接混用会成比例偏移(用户实测 125% 下差约 25% 距离)。 - * 统一在执行前换算；非 Electron 环境(vitest)screen 不可用，回退原值保持旧行为 */ + * 统一在执行前换算；宿主桥未注入时(vitest)回退原值保持旧行为 */ function dipToPhysical(x: number, y: number): [number, number] { try { - if (screen && typeof screen.dipToScreenPoint === "function") { - const physical = screen.dipToScreenPoint({ x, y }); + const bridge = getHostBridge(); + if (bridge) { + const physical = bridge.dipToScreenPoint({ x, y }); return [Math.round(physical.x), Math.round(physical.y)]; } } catch {} @@ -150,8 +151,9 @@ function dipToPhysical(x: number, y: number): [number, number] { /** 反向换算：物理像素(如 GetWindowRect 结果) → DIP，用于「相对激活窗口」偏移叠加 */ function physicalToDip(x: number, y: number): [number, number] { try { - if (screen && typeof screen.screenToDipPoint === "function") { - const dip = screen.screenToDipPoint({ x, y }); + const bridge = getHostBridge(); + if (bridge) { + const dip = bridge.screenToDipPoint({ x, y }); return [Math.round(dip.x), Math.round(dip.y)]; } } catch {} diff --git a/packages/rpa-core/src/engine/host-bridge.ts b/packages/rpa-core/src/engine/host-bridge.ts index bf49407..854410d 100644 --- a/packages/rpa-core/src/engine/host-bridge.ts +++ b/packages/rpa-core/src/engine/host-bridge.ts @@ -1,11 +1,13 @@ import type { FlowDefinition } from "../types"; /** - * 宿主桥（HostBridge 候选，阶段 2 正式化接口）。 + * 宿主桥（任务 2.1 正式化）。 * - * 包内个别能力需要宿主数据（目前仅子流程仓库），通过显式 setter 注入， - * 包本身不得 import 宿主模块。宿主启动时调用 setSubflowSource()。 + * 包内能力需要的宿主能力（子流程仓库 / Electron 窗口 / DPI 换算 / 应用配置） + * 通过本接口注入，包本身禁止 import "electron" 或宿主模块（见包内依赖检查脚本）。 + * 宿主启动时调用 setSubflowSource() / setHostBridge()。 */ + export interface SubflowSource { listFlows(): Promise>; getFlow(id: string): Promise; @@ -20,3 +22,48 @@ export function setSubflowSource(source: SubflowSource): void { export function getSubflowSource(): SubflowSource | undefined { return subflowSource; } + +/** 人机交互表单窗口的最小面（能力只用到这些成员，宿主用 BrowserWindow 适配） */ +export interface HostDialogWindow { + isDestroyed(): boolean; + destroy(): void; + loadURL(url: string): Promise; + on(event: "closed", listener: () => void): void; + /** 在窗口页面里执行 JS 并取回结果（宿主侧以 userGesture=true 执行） */ + executeJavaScript(script: string): Promise; +} + +export interface HostPoint { + x: number; + y: number; +} + +/** AI 服务配置（宿主 settings.aiConfig） */ +export interface HostAiConfig { + baseUrl: string; + apiKey: string; + model: string; + visionModel: string; +} + +/** 宿主能力桥最小面（决策点 4）：dialog 窗口工厂 / DPI 坐标换算 / 配置读取 */ +export interface HostBridge { + /** 创建人机交互表单窗口（human-dialog 用） */ + createFormWindow(options: { title: string }): HostDialogWindow; + /** DIP 逻辑坐标 → 物理像素（window-extended 的鼠标/拖拽坐标换算） */ + dipToScreenPoint(point: HostPoint): HostPoint; + /** 物理像素 → DIP 逻辑坐标 */ + screenToDipPoint(point: HostPoint): HostPoint; + /** 读取 AI 服务配置（ai-service 用） */ + loadAiConfig(): HostAiConfig; +} + +let hostBridge: HostBridge | undefined; + +export function setHostBridge(bridge: HostBridge): void { + hostBridge = bridge; +} + +export function getHostBridge(): HostBridge | undefined { + return hostBridge; +} diff --git a/packages/rpa-core/src/index.ts b/packages/rpa-core/src/index.ts index e36cd81..1dc1f4e 100644 --- a/packages/rpa-core/src/index.ts +++ b/packages/rpa-core/src/index.ts @@ -10,6 +10,40 @@ export { register, getHandler, hasCapability, capabilityIds, capabilityCount } f export { runFlow } from "./engine/runtime"; export { RuntimeError, BreakSignal, ContinueSignal, StopSignal, asRuntimeError } from "./engine/types"; export { setSubflowSource, getSubflowSource, type SubflowSource } from "./engine/host-bridge"; +export { + setHostBridge, + getHostBridge, + type HostBridge, + type HostDialogWindow, + type HostPoint, + type HostAiConfig +} from "./engine/host-bridge"; +// AI 服务（阶段 2 经 HostBridge 迁入包内）：宿主注入 loadAiConfig，消费方走此导出 +export { + chatCompletion, + getAiConfig, + requireAiConfig, + type ChatCompletionOptions, + type ChatContentPart, + type ChatMessage, + type ResolvedAiConfig +} from "./capabilities/ai-service"; +// 浏览器 CDP 外部路径（阶段 2 迁入）：ExternalTransport + 外部浏览器进程管理 +export { ExternalTransport, mapCdpError, type CdpTransport, type CdpEventHandler } from "./browser/transport"; +export { + listPageTargets, + openExternalBrowser, + openNewPageTarget, + sessionKeyOf, + detectBrowserPath, + connectRemoteBrowser, + launchExternalBrowser, + type ExternalBrowserOptions, + type ExternalBrowserType, + type ExternalLaunchOptions, + type ExternalRemoteOptions, + type ExternalConnection +} from "./browser/external"; // PowerShell 助手（阶段 3 将移入 driver/windows/）：宿主留守能力（window-extended 等）经此引用 export { powershell, psString } from "./capabilities/desktop"; export type { diff --git a/src/main/browser/session.ts b/src/main/browser/session.ts index cc2ff55..34bbb94 100644 --- a/src/main/browser/session.ts +++ b/src/main/browser/session.ts @@ -2,15 +2,18 @@ import { BrowserWindow } from "electron"; import { promises as fs } from "node:fs"; import path from "node:path"; import type { ElementPickResult } from "../../shared/types"; -import { EmbeddedTransport, ExternalTransport, type CdpTransport, type CdpEventHandler } from "./transport"; +import { EmbeddedTransport } from "./transport"; import { + ExternalTransport, listPageTargets, openExternalBrowser, openNewPageTarget, sessionKeyOf, + type CdpEventHandler, + type CdpTransport, type ExternalBrowserOptions, type ExternalConnection -} from "./external"; +} from "@rpa/core"; const DEFAULT_ELEMENT_WAIT_MS = 5000; let globalElementWaitMs = DEFAULT_ELEMENT_WAIT_MS; diff --git a/src/main/browser/transport.ts b/src/main/browser/transport.ts index c312225..59e8f2e 100644 --- a/src/main/browser/transport.ts +++ b/src/main/browser/transport.ts @@ -1,44 +1,12 @@ import type { BrowserWindow } from "electron"; -import WebSocket from "ws"; -import { RuntimeError } from "@rpa/core"; +import { RuntimeError, mapCdpError, type CdpEventHandler, type CdpTransport } from "@rpa/core"; /** - * CDP 传输层抽象。 + * 内置运行浏览器传输层（宿主适配层，任务 2.3）。 * - * 上层 BrowserSession 的全部 CDP 通信都经由本接口： - * - EmbeddedTransport：内置运行浏览器（Electron BrowserWindow.webContents.debugger） - * - ExternalTransport：外部 Edge/Chrome 浏览器（WebSocket 直连调试端口） - * - * 两者只在传输方式上不同，CDP 协议域（DOM/Input/Page/Network/Runtime）完全一致。 + * 包装 Electron BrowserWindow.webContents.debugger，实现包内的 CdpTransport 接口； + * CdpTransport / mapCdpError / ExternalTransport（外部浏览器）已迁入 @rpa/core。 */ -export type CdpEventHandler = (params: Record) => void; - -export interface CdpTransport { - readonly kind: "embedded" | "external"; - /** 连接是否已失效（窗口销毁 / ws 断开） */ - readonly closed: boolean; - send(method: string, params?: Record): Promise; - /** 订阅 CDP 事件（如 Page.loadEventFired、Network.responseReceived），handler 只收到对应 method 的 params */ - on(method: string, handler: CdpEventHandler): void; - off(method: string, handler: CdpEventHandler): void; - close(): Promise; -} - -export function mapCdpError(error: unknown, kind: "embedded" | "external"): RuntimeError { - const message = (error as Error)?.message || String(error); - if (/destroyed|Target closed|not attached|connection.*(closed|lost)|websocket/i.test(message)) { - if (kind === "external") { - return new RuntimeError("browser_disconnected", "外部浏览器连接已断开", { - detail: message, - recommendation: "确认浏览器仍处于打开状态；若被手动关闭请重新运行「打开网页」" - }); - } - return new RuntimeError("tab_unavailable", "浏览器连接已失效", { detail: message }); - } - return new RuntimeError("browser_error", message, { recommendation: "重试或检查页面状态" }); -} - -/** 内置运行浏览器：包装 webContents.debugger */ export class EmbeddedTransport implements CdpTransport { readonly kind = "embedded" as const; private wrappers = new Map) => void>(); @@ -88,145 +56,3 @@ export class EmbeddedTransport implements CdpTransport { } } } - -interface CdpMessage { - id?: number; - method?: string; - params?: Record; - result?: unknown; - error?: { message?: string; data?: unknown }; -} - -/** 外部 Edge/Chrome：WebSocket 直连页面级 CDP 端点（webSocketDebuggerUrl） */ -export class ExternalTransport implements CdpTransport { - readonly kind = "external" as const; - private ws: WebSocket | null = null; - private nextId = 1; - private pending = new Map void; reject: (error: Error) => void }>(); - private handlers = new Map>(); - private closedFlag = false; - private readonly openPromise: Promise; - - private constructor( - wsUrl: string, - readonly description: string - ) { - this.openPromise = new Promise((resolve, reject) => { - const ws = new WebSocket(wsUrl, { maxPayload: 256 * 1024 * 1024 }); - this.ws = ws; - ws.on("open", () => resolve()); - ws.on("message", (data: unknown) => this.handleMessage(String(data))); - ws.on("error", (error: Error) => { - this.closedFlag = true; - this.failAllPending(new Error(`外部浏览器连接异常：${error.message}`)); - reject(error); - }); - ws.on("close", () => { - this.closedFlag = true; - this.failAllPending(new Error("外部浏览器连接已关闭")); - }); - }); - } - - static async connect(wsUrl: string, description: string, timeoutMs = 10000): Promise { - const transport = new ExternalTransport(wsUrl, description); - const timeout = new Promise((_, reject) => - setTimeout(() => reject(new RuntimeError("browser_disconnected", `连接外部浏览器超时：${description}`)), timeoutMs) - ); - await Promise.race([transport.openPromise, timeout]); - return transport; - } - - get closed(): boolean { - return this.closedFlag || !this.ws || this.ws.readyState !== this.ws.OPEN; - } - - async send(method: string, params: Record = {}): Promise { - if (this.closed) { - throw new RuntimeError("browser_disconnected", "外部浏览器连接已断开", { - recommendation: "确认浏览器未被手动关闭，然后重新运行「打开网页」" - }); - } - const id = this.nextId++; - const ws = this.ws as WebSocket; - const reply = new Promise((resolve, reject) => { - this.pending.set(id, { resolve, reject }); - try { - ws.send(JSON.stringify({ id, method, params })); - } catch (error) { - this.pending.delete(id); - reject(error as Error); - } - }); - try { - return (await reply) as T; - } catch (error) { - throw mapCdpError(error, "external"); - } - } - - on(method: string, handler: CdpEventHandler): void { - if (!this.handlers.has(method)) this.handlers.set(method, new Set()); - this.handlers.get(method)?.add(handler); - } - - off(method: string, handler: CdpEventHandler): void { - this.handlers.get(method)?.delete(handler); - } - - async close(): Promise { - const ws = this.ws; - this.closedFlag = true; - this.failAllPending(new Error("外部浏览器连接已主动关闭")); - if (!ws) return; - await new Promise((resolve) => { - if (ws.readyState === ws.CLOSED) return resolve(); - ws.once("close", () => resolve()); - try { - ws.close(); - } catch { - resolve(); - } - setTimeout(() => resolve(), 1500); - }); - } - - private handleMessage(raw: string): void { - let message: CdpMessage; - try { - message = JSON.parse(raw) as CdpMessage; - } catch { - return; - } - if (typeof message.id === "number") { - const entry = this.pending.get(message.id); - if (!entry) return; - this.pending.delete(message.id); - if (message.error) { - entry.reject(new Error(message.error.message || "CDP 调用失败")); - } else { - entry.resolve(message.result); - } - return; - } - if (message.method) { - const handlers = this.handlers.get(message.method); - if (!handlers) return; - const params = message.params ?? {}; - for (const handler of handlers) { - try { - handler(params); - } catch { - // 单个监听器异常不影响其他监听器 - } - } - } - } - - private failAllPending(error: Error): void { - for (const [id, entry] of this.pending) { - entry.reject(error); - this.pending.delete(id); - } - } -} diff --git a/src/main/capabilities/ai-caps.ts b/src/main/capabilities/ai-caps.ts index 5255419..f9b3626 100644 --- a/src/main/capabilities/ai-caps.ts +++ b/src/main/capabilities/ai-caps.ts @@ -1,6 +1,6 @@ import { register } from "@rpa/core"; import { RuntimeError } from "@rpa/core"; -import { chatCompletion, requireAiConfig } from "./ai-service"; +import { chatCompletion, requireAiConfig } from "@rpa/core"; function inputText(args: Record, nodeId: string): string { for (const key of ["input", "text", "content", "sentence", "prompt", "sourceFilePath", "filePath", "source", "document"]) { diff --git a/src/main/capabilities/index.ts b/src/main/capabilities/index.ts index 7b96fa0..ca4901e 100644 --- a/src/main/capabilities/index.ts +++ b/src/main/capabilities/index.ts @@ -1,8 +1,9 @@ /** * 宿主能力注册入口。 * - * 纯 Node 能力已迁入 @rpa/core（packages/rpa-core），第一行 import 拉起包内全部注册； - * 此处只保留依赖 Electron / 宿主模块的能力，阶段 2 通过 HostBridge 逐步迁入。 + * import "@rpa/core/capabilities" 拉起包内全部能力注册（含阶段 2 经 + * HostBridge 迁入的 human-dialog / ai-service / window-extended）； + * 此处只保留依赖 Electron 窗口/会话的浏览器能力与 AI 编排能力。 * 导出顺序整体保持改造前 index.ts 的注册顺序（别名/重复 id last-wins 语义不变）。 */ import "@rpa/core/capabilities"; @@ -13,9 +14,6 @@ import "./browser-common"; import "./browser-control"; import "./web-batch2"; import "./web-crawl"; -import "./human-dialog"; -import "./ai-service"; import "./ai-caps"; -import "./window-extended"; export { capabilityCount, capabilityIds, getHandler, hasCapability, register } from "@rpa/core"; diff --git a/src/main/index.ts b/src/main/index.ts index 7cdcc42..0d3cd92 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -1,4 +1,4 @@ -import { app, BrowserWindow, dialog, ipcMain, shell } from "electron"; +import { app, BrowserWindow, dialog, ipcMain, screen, shell } from "electron"; import path from "node:path"; import type { AppInfo, AppSettings, FlowDefinition, FlowSaveOptions, FlowSaveResult } from "../shared/types"; import { @@ -11,7 +11,7 @@ import { import "./capabilities"; import { catalogPath, loadCatalog } from "./catalog"; import { createDemoFlow, createExampleFlows, createFlow, deleteFlow, getFlow, listFlows, saveFlow } from "./flows"; -import { capabilityCount, capabilityIds, setSubflowSource } from "@rpa/core"; +import { capabilityCount, capabilityIds, setHostBridge, setSubflowSource, type HostDialogWindow } from "@rpa/core"; import { closeBrowserSession, setMainWindowGetter, startRun, stopRun } from "./run/manager"; import { registerOverlayIpc } from "./overlay"; import { fixtureUrl, startFixtureServer, stopFixtureServer } from "./fixture/server"; @@ -24,6 +24,31 @@ import { logMain } from "./logger"; // 子流程仓库在宿主（flows.ts），包内能力经 host-bridge 取用（阶段 2 正式化 HostBridge） setSubflowSource({ listFlows: async () => listFlows(), getFlow: async (id) => getFlow(id) }); +// 宿主能力桥（决策点 4 最小面）：人机交互窗口工厂 / DPI 坐标换算 / AI 配置读取 +setHostBridge({ + createFormWindow: ({ title }) => { + const win = new BrowserWindow({ + width: 480, height: 0, useContentSize: true, autoHideMenuBar: true, + title, alwaysOnTop: true, + webPreferences: { contextIsolation: true, nodeIntegration: false, sandbox: true } + }); + const dialogWindow: HostDialogWindow = { + isDestroyed: () => win.isDestroyed(), + destroy: () => { if (!win.isDestroyed()) win.destroy(); }, + loadURL: (url) => win.loadURL(url), + on: (event, listener) => { win.on(event as "closed", listener); }, + executeJavaScript: (script) => win.webContents.executeJavaScript(script, true) + }; + return dialogWindow; + }, + dipToScreenPoint: (point) => screen.dipToScreenPoint(point), + screenToDipPoint: (point) => screen.screenToDipPoint(point), + loadAiConfig: () => { + const ai = loadSettings().aiConfig; + return { baseUrl: ai.baseUrl ?? "", apiKey: ai.apiKey ?? "", model: ai.model ?? "", visionModel: ai.visionModel ?? "" }; + } +}); + let mainWindow: BrowserWindow | null = null; if (process.env.RPA_ENABLE_GPU !== "1") app.disableHardwareAcceleration(); @@ -117,7 +142,7 @@ function registerIpc(): void { }); type BrowserPickOptions = { url?: string; - browser?: import("./browser/external").ExternalBrowserOptions; + browser?: import("@rpa/core").ExternalBrowserOptions; }; ipcMain.handle("browser:pick-element", async (_event, options: BrowserPickOptions = {}) => {