diff --git a/README.md b/README.md index db458d3..227e2ef 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ npm run dev:local # 使用 RPA_ELECTRON_DIST 或 C:\electron-rpa\dist 的 # 或 npm run build node scripts/start.mjs --no-sandbox +npm run test:e2e # 隔离启动应用并执行本地网页订单完整流程 ``` > 为什么需要 `dev:local`：当前工作区 `D:\RPA` 的 ACL 会让 Electron 自带的 Chromium 沙箱启动失败（`--version` 都返回 0x80000003）。 @@ -52,5 +53,6 @@ docs 现状分析、实施方案 - 1016 项目录中当前有 398 项通过能力注册表或运行时容器标记为可执行，覆盖控制/触发、字符串/集合/INI/二维表格、文件、浏览器元素常用及 DOM 高级能力、HTTP、Excel/CSV与表格辅助、系统路径与环境变量、进程管理、桌面键鼠/窗口对象/剪贴板/对话框、PDF、Word、邮件发送、屏幕截图、离线图像查找与点击；其余灰标「待接入」。运行前检查会提前定位待接入组件、必填参数、多出口与断链节点。 - 内置本地演示站默认监听 `127.0.0.1:17863`；新用户首次启动会获得“本地网页订单演示”任务，可离线验证网页输入、点击和文本读取。 +- 浏览器可信输入与定位机制参考 `C:\rpa-demo`：CDP Input、唯一定位、遮挡检查和输入回读校验；不使用 `el.click()` 伪造点击。 - 不包含厂商代码、二进制、模型或前端资产；目录是从厂商元数据派生的静态定义。 - 运行期不访问云服务；HTTP/邮件属于用户显式配置的外部系统。 diff --git a/docs/02-实施方案.md b/docs/02-实施方案.md index 7ef8b48..be294d7 100644 --- a/docs/02-实施方案.md +++ b/docs/02-实施方案.md @@ -6,6 +6,8 @@ > **2026-10-27 当前状态**：计划点名的代表能力已全部进入注册表或运行时容器，覆盖控制/触发、字符串/集合/INI/二维表格、文件、浏览器元素常用及 DOM 高级能力、HTTP、Excel/CSV与表格辅助、系统路径与环境变量、进程管理、桌面键鼠/窗口对象/剪贴板/对话框、PDF、Word、邮件发送、屏幕截图、离线图像查找与点击；启动后目录显示 **398 项可执行 / 1016 项总目录**。当前 15 个测试文件共 99 条测试，覆盖计划能力审计、浏览器参数路由/变体/DOM、Excel/PDF/Word/SMTP/模板匹配、文件与数据扩展、系统/进程/窗口解析、直线/IF/循环/重试/跳过、主出口与异常出口语义、运行记录回写、运行前检查、本地演示站与示例流程。设置页可查看分类覆盖和处理器 ID 数量。未通过实际场景验证的变体仍不计入最终交付数量。 +> 浏览器端到端门禁：`npm run test:e2e` 使用隔离用户目录启动应用和本地演示站，真实执行打开网页、两次可信输入、点击提交和读取结果；期望结果必须精确包含“订单创建成功：RPA 客户，金额 128”。浏览器输入、点击、定位歧义和遮挡检查机制已对照 `C:\rpa-demo/actions.js` 校准。 + --- ## 0. 决策记录（2026-10-26 确认） diff --git a/package.json b/package.json index e479dd4..0b96579 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,8 @@ "typecheck": "tsc --noEmit -p tsconfig.json", "catalog": "node scripts/build-catalog.mjs", "package": "node scripts/package.mjs", - "test": "vitest run" + "test": "vitest run", + "test:e2e": "node scripts/demo-e2e.mjs" }, "dependencies": { "@xyflow/react": "^12.12.0", diff --git a/scripts/demo-e2e.mjs b/scripts/demo-e2e.mjs new file mode 100644 index 0000000..e6eccaf --- /dev/null +++ b/scripts/demo-e2e.mjs @@ -0,0 +1,91 @@ +import { spawn } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const root = process.cwd(); +const debugPort = 9231; +const fixturePort = 17963; +const profile = fs.mkdtempSync(path.join(os.tmpdir(), "rpa-studio-e2e-")); +const candidates = [process.env.RPA_ELECTRON_DIST, "C:/electron-rpa/dist", path.join(root, "node_modules", "electron", "dist")].filter(Boolean); +const dist = candidates.find((candidate) => fs.existsSync(path.join(candidate, "electron.exe"))); +if (!dist) throw new Error("找不到 Electron 运行时，请设置 RPA_ELECTRON_DIST"); + +const env = { ...process.env, RPA_E2E: "1", RPA_FIXTURE_PORT: String(fixturePort) }; +delete env.ELECTRON_RUN_AS_NODE; +const child = spawn(path.join(dist, "electron.exe"), [root, "--no-sandbox", `--remote-debugging-port=${debugPort}`, `--user-data-dir=${profile}`], { + cwd: root, + env, + stdio: ["ignore", "pipe", "pipe"], + windowsHide: true +}); + +let stderr = ""; +child.stderr.on("data", (chunk) => { stderr += String(chunk); }); + +async function waitForTarget() { + const deadline = Date.now() + 30000; + while (Date.now() < deadline) { + if (child.exitCode !== null) throw new Error(`RPA Studio 提前退出：${child.exitCode}\n${stderr}`); + try { + const targets = await (await fetch(`http://127.0.0.1:${debugPort}/json/list`)).json(); + const page = targets.find((item) => item.type === "page" && item.title === "RPA Studio"); + if (page) return page; + } catch { /* wait for CDP */ } + await new Promise((resolve) => setTimeout(resolve, 200)); + } + throw new Error(`等待 RPA Studio CDP 超时\n${stderr}`); +} + +async function evaluate(page, expression) { + const socket = new WebSocket(page.webSocketDebuggerUrl); + await new Promise((resolve, reject) => { socket.onopen = resolve; socket.onerror = reject; }); + const result = await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error("E2E Runtime.evaluate 超时")), 60000); + socket.onmessage = (event) => { + const message = JSON.parse(event.data); + if (message.id !== 1) return; + clearTimeout(timer); + if (message.error) reject(new Error(JSON.stringify(message.error))); + else if (message.result?.exceptionDetails) reject(new Error(message.result.exceptionDetails.text || "E2E 页面异常")); + else resolve(message.result?.result?.value); + }; + socket.send(JSON.stringify({ id: 1, method: "Runtime.evaluate", params: { expression, returnByValue: true, awaitPromise: true } })); + }); + socket.close(); + return result; +} + +const expression = `(async () => { + const flow = await window.rpa.flows.createDemo(); + const events = []; + const off = window.rpa.run.onEvent((event) => events.push(event)); + const started = await window.rpa.run.start(flow); + const finished = await new Promise((resolve, reject) => { + const deadline = Date.now() + 45000; + const timer = setInterval(() => { + const event = events.find((item) => item.type === 'run.finished'); + if (event) { clearInterval(timer); resolve(event); } + else if (Date.now() > deadline) { clearInterval(timer); reject(new Error('demo flow timeout')); } + }, 100); + }); + off(); + await window.rpa.flows.delete(flow.id); + const result = events.find((item) => item.type === 'node.finished' && item.nodeId === 'node_result'); + return { runId: started.runId, status: finished.status, durationMs: finished.durationMs, result: result?.output, failed: events.filter((item) => item.type === 'node.finished' && item.status === 'failed') }; +})()`; + +try { + const page = await waitForTarget(); + const result = await evaluate(page, expression); + if (result?.status !== "succeeded") throw new Error(`演示流程失败：${JSON.stringify(result)}`); + if (!String(result?.result || "").includes("订单创建成功：RPA 客户，金额 128")) { + throw new Error(`演示结果不正确：${JSON.stringify(result)}`); + } + if (result.failed?.length) throw new Error(`存在失败节点：${JSON.stringify(result.failed)}`); + console.log(JSON.stringify({ ok: true, ...result }, null, 2)); +} finally { + child.kill(); + await new Promise((resolve) => setTimeout(resolve, 300)); + fs.rmSync(profile, { recursive: true, force: true }); +} diff --git a/src/main/browser/session.ts b/src/main/browser/session.ts index 90ba211..f596f74 100644 --- a/src/main/browser/session.ts +++ b/src/main/browser/session.ts @@ -40,6 +40,7 @@ interface ElementInfo { visible: boolean; disabled: boolean; rect: { x: number; y: number; width: number; height: number }; + count: number; } function selectorSpec(selector: unknown): { kind: string; value: string; role?: string; name?: string; attr?: string } { @@ -93,7 +94,7 @@ export class BrowserSession implements BrowserLike { const win = new BrowserWindow({ width: 1200, height: 820, - show: true, + show: process.env.RPA_E2E !== "1", title: "RPA 运行浏览器", backgroundColor: "#ffffff", webPreferences: { @@ -200,35 +201,37 @@ export class BrowserSession implements BrowserLike { const spec = selectorSpec(selector); const expression = `(() => { const spec = ${JSON.stringify(spec)}; - const pick = (nodes) => { + const visible = (nodes) => { + const result = []; for (const el of nodes) { const style = getComputedStyle(el); const rect = el.getBoundingClientRect(); if (rect.width <= 0 || rect.height <= 0) continue; if (style.display === 'none' || style.visibility === 'hidden' || Number(style.opacity) === 0) continue; - return el; + result.push(el); } - return null; + return result; }; - let el = null; + let matches = []; if (spec.kind === 'text') { const target = spec.value; const all = [...document.querySelectorAll('*')].filter((node) => { const text = (node.innerText || node.textContent || '').trim(); return text === target || (text.includes(target) && text.length <= target.length + 20); }); - el = pick(all.sort((a, b) => (a.innerText || '').length - (b.innerText || '').length)); + matches = visible(all.sort((a, b) => (a.innerText || '').length - (b.innerText || '').length)); } else if (spec.kind === 'aria') { const role = spec.role || spec.value; const name = spec.name || ''; const all = [...document.querySelectorAll('[role="' + role + '"], ' + role)]; - el = pick(all.filter((node) => { + matches = visible(all.filter((node) => { const acc = node.getAttribute('aria-label') || node.innerText || ''; return !name || acc.includes(name); })); } else { - el = pick([...document.querySelectorAll(spec.value)]); + matches = visible([...document.querySelectorAll(spec.value)]); } + const el = matches[0] || null; if (!el) return null; el.scrollIntoView({ block: 'center', inline: 'center' }); const rect = el.getBoundingClientRect(); @@ -238,6 +241,7 @@ export class BrowserSession implements BrowserLike { value: el.value ?? '', visible: rect.width > 0 && rect.height > 0, disabled: Boolean(el.disabled), + count: matches.length, rect: { x: rect.x, y: rect.y, width: rect.width, height: rect.height } }; })()`; @@ -247,14 +251,21 @@ export class BrowserSession implements BrowserLike { recommendation: "检查选择器，或先用拾取器重新采集" }); } + if (info.count > 1) { + throw new RuntimeError("ambiguous", `选择器匹配到 ${info.count} 个可见元素：${spec.kind}:${spec.value}`, { + recommendation: "使用更稳定的 data-testid、aria-label 或唯一 CSS 选择器" + }); + } return { ...info, selector: `${spec.kind}:${spec.value}` }; } private async mouseClick(x: number, y: number): Promise { const point = { x: Math.round(x), y: Math.round(y) }; - await this.cdp("Input.dispatchMouseEvent", { type: "mouseMoved", ...point, button: "none" }); - await this.cdp("Input.dispatchMouseEvent", { type: "mousePressed", ...point, button: "left", clickCount: 1 }); - await this.cdp("Input.dispatchMouseEvent", { type: "mouseReleased", ...point, button: "left", clickCount: 1 }); + await this.cdp("Input.dispatchMouseEvent", { type: "mouseMoved", ...point, button: "none", buttons: 0 }); + await new Promise((resolve) => setTimeout(resolve, 35)); + await this.cdp("Input.dispatchMouseEvent", { type: "mousePressed", ...point, button: "left", buttons: 1, clickCount: 1 }); + await new Promise((resolve) => setTimeout(resolve, 25)); + await this.cdp("Input.dispatchMouseEvent", { type: "mouseReleased", ...point, button: "left", buttons: 0, clickCount: 1 }); } async click(selector: unknown): Promise { @@ -265,8 +276,26 @@ export class BrowserSession implements BrowserLike { if (info.disabled) { throw new RuntimeError("not_found", `元素已禁用：${info.selector}`); } + const clickable = await this.withElement(selector, ` + const rect = el.getBoundingClientRect(); + const hit = document.elementFromPoint(rect.left + rect.width / 2, rect.top + rect.height / 2); + return Boolean(hit && (hit === el || el.contains(hit) || hit.contains(el))); + `); + if (!clickable) throw new RuntimeError("not_found", `目标元素被其他内容遮挡：${info.selector}`, { recommendation: "关闭遮挡弹窗或等待页面稳定" }); + const clickToken = `rpa_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 7)}`; + await this.withElement(selector, ` + const token = ${JSON.stringify(clickToken)}; + el.addEventListener('click', () => { el.dataset.rpaClickToken = token; }, { once: true }); + return true; + `); await this.mouseClick(info.rect.x + info.rect.width / 2, info.rect.y + info.rect.height / 2); await new Promise((resolve) => setTimeout(resolve, 180)); + const clicked = await this.withElement(selector, ` + const matched = el.dataset.rpaClickToken === ${JSON.stringify(clickToken)}; + delete el.dataset.rpaClickToken; + return matched; + `); + if (!clicked) throw new RuntimeError("not_found", `目标未收到可信点击：${info.selector}`, { recommendation: "检查元素是否被遮挡或重新拾取元素" }); return { selector: info.selector, tag: info.tag, text: info.text }; } @@ -274,17 +303,15 @@ export class BrowserSession implements BrowserLike { const info = await this.locate(selector); await this.mouseClick(info.rect.x + info.rect.width / 2, info.rect.y + info.rect.height / 2); if (clearFirst) { - await this.evaluate(`(() => { - const el = document.activeElement; - if (el && (el.tagName === 'INPUT' || el.tagName === 'TEXTAREA')) { el.select(); return true; } - return false; - })()`); - await this.cdp("Input.dispatchKeyEvent", { type: "keyDown", key: "Backspace", code: "Backspace", windowsVirtualKeyCode: 8 }); - await this.cdp("Input.dispatchKeyEvent", { type: "keyUp", key: "Backspace", code: "Backspace", windowsVirtualKeyCode: 8 }); + await this.cdp("Input.dispatchKeyEvent", { type: "rawKeyDown", key: "a", code: "KeyA", windowsVirtualKeyCode: 65, modifiers: 2 }); + await this.cdp("Input.dispatchKeyEvent", { type: "keyUp", key: "a", code: "KeyA", windowsVirtualKeyCode: 65, modifiers: 2 }); + await new Promise((resolve) => setTimeout(resolve, 30)); } await this.cdp("Input.insertText", { text }); await new Promise((resolve) => setTimeout(resolve, 120)); - const actual = await this.evaluate(`(() => { const el = document.activeElement; return el ? (el.value ?? '') : ''; })()`); + const expected = clearFirst ? text : `${info.value || ""}${text}`; + const actual = await this.evaluate(`(() => { const el = document.activeElement; return el ? (el.value ?? el.textContent ?? '') : ''; })()`); + if (actual !== expected) throw new RuntimeError("evaluate_error", `输入校验失败：期望「${expected}」，实际「${actual}」`, { recommendation: "检查输入框状态或重新拾取元素" }); return { selector: info.selector, value: actual }; } diff --git a/src/main/index.ts b/src/main/index.ts index d0f1ae1..f416fd0 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -11,6 +11,7 @@ import { fixtureUrl, startFixtureServer, stopFixtureServer } from "./fixture/ser let mainWindow: BrowserWindow | null = null; function createWindow(): void { + const e2e = process.env.RPA_E2E === "1"; mainWindow = new BrowserWindow({ width: 1440, height: 900, @@ -33,7 +34,7 @@ function createWindow(): void { } }); - mainWindow.on("ready-to-show", () => mainWindow?.show()); + mainWindow.on("ready-to-show", () => { if (!e2e) mainWindow?.show(); }); mainWindow.on("closed", () => { mainWindow = null; }); @@ -115,7 +116,7 @@ if (!gotLock) { app.whenReady().then(async () => { try { - await startFixtureServer(); + await startFixtureServer(Number(process.env.RPA_FIXTURE_PORT || 17863)); } catch (error) { console.warn("local fixture server unavailable", error); }