fix: 对齐浏览器可信输入并增加端到端门禁

2026-10-27 14:18:51 +08:00 · 2026-10-27 14:18:51 +08:00 · 65591322ee
commit 65591322ee
parent 6adf772987
6 changed files with 146 additions and 22 deletions
--- a/README.md
+++ b/README.md
@ -25,6 +25,7 @@ npm run dev:local        # 使用 RPA_ELECTRON_DIST 或 C:\electron-rpa\dist 的
 # 或
 npm run build
 node scripts/start.mjs --no-sandbox
+npm run test:e2e       # 隔离启动应用并执行本地网页订单完整流程
 ```

 > 为什么需要 `dev:local`：当前工作区 `D:\RPA` 的 ACL 会让 Electron 自带的 Chromium 沙箱启动失败（`--version` 都返回 0x80000003）。
@ -52,5 +53,6 @@ docs              现状分析、实施方案

 - 1016 项目录中当前有 398 项通过能力注册表或运行时容器标记为可执行，覆盖控制/触发、字符串/集合/INI/二维表格、文件、浏览器元素常用及 DOM 高级能力、HTTP、Excel/CSV与表格辅助、系统路径与环境变量、进程管理、桌面键鼠/窗口对象/剪贴板/对话框、PDF、Word、邮件发送、屏幕截图、离线图像查找与点击；其余灰标「待接入」。运行前检查会提前定位待接入组件、必填参数、多出口与断链节点。
 - 内置本地演示站默认监听 `127.0.0.1:17863`；新用户首次启动会获得“本地网页订单演示”任务，可离线验证网页输入、点击和文本读取。
+- 浏览器可信输入与定位机制参考 `C:\rpa-demo`：CDP Input、唯一定位、遮挡检查和输入回读校验；不使用 `el.click()` 伪造点击。
 - 不包含厂商代码、二进制、模型或前端资产；目录是从厂商元数据派生的静态定义。
 - 运行期不访问云服务；HTTP/邮件属于用户显式配置的外部系统。
--- a/docs/02-实施方案.md
+++ b/docs/02-实施方案.md
@ -6,6 +6,8 @@

 > **2026-10-27 当前状态**：计划点名的代表能力已全部进入注册表或运行时容器，覆盖控制/触发、字符串/集合/INI/二维表格、文件、浏览器元素常用及 DOM 高级能力、HTTP、Excel/CSV与表格辅助、系统路径与环境变量、进程管理、桌面键鼠/窗口对象/剪贴板/对话框、PDF、Word、邮件发送、屏幕截图、离线图像查找与点击；启动后目录显示 **398 项可执行 / 1016 项总目录**。当前 15 个测试文件共 99 条测试，覆盖计划能力审计、浏览器参数路由/变体/DOM、Excel/PDF/Word/SMTP/模板匹配、文件与数据扩展、系统/进程/窗口解析、直线/IF/循环/重试/跳过、主出口与异常出口语义、运行记录回写、运行前检查、本地演示站与示例流程。设置页可查看分类覆盖和处理器 ID 数量。未通过实际场景验证的变体仍不计入最终交付数量。

+> 浏览器端到端门禁：`npm run test:e2e` 使用隔离用户目录启动应用和本地演示站，真实执行打开网页、两次可信输入、点击提交和读取结果；期望结果必须精确包含“订单创建成功：RPA 客户，金额 128”。浏览器输入、点击、定位歧义和遮挡检查机制已对照 `C:\rpa-demo/actions.js` 校准。
+
 ---

 ## 0. 决策记录（2026-10-26 确认）
--- a/package.json
+++ b/package.json
@ -13,7 +13,8 @@
    "typecheck": "tsc --noEmit -p tsconfig.json",
    "catalog": "node scripts/build-catalog.mjs",
    "package": "node scripts/package.mjs",
-    "test": "vitest run"
+    "test": "vitest run",
+    "test:e2e": "node scripts/demo-e2e.mjs"
  },
  "dependencies": {
    "@xyflow/react": "^12.12.0",
--- a/scripts/demo-e2e.mjs
+++ b/scripts/demo-e2e.mjs
@ -0,0 +1,91 @@
+import { spawn } from "node:child_process";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+const root = process.cwd();
+const debugPort = 9231;
+const fixturePort = 17963;
+const profile = fs.mkdtempSync(path.join(os.tmpdir(), "rpa-studio-e2e-"));
+const candidates = [process.env.RPA_ELECTRON_DIST, "C:/electron-rpa/dist", path.join(root, "node_modules", "electron", "dist")].filter(Boolean);
+const dist = candidates.find((candidate) => fs.existsSync(path.join(candidate, "electron.exe")));
+if (!dist) throw new Error("找不到 Electron 运行时，请设置 RPA_ELECTRON_DIST");
+
+const env = { ...process.env, RPA_E2E: "1", RPA_FIXTURE_PORT: String(fixturePort) };
+delete env.ELECTRON_RUN_AS_NODE;
+const child = spawn(path.join(dist, "electron.exe"), [root, "--no-sandbox", `--remote-debugging-port=${debugPort}`, `--user-data-dir=${profile}`], {
+  cwd: root,
+  env,
+  stdio: ["ignore", "pipe", "pipe"],
+  windowsHide: true
+});
+
+let stderr = "";
+child.stderr.on("data", (chunk) => { stderr += String(chunk); });
+
+async function waitForTarget() {
+  const deadline = Date.now() + 30000;
+  while (Date.now() < deadline) {
+    if (child.exitCode !== null) throw new Error(`RPA Studio 提前退出：${child.exitCode}\n${stderr}`);
+    try {
+      const targets = await (await fetch(`http://127.0.0.1:${debugPort}/json/list`)).json();
+      const page = targets.find((item) => item.type === "page" && item.title === "RPA Studio");
+      if (page) return page;
+    } catch { /* wait for CDP */ }
+    await new Promise((resolve) => setTimeout(resolve, 200));
+  }
+  throw new Error(`等待 RPA Studio CDP 超时\n${stderr}`);
+}
+
+async function evaluate(page, expression) {
+  const socket = new WebSocket(page.webSocketDebuggerUrl);
+  await new Promise((resolve, reject) => { socket.onopen = resolve; socket.onerror = reject; });
+  const result = await new Promise((resolve, reject) => {
+    const timer = setTimeout(() => reject(new Error("E2E Runtime.evaluate 超时")), 60000);
+    socket.onmessage = (event) => {
+      const message = JSON.parse(event.data);
+      if (message.id !== 1) return;
+      clearTimeout(timer);
+      if (message.error) reject(new Error(JSON.stringify(message.error)));
+      else if (message.result?.exceptionDetails) reject(new Error(message.result.exceptionDetails.text || "E2E 页面异常"));
+      else resolve(message.result?.result?.value);
+    };
+    socket.send(JSON.stringify({ id: 1, method: "Runtime.evaluate", params: { expression, returnByValue: true, awaitPromise: true } }));
+  });
+  socket.close();
+  return result;
+}
+
+const expression = `(async () => {
+  const flow = await window.rpa.flows.createDemo();
+  const events = [];
+  const off = window.rpa.run.onEvent((event) => events.push(event));
+  const started = await window.rpa.run.start(flow);
+  const finished = await new Promise((resolve, reject) => {
+    const deadline = Date.now() + 45000;
+    const timer = setInterval(() => {
+      const event = events.find((item) => item.type === 'run.finished');
+      if (event) { clearInterval(timer); resolve(event); }
+      else if (Date.now() > deadline) { clearInterval(timer); reject(new Error('demo flow timeout')); }
+    }, 100);
+  });
+  off();
+  await window.rpa.flows.delete(flow.id);
+  const result = events.find((item) => item.type === 'node.finished' && item.nodeId === 'node_result');
+  return { runId: started.runId, status: finished.status, durationMs: finished.durationMs, result: result?.output, failed: events.filter((item) => item.type === 'node.finished' && item.status === 'failed') };
+})()`;
+
+try {
+  const page = await waitForTarget();
+  const result = await evaluate(page, expression);
+  if (result?.status !== "succeeded") throw new Error(`演示流程失败：${JSON.stringify(result)}`);
+  if (!String(result?.result || "").includes("订单创建成功：RPA 客户，金额 128")) {
+    throw new Error(`演示结果不正确：${JSON.stringify(result)}`);
+  }
+  if (result.failed?.length) throw new Error(`存在失败节点：${JSON.stringify(result.failed)}`);
+  console.log(JSON.stringify({ ok: true, ...result }, null, 2));
+} finally {
+  child.kill();
+  await new Promise((resolve) => setTimeout(resolve, 300));
+  fs.rmSync(profile, { recursive: true, force: true });
+}
--- a/src/main/browser/session.ts
+++ b/src/main/browser/session.ts
@ -40,6 +40,7 @@ interface ElementInfo {
  visible: boolean;
  disabled: boolean;
  rect: { x: number; y: number; width: number; height: number };
+  count: number;
 }

 function selectorSpec(selector: unknown): { kind: string; value: string; role?: string; name?: string; attr?: string } {
@ -93,7 +94,7 @@ export class BrowserSession implements BrowserLike {
    const win = new BrowserWindow({
      width: 1200,
      height: 820,
-      show: true,
+      show: process.env.RPA_E2E !== "1",
      title: "RPA 运行浏览器",
      backgroundColor: "#ffffff",
      webPreferences: {
@ -200,35 +201,37 @@ export class BrowserSession implements BrowserLike {
    const spec = selectorSpec(selector);
    const expression = `(() => {
      const spec = ${JSON.stringify(spec)};
-      const pick = (nodes) => {
+      const visible = (nodes) => {
+        const result = [];
        for (const el of nodes) {
          const style = getComputedStyle(el);
          const rect = el.getBoundingClientRect();
          if (rect.width <= 0 || rect.height <= 0) continue;
          if (style.display === 'none' || style.visibility === 'hidden' || Number(style.opacity) === 0) continue;
-          return el;
+          result.push(el);
        }
-        return null;
+        return result;
      };
-      let el = null;
+      let matches = [];
      if (spec.kind === 'text') {
        const target = spec.value;
        const all = [...document.querySelectorAll('*')].filter((node) => {
          const text = (node.innerText || node.textContent || '').trim();
          return text === target || (text.includes(target) && text.length <= target.length + 20);
        });
-        el = pick(all.sort((a, b) => (a.innerText || '').length - (b.innerText || '').length));
+        matches = visible(all.sort((a, b) => (a.innerText || '').length - (b.innerText || '').length));
      } else if (spec.kind === 'aria') {
        const role = spec.role || spec.value;
        const name = spec.name || '';
        const all = [...document.querySelectorAll('[role="' + role + '"], ' + role)];
-        el = pick(all.filter((node) => {
+        matches = visible(all.filter((node) => {
          const acc = node.getAttribute('aria-label') || node.innerText || '';
          return !name || acc.includes(name);
        }));
      } else {
-        el = pick([...document.querySelectorAll(spec.value)]);
+        matches = visible([...document.querySelectorAll(spec.value)]);
      }
+      const el = matches[0] || null;
      if (!el) return null;
      el.scrollIntoView({ block: 'center', inline: 'center' });
      const rect = el.getBoundingClientRect();
@ -238,6 +241,7 @@ export class BrowserSession implements BrowserLike {
        value: el.value ?? '',
        visible: rect.width > 0 && rect.height > 0,
        disabled: Boolean(el.disabled),
+        count: matches.length,
        rect: { x: rect.x, y: rect.y, width: rect.width, height: rect.height }
      };
    })()`;
@ -247,14 +251,21 @@ export class BrowserSession implements BrowserLike {
        recommendation: "检查选择器，或先用拾取器重新采集"
      });
    }
+    if (info.count > 1) {
+      throw new RuntimeError("ambiguous", `选择器匹配到 ${info.count} 个可见元素：${spec.kind}:${spec.value}`, {
+        recommendation: "使用更稳定的 data-testid、aria-label 或唯一 CSS 选择器"
+      });
+    }
    return { ...info, selector: `${spec.kind}:${spec.value}` };
  }

  private async mouseClick(x: number, y: number): Promise<void> {
    const point = { x: Math.round(x), y: Math.round(y) };
-    await this.cdp("Input.dispatchMouseEvent", { type: "mouseMoved", ...point, button: "none" });
-    await this.cdp("Input.dispatchMouseEvent", { type: "mousePressed", ...point, button: "left", clickCount: 1 });
-    await this.cdp("Input.dispatchMouseEvent", { type: "mouseReleased", ...point, button: "left", clickCount: 1 });
+    await this.cdp("Input.dispatchMouseEvent", { type: "mouseMoved", ...point, button: "none", buttons: 0 });
+    await new Promise((resolve) => setTimeout(resolve, 35));
+    await this.cdp("Input.dispatchMouseEvent", { type: "mousePressed", ...point, button: "left", buttons: 1, clickCount: 1 });
+    await new Promise((resolve) => setTimeout(resolve, 25));
+    await this.cdp("Input.dispatchMouseEvent", { type: "mouseReleased", ...point, button: "left", buttons: 0, clickCount: 1 });
  }

  async click(selector: unknown): Promise<unknown> {
@ -265,8 +276,26 @@ export class BrowserSession implements BrowserLike {
    if (info.disabled) {
      throw new RuntimeError("not_found", `元素已禁用：${info.selector}`);
    }
+    const clickable = await this.withElement<boolean>(selector, `
+      const rect = el.getBoundingClientRect();
+      const hit = document.elementFromPoint(rect.left + rect.width / 2, rect.top + rect.height / 2);
+      return Boolean(hit && (hit === el || el.contains(hit) || hit.contains(el)));
+    `);
+    if (!clickable) throw new RuntimeError("not_found", `目标元素被其他内容遮挡：${info.selector}`, { recommendation: "关闭遮挡弹窗或等待页面稳定" });
+    const clickToken = `rpa_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 7)}`;
+    await this.withElement(selector, `
+      const token = ${JSON.stringify(clickToken)};
+      el.addEventListener('click', () => { el.dataset.rpaClickToken = token; }, { once: true });
+      return true;
+    `);
    await this.mouseClick(info.rect.x + info.rect.width / 2, info.rect.y + info.rect.height / 2);
    await new Promise((resolve) => setTimeout(resolve, 180));
+    const clicked = await this.withElement<boolean>(selector, `
+      const matched = el.dataset.rpaClickToken === ${JSON.stringify(clickToken)};
+      delete el.dataset.rpaClickToken;
+      return matched;
+    `);
+    if (!clicked) throw new RuntimeError("not_found", `目标未收到可信点击：${info.selector}`, { recommendation: "检查元素是否被遮挡或重新拾取元素" });
    return { selector: info.selector, tag: info.tag, text: info.text };
  }

@ -274,17 +303,15 @@ export class BrowserSession implements BrowserLike {
    const info = await this.locate(selector);
    await this.mouseClick(info.rect.x + info.rect.width / 2, info.rect.y + info.rect.height / 2);
    if (clearFirst) {
-      await this.evaluate(`(() => {
-        const el = document.activeElement;
-        if (el && (el.tagName === 'INPUT' || el.tagName === 'TEXTAREA')) { el.select(); return true; }
-        return false;
-      })()`);
-      await this.cdp("Input.dispatchKeyEvent", { type: "keyDown", key: "Backspace", code: "Backspace", windowsVirtualKeyCode: 8 });
-      await this.cdp("Input.dispatchKeyEvent", { type: "keyUp", key: "Backspace", code: "Backspace", windowsVirtualKeyCode: 8 });
+      await this.cdp("Input.dispatchKeyEvent", { type: "rawKeyDown", key: "a", code: "KeyA", windowsVirtualKeyCode: 65, modifiers: 2 });
+      await this.cdp("Input.dispatchKeyEvent", { type: "keyUp", key: "a", code: "KeyA", windowsVirtualKeyCode: 65, modifiers: 2 });
+      await new Promise((resolve) => setTimeout(resolve, 30));
    }
    await this.cdp("Input.insertText", { text });
    await new Promise((resolve) => setTimeout(resolve, 120));
-    const actual = await this.evaluate<string>(`(() => { const el = document.activeElement; return el ? (el.value ?? '') : ''; })()`);
+    const expected = clearFirst ? text : `${info.value || ""}${text}`;
+    const actual = await this.evaluate<string>(`(() => { const el = document.activeElement; return el ? (el.value ?? el.textContent ?? '') : ''; })()`);
+    if (actual !== expected) throw new RuntimeError("evaluate_error", `输入校验失败：期望「${expected}」，实际「${actual}」`, { recommendation: "检查输入框状态或重新拾取元素" });
    return { selector: info.selector, value: actual };
  }

--- a/src/main/index.ts
+++ b/src/main/index.ts
@ -11,6 +11,7 @@ import { fixtureUrl, startFixtureServer, stopFixtureServer } from "./fixture/ser
 let mainWindow: BrowserWindow | null = null;

 function createWindow(): void {
+  const e2e = process.env.RPA_E2E === "1";
  mainWindow = new BrowserWindow({
    width: 1440,
    height: 900,
@ -33,7 +34,7 @@ function createWindow(): void {
    }
  });

-  mainWindow.on("ready-to-show", () => mainWindow?.show());
+  mainWindow.on("ready-to-show", () => { if (!e2e) mainWindow?.show(); });
  mainWindow.on("closed", () => {
    mainWindow = null;
  });
@ -115,7 +116,7 @@ if (!gotLock) {

  app.whenReady().then(async () => {
    try {
-      await startFixtureServer();
+      await startFixtureServer(Number(process.env.RPA_FIXTURE_PORT || 17863));
    } catch (error) {
      console.warn("local fixture server unavailable", error);
    }