Merge branch 'main' of http://43.140.215.244:7040/wangyukun/PRA-only-demo

2026-11-02 14:22:41 +08:00 · 2026-11-02 14:22:41 +08:00 · e5f64c21e2
commit e5f64c21e2
parent 1ddd1f1b35 b1fc8c25b8
6 changed files with 194 additions and 34 deletions
--- a/scripts/app-system-tests.mjs
+++ b/scripts/app-system-tests.mjs
@ -47,9 +47,13 @@ export async function validator() {

 export async function verifyBundle(catalog = staticCatalog()) {
  const { manifest, groups } = readBundle();
-  const expected = catalog.components.filter(c => c.group === "app" && c.registered).map(c => c.id).sort();
+  const expected = catalog.components.filter(c => c.group === "app" && (c.registered || c.implemented)).map(c => c.id).sort();
  const actual = groups.flatMap(g => g.entry.components).sort();
-  if (groups.length !== 13 || actual.length !== 50 || new Set(actual).size !== 50 || JSON.stringify(actual) !== JSON.stringify(expected)) throw new Error("流程包与当前 50 项验收清单不一致，请重新生成并核对覆盖范围。");
+  if (groups.length !== 13 || actual.length !== 50 || new Set(actual).size !== 50 || JSON.stringify(actual) !== JSON.stringify(expected)) {
+    const expectedOnly = expected.filter(id => !actual.includes(id));
+    const actualOnly = actual.filter(id => !expected.includes(id));
+    throw new Error(`流程包与当前 50 项验收清单不一致；expectedOnly=${JSON.stringify(expectedOnly)}；actualOnly=${JSON.stringify(actualOnly)}`);
+  }
  const validateFlow = await validator();
  const validation = {};
  for (const { flow, entry } of groups) {
@ -248,6 +252,9 @@ export async function runSafe(outputDirectory, checkUi = false) {
    const info = await evaluate(page, "window.rpa.app.info()");
    if (path.resolve(info.userData).toLowerCase() !== profile.toLowerCase()) throw new Error("隔离用户目录不匹配，已拒绝执行测试。");
    const catalog = await evaluate(page, "window.rpa.catalog.get()");
+    const runtime = await evaluate(page, "window.rpa.run.capabilities()");
+    const registered = new Set(runtime.ids || []);
+    for (const component of catalog.components) component.registered = registered.has(component.id) || registered.has(component.functionName);
    const checked = await verifyBundle(catalog);
    for (const { flow, entry } of checked.groups) {
      const issues = checked.validation[entry.id];
--- a/src/main/browser/external.ts
+++ b/src/main/browser/external.ts
@ -24,6 +24,8 @@ export interface ExternalLaunchOptions {
  extraArgs?: string[];
  /** 独立用户数据目录；缺省使用系统临时目录下的独立 profile */
  userDataDir?: string;
+  /** 首次启动时直接打开的 URL，避免先出现 about:blank */
+  initialUrl?: string;
 }

 export interface ExternalRemoteOptions {
@ -225,12 +227,13 @@ export async function launchExternalBrowser(options: ExternalLaunchOptions): Pro
  await fs.mkdir(userDataDir, { recursive: true });

  const extraArgs = (options.extraArgs ?? []).filter(Boolean);
+  const startUrl = options.initialUrl?.trim() || "about:blank";
  const args = [
    `--remote-debugging-port=${port}`,
    `--user-data-dir=${userDataDir}`,
    "--no-first-run",
    "--no-default-browser-check",
-    "about:blank",
+    startUrl,
    ...extraArgs
  ];

--- a/src/main/browser/session.ts
+++ b/src/main/browser/session.ts
@ -166,6 +166,16 @@ export class BrowserSession implements BrowserLike {
  }

  /** 当前会话是否为外部浏览器 */
+  /** 会话底层 CDP 是否已断开（缓存清理用） */
+  get closed(): boolean {
+    return this.transport.closed;
+  }
+
+  /** 缓存键，外部会话排序用 */
+  get sessionKey(): string {
+    return this.externalKey;
+  }
+
  get isExternal(): boolean {
    return this.external !== null;
  }
@ -1157,6 +1167,10 @@ export class BrowserSession implements BrowserLike {
      let box = null;
      let badge = null;
      let target = null;
+      let locked = false;
+      let chain = [];
+      let chainIndex = 0;
+      let selected = null;
      let timer = 0;
      let finished = false;

@ -1225,18 +1239,24 @@ export class BrowserSession implements BrowserLike {
          url: location.href
        };
      };
+      const nodeName = (node) => node.getAttribute('aria-label') || node.getAttribute('placeholder') || node.getAttribute('name') || node.id || (node.innerText || '').trim().slice(0, 60) || '';
      const updateBox = () => {
-        if (!target || !box || !badge) return;
-        const rect = target.getBoundingClientRect();
+        const node = locked ? selected : target;
+        if (!node || !box || !badge) return;
+        const rect = node.getBoundingClientRect();
        box.style.left = Math.max(0, rect.left - 3) + 'px';
        box.style.top = Math.max(0, rect.top - 3) + 'px';
        box.style.width = Math.max(4, rect.width + 6) + 'px';
        box.style.height = Math.max(4, rect.height + 6) + 'px';
-        const tag = target.tagName.toLowerCase();
-        const name = target.getAttribute('aria-label') || target.getAttribute('placeholder') || target.getAttribute('name') || target.id || '';
-        badge.textContent = '拾取：' + tag + (name ? ' · ' + name.slice(0, 60) : '');
+        const tag = node.tagName.toLowerCase();
+        const name = nodeName(node);
+        if (locked) {
+          badge.textContent = '已锁定 ' + (chainIndex + 1) + '/' + chain.length + ' · ' + tag + (name ? ' · ' + name.slice(0, 42) : '') + ' · [PageUp/PageDown]调层级 [空格]确认 [Esc]取消';
+        } else {
+          badge.textContent = '拾取 · ' + tag + (name ? ' · ' + name.slice(0, 42) : '') + ' · [空格]锁定 [Esc]取消';
+        }
        badge.style.left = Math.max(0, rect.left) + 'px';
-        badge.style.top = Math.max(0, rect.top - 30) + 'px';
+        badge.style.top = Math.max(0, rect.top - 34) + 'px';
      };
      const interactiveSelector = [
        'button', 'a', 'input', 'select', 'textarea', 'summary', 'label', 'option',
@ -1266,20 +1286,48 @@ export class BrowserSession implements BrowserLike {
        if (!deepest && event.target instanceof Element) deepest = event.target;
        return closestInteractive(deepest);
      };
+      const buildChain = (node) => {
+        const result = [];
+        let current = node;
+        while (current && current.nodeType === 1) {
+          result.unshift(current);
+          const root = current.getRootNode();
+          current = root instanceof ShadowRoot ? root.host : null;
+        }
+        return result;
+      };
+      const lock = () => {
+        if (locked || !target) return;
+        locked = true;
+        chain = buildChain(target);
+        chainIndex = Math.max(0, chain.length - 1);
+        selected = chain[chainIndex] || target;
+        updateBox();
+      };
+      const adjustLevel = (delta) => {
+        if (!locked || !chain.length) return;
+        const next = Math.min(chain.length - 1, Math.max(0, chainIndex + delta));
+        if (next === chainIndex) return;
+        chainIndex = next;
+        selected = chain[chainIndex];
+        updateBox();
+      };
+      const confirm = () => {
+        if (!locked || !selected) return;
+        cleanup();
+        resolve(describe(selected));
+      };
      const move = (event) => {
-        if (finished) return;
+        if (finished || locked) return;
        target = firstElement(event);
        updateBox();
      };
      const click = (event) => {
        if (finished) return;
-        const selected = firstElement(event);
-        if (!selected) return;
+        // 与桌面拾取对齐：鼠标只用于定位，空格锁定/确认，避免误触发页面按钮。
        event.preventDefault();
        event.stopPropagation();
        event.stopImmediatePropagation();
-        cleanup();
-        resolve(describe(selected));
      };
      const key = (event) => {
        if (event.key === 'Escape') {
@ -1287,6 +1335,25 @@ export class BrowserSession implements BrowserLike {
          event.stopPropagation();
          cleanup();
          reject(new Error('已取消元素拾取'));
+          return;
+        }
+        if (event.code === 'Space' && !event.repeat) {
+          event.preventDefault();
+          event.stopPropagation();
+          if (locked) confirm();
+          else lock();
+          return;
+        }
+        if (locked && event.key === 'PageUp' && !event.repeat) {
+          event.preventDefault();
+          event.stopPropagation();
+          adjustLevel(-1);
+          return;
+        }
+        if (locked && event.key === 'PageDown' && !event.repeat) {
+          event.preventDefault();
+          event.stopPropagation();
+          adjustLevel(1);
        }
      };
      const cleanup = () => {
@ -1317,6 +1384,7 @@ export class BrowserSession implements BrowserLike {
      window.addEventListener('click', click, true);
      window.addEventListener('scroll', updateBox, true);
      window.addEventListener('resize', updateBox);
+      window.addEventListener('keyup', () => {}, true);
      window.addEventListener('keydown', key, true);
      window.__rpaElementPicker = { cancel: () => { cleanup(); reject(new Error('已取消元素拾取')); } };
      timer = setTimeout(() => {
@ -1370,6 +1438,16 @@ export async function externalBrowserSession(options: ExternalBrowserOptions): P
  return BrowserSession.createExternal(options);
 }

+export function listExternalSessions(): BrowserSession[] {
+  return [...externalSessionCache().values()]
+    .filter((session) => session.isExternal && !session.closed)
+    .sort((left, right) => right.sessionKey.localeCompare(left.sessionKey));
+}
+
+export function latestExternalSession(): BrowserSession | null {
+  return listExternalSessions()[0] || null;
+}
+
 export function hasBrowserSession(): boolean {
  return Boolean(globalThis.__rpaBrowserWindow && !globalThis.__rpaBrowserWindow.isDestroyed());
 }
--- a/src/main/index.ts
+++ b/src/main/index.ts
@ -1,7 +1,13 @@
 import { app, BrowserWindow, dialog, ipcMain, shell } from "electron";
 import path from "node:path";
 import type { AppInfo, AppSettings, FlowDefinition, FlowSaveOptions, FlowSaveResult } from "../shared/types";
-import { BrowserSession, hasBrowserSession, setBrowserElementWaitMs } from "./browser/session";
+import {
+  BrowserSession,
+  externalBrowserSession,
+  hasBrowserSession,
+  latestExternalSession,
+  setBrowserElementWaitMs
+} from "./browser/session";
 import "./capabilities";
 import { catalogPath, loadCatalog } from "./catalog";
 import { createDemoFlow, createExampleFlows, createFlow, deleteFlow, getFlow, listFlows, saveFlow } from "./flows";
@ -103,14 +109,30 @@ function registerIpc(): void {
    setBrowserElementWaitMs(settings.browserElementWaitMs);
    return settings;
  });
-  ipcMain.handle("browser:pick-element", async (_event, options: { url?: string } = {}) => {
+  type BrowserPickOptions = {
+    url?: string;
+    browser?: import("./browser/external").ExternalBrowserOptions;
+  };
+
+  ipcMain.handle("browser:pick-element", async (_event, options: BrowserPickOptions = {}) => {
    const url = options.url?.trim();
-    if (!hasBrowserSession() && !url) {
-      throw new Error("请先配置或运行「打开网页」，再拾取元素");
+    const browserOptions = options.browser;
+    let session;
+    if (browserOptions) {
+      session = await externalBrowserSession(
+        browserOptions.mode === "launch" && url ? { ...browserOptions, initialUrl: url } : browserOptions
+      );
+    } else {
+      session = latestExternalSession();
+    }
+    if (!session) {
+      if (!hasBrowserSession() && !url) {
+        throw new Error("请先配置或运行「打开网页」，再拾取元素");
+      }
+      session = await BrowserSession.create();
    }
-    const session = await BrowserSession.create();
    const currentUrl = await session.currentUrl();
-    if (url && (currentUrl === "about:blank" || currentUrl !== url)) {
+    if (url && currentUrl !== url) {
      await session.navigate(url);
    }
    return session.pickElement();
--- a/src/preload/index.ts
+++ b/src/preload/index.ts
@ -29,7 +29,7 @@ const api = {
      ipcRenderer.invoke("dialog:save", options || {})
  },
  browser: {
-    pickElement: (options?: { url?: string }): Promise<import("../shared/types").ElementPickResult> =>
+    pickElement: (options?: { url?: string; browser?: { mode: "launch"; browserType: "chrome" | "edge"; port?: number; extraArgs?: string[] } | { mode: "remote"; host: string; port: number } }): Promise<import("../shared/types").ElementPickResult> =>
      ipcRenderer.invoke("browser:pick-element", options || {})
  },
  visionLab: {
--- a/src/renderer/src/panels/Properties.tsx
+++ b/src/renderer/src/panels/Properties.tsx
@ -438,6 +438,49 @@ function PropertyTab({
  );
 }

+interface FileFilterOption {
+  name: string;
+  extensions: string[];
+}
+
+/**
+ * 按组件与字段上下文推导文件选择对话框的类型过滤器。
+ * 先匹配字段本身的含义（字段键/标签），再匹配组件类别；无法识别时退回「全部文件」。
+ */
+function fileFiltersFor(componentId: string, field: CatalogField): FileFilterOption[] {
+  const haystack = `${componentId} ${field.key} ${field.label ?? ""}`;
+  const filters: FileFilterOption[] = [];
+  if (/excel|bivartable|databook|工作簿|表格/i.test(haystack)) {
+    filters.push({ name: "Excel 表格", extensions: ["xlsx", "xlsm", "xls", "csv"] });
+  }
+  if (/word|文档/i.test(haystack)) {
+    filters.push({ name: "Word 文档", extensions: ["docx", "doc"] });
+  }
+  if (/pdf/i.test(haystack)) {
+    filters.push({ name: "PDF 文档", extensions: ["pdf"] });
+  }
+  if (/image|ocr|CV_|vision|图片|图像|截图|模板|照片/i.test(haystack)) {
+    filters.push({ name: "图片文件", extensions: ["png", "jpg", "jpeg", "bmp", "gif", "webp"] });
+  }
+  if (/zip|compress|archive|压缩/i.test(haystack)) {
+    filters.push({ name: "压缩文件", extensions: ["zip", "rar", "7z", "gz"] });
+  }
+  if (/audio|speech|音频|语音|录音/i.test(haystack)) {
+    filters.push({ name: "音频文件", extensions: ["wav", "mp3", "flac", "ogg"] });
+  }
+  if (/video|视频/i.test(haystack)) {
+    filters.push({ name: "视频文件", extensions: ["mp4", "avi", "mkv", "mov"] });
+  }
+  if (/json/i.test(haystack)) {
+    filters.push({ name: "JSON 文件", extensions: ["json"] });
+  }
+  if (!filters.length) {
+    return [{ name: "全部文件", extensions: ["*"] }];
+  }
+  filters.push({ name: "全部文件", extensions: ["*"] });
+  return filters;
+}
+
 function FieldControl({
  field,
  node,
@ -486,12 +529,7 @@ function FieldControl({
      const selected = await window.rpa.dialog.open({
        title: field.label,
        directories,
-        filters: directories
-          ? undefined
-          : [
-              { name: "Excel", extensions: ["xlsx", "xlsm", "xls"] },
-              { name: "全部文件", extensions: ["*"] }
-            ]
+        filters: directories ? undefined : fileFiltersFor(node.componentId, field)
      });
      if (selected) onChange(field.key, selected);
    } catch (error) {
@ -506,12 +544,24 @@ function FieldControl({
      const currentFlow = useEditorStore.getState().flow;
      const nodeIndex = currentFlow?.nodes.findIndex((item) => item.id === node.id) ?? -1;
      const openNode = currentFlow && nodeIndex > 0
-        ? [...currentFlow.nodes.slice(0, nodeIndex)].reverse().find((item) => {
-            const url = textOf(item.values.url).trim();
-            return Boolean(url && !url.startsWith("${"));
-          })
+        ? [...currentFlow.nodes.slice(0, nodeIndex)].reverse().find((item) => /^WebBrowser_Create/.test(item.componentId))
        : undefined;
-      const picked = await window.rpa.browser.pickElement({ url: textOf(openNode?.values.url) || undefined });
+      let browser;
+      if (openNode) {
+        const type = textOf(openNode.values.type).trim().toLowerCase();
+        const isRemote = ["1", "true", "yes"].includes(textOf(openNode.values.isRemote).trim().toLowerCase());
+        const port = Number(textOf(openNode.values.port) || 9222) || 9222;
+        if (isRemote) {
+          browser = { mode: "remote" as const, host: textOf(openNode.values.ipAddress) || "127.0.0.1", port };
+        } else if (type === "chrome" || type === "edge") {
+          const extraArgs = textOf(openNode.values.arguments).split(/\s+/).map((item) => item.trim()).filter(Boolean);
+          browser = { mode: "launch" as const, browserType: type === "edge" ? "edge" as const : "chrome" as const, port, extraArgs };
+        }
+      }
+      const picked = await window.rpa.browser.pickElement({
+        url: textOf(openNode?.values.url).trim() || undefined,
+        browser
+      });
      onChange(field.key, picked.selector);
      onToast(`已拾取 ${picked.tag}${picked.label ? ` · ${picked.label}` : ""}`);
    } catch (error) {
@ -652,8 +702,8 @@ function FieldControl({
              </button>
            </>
          ) : (
-            <button type="button" className="mini-btn" disabled={picking} onClick={() => void pickElement()} title="从运行浏览器拾取元素">
-            {picking ? "拾取中" : "拾取"}
+            <button type="button" className="mini-btn" disabled={picking} onClick={() => void pickElement()} title="从浏览器拾取元素（移到目标按空格锁定，PageUp/PageDown 调层级）">
+            {picking ? "浏览器拾取中" : "浏览器拾取"}
            </button>
          )}
        </div>