feat: 接入浏览器元素拾取

src/main/browser/session.ts

@@ -1,6 +1,7 @@
 import { BrowserWindow } from "electron";
 import { promises as fs } from "node:fs";
 import path from "node:path";
+import type { ElementPickResult } from "../../shared/types";
 import {
   RuntimeError,
   type BrowserElementInfo,
@@ -711,6 +712,155 @@ export class BrowserSession implements BrowserLike {
     return this.win.webContents.getURL();
   }
 
+  async pickElement(timeoutMs = 90000): Promise<import("../../shared/types").ElementPickResult> {
+    const url = this.win.webContents.getURL();
+    if (!url || url === "about:blank") {
+      throw new RuntimeError("no_page", "浏览器还没有打开网页", { recommendation: "先运行或配置「打开网页」，再拾取元素" });
+    }
+    this.win.show();
+    this.win.focus();
+    const result = await this.evaluate<ElementPickResult>(`(() => new Promise((resolve, reject) => {
+      const timeout = ${JSON.stringify(timeoutMs)};
+      if (window.__rpaElementPicker?.cancel) window.__rpaElementPicker.cancel();
+
+      let host = null;
+      let root = null;
+      let box = null;
+      let badge = null;
+      let target = null;
+      let timer = 0;
+      let finished = false;
+
+      const cssEscape = (value) => {
+        if (globalThis.CSS && typeof CSS.escape === 'function') return CSS.escape(String(value));
+        return String(value).replace(/[^a-zA-Z0-9_-]/g, (char) => '\\\\' + char);
+      };
+      const isUnique = (candidate) => {
+        try { return document.querySelectorAll(candidate).length === 1; } catch { return false; }
+      };
+      const cssPath = (node) => {
+        if (node.id) {
+          const candidate = '#' + cssEscape(node.id);
+          if (isUnique(candidate)) return candidate;
+        }
+        const testId = node.getAttribute('data-testid');
+        if (testId) {
+          const candidate = '[data-testid="' + cssEscape(testId) + '"]';
+          if (isUnique(candidate)) return candidate;
+        }
+        const parts = [];
+        let current = node;
+        while (current && current.nodeType === 1) {
+          let part = current.tagName.toLowerCase();
+          if (current.id) {
+            part += '#' + cssEscape(current.id);
+            parts.unshift(part);
+            break;
+          }
+          const parent = current.parentElement;
+          if (parent) {
+            const siblings = [...parent.children].filter((item) => item.tagName === current.tagName);
+            if (siblings.length > 1) part += ':nth-of-type(' + (siblings.indexOf(current) + 1) + ')';
+          }
+          parts.unshift(part);
+          current = parent;
+        }
+        return parts.join(' > ');
+      };
+      const describe = (node) => {
+        const rect = node.getBoundingClientRect();
+        return {
+          selector: 'css:' + cssPath(node),
+          tag: node.tagName.toLowerCase(),
+          id: node.id || undefined,
+          className: node.getAttribute('class') || undefined,
+          name: node.getAttribute('name') || undefined,
+          type: node.getAttribute('type') || undefined,
+          placeholder: node.getAttribute('placeholder') || undefined,
+          label: node.getAttribute('aria-label') || node.getAttribute('placeholder') || node.getAttribute('name') || (node.innerText || '').trim().slice(0, 100) || undefined,
+          text: (node.innerText || node.textContent || '').trim().slice(0, 200) || undefined,
+          value: String(node.value ?? '') || undefined,
+          url: location.href
+        };
+      };
+      const updateBox = () => {
+        if (!target || !box || !badge) return;
+        const rect = target.getBoundingClientRect();
+        box.style.left = Math.max(0, rect.left - 3) + 'px';
+        box.style.top = Math.max(0, rect.top - 3) + 'px';
+        box.style.width = Math.max(4, rect.width + 6) + 'px';
+        box.style.height = Math.max(4, rect.height + 6) + 'px';
+        const tag = target.tagName.toLowerCase();
+        const name = target.getAttribute('aria-label') || target.getAttribute('placeholder') || target.getAttribute('name') || target.id || '';
+        badge.textContent = '拾取：' + tag + (name ? ' · ' + name.slice(0, 60) : '');
+        badge.style.left = Math.max(0, rect.left) + 'px';
+        badge.style.top = Math.max(0, rect.top - 30) + 'px';
+      };
+      const firstElement = (event) => {
+        for (const item of event.composedPath()) if (item instanceof Element) return item;
+        return event.target instanceof Element ? event.target : null;
+      };
+      const move = (event) => {
+        if (finished) return;
+        target = firstElement(event);
+        updateBox();
+      };
+      const click = (event) => {
+        if (finished) return;
+        const selected = firstElement(event);
+        if (!selected) return;
+        event.preventDefault();
+        event.stopPropagation();
+        event.stopImmediatePropagation();
+        cleanup();
+        resolve(describe(selected));
+      };
+      const key = (event) => {
+        if (event.key === 'Escape') {
+          event.preventDefault();
+          event.stopPropagation();
+          cleanup();
+          reject(new Error('已取消元素拾取'));
+        }
+      };
+      const cleanup = () => {
+        if (finished) return;
+        finished = true;
+        clearTimeout(timer);
+        window.removeEventListener('mousemove', move, true);
+        window.removeEventListener('mousedown', click, true);
+        window.removeEventListener('click', click, true);
+        window.removeEventListener('scroll', updateBox, true);
+        window.removeEventListener('resize', updateBox);
+        window.removeEventListener('keydown', key, true);
+        host?.remove();
+        delete window.__rpaElementPicker;
+      };
+
+      host = document.createElement('div');
+      host.style.cssText = 'position:fixed;inset:0;width:100vw;height:100vh;z-index:2147483647;pointer-events:none;';
+      root = host.attachShadow({ mode: 'closed' });
+      box = document.createElement('div');
+      badge = document.createElement('div');
+      box.style.cssText = 'position:fixed;border:2px solid #2563eb;background:rgba(37,99,235,.16);border-radius:3px;pointer-events:none;';
+      badge.style.cssText = 'position:fixed;z-index:2147483647;padding:4px 7px;border-radius:6px;background:#2563eb;color:#fff;font:600 12px/16px system-ui,sans-serif;pointer-events:none;white-space:nowrap;';
+      root.append(box, badge);
+      document.documentElement.append(host);
+      window.addEventListener('mousemove', move, true);
+      window.addEventListener('mousedown', click, true);
+      window.addEventListener('click', click, true);
+      window.addEventListener('scroll', updateBox, true);
+      window.addEventListener('resize', updateBox);
+      window.addEventListener('keydown', key, true);
+      window.__rpaElementPicker = { cancel: () => { cleanup(); reject(new Error('已取消元素拾取')); } };
+      timer = setTimeout(() => {
+        cleanup();
+        reject(new Error('拾取超时，请重新点击拾取'));
+      }, timeout);
+    }))()`);
+    return result;
+  }
+
   async close(): Promise<void> {
     if (!this.win.isDestroyed()) this.win.close();
     globalThis.__rpaBrowserWindow = null;
@@ -726,6 +876,10 @@ export async function browserSession(): Promise<BrowserSession> {
   return BrowserSession.create();
 }
 
+export function hasBrowserSession(): boolean {
+  return Boolean(globalThis.__rpaBrowserWindow && !globalThis.__rpaBrowserWindow.isDestroyed());
+}
+
 export function assertAllowedUrl(url: string): URL {
   return assertUrl(url);
 }

src/main/index.ts

@@ -1,6 +1,7 @@
 import { app, BrowserWindow, dialog, ipcMain, shell } from "electron";
 import path from "node:path";
 import type { AppInfo, FlowDefinition } from "../shared/types";
+import { BrowserSession, hasBrowserSession } from "./browser/session";
 import "./capabilities";
 import { catalogPath, loadCatalog } from "./catalog";
 import { createDemoFlow, createExampleFlows, createFlow, deleteFlow, getFlow, listFlows, saveFlow } from "./flows";
@@ -92,6 +93,18 @@ function registerIpc(): void {
   ipcMain.handle("run:start", (event, flow: FlowDefinition) => startRun(flow, event.sender));
   ipcMain.handle("run:stop", (_event, runId?: string) => stopRun(runId));
   ipcMain.handle("run:capabilities", () => ({ count: capabilityCount(), ids: capabilityIds() }));
+  ipcMain.handle("browser:pick-element", async (_event, options: { url?: string } = {}) => {
+    const url = options.url?.trim();
+    if (!hasBrowserSession() && !url) {
+      throw new Error("请先配置或运行「打开网页」，再拾取元素");
+    }
+    const session = await BrowserSession.create();
+    const currentUrl = await session.currentUrl();
+    if (url && (currentUrl === "about:blank" || currentUrl !== url)) {
+      await session.navigate(url);
+    }
+    return session.pickElement();
+  });
   ipcMain.handle("flows:list", () => listFlows());
   ipcMain.handle("flows:create", (_event, name?: string) => {
     const flow = createFlow(name);

src/preload/index.ts

@@ -23,6 +23,10 @@ const api = {
     save: (options?: { title?: string; defaultPath?: string; filters?: Array<{ name: string; extensions: string[] }> }): Promise<string | null> =>
       ipcRenderer.invoke("dialog:save", options || {})
   },
+  browser: {
+    pickElement: (options?: { url?: string }): Promise<import("../shared/types").ElementPickResult> =>
+      ipcRenderer.invoke("browser:pick-element", options || {})
+  },
   run: {
     start: (flow: FlowDefinition): Promise<{ runId: string }> => ipcRenderer.invoke("run:start", flow),
     stop: (runId?: string): Promise<boolean> => ipcRenderer.invoke("run:stop", runId),

src/renderer/src/panels/Properties.tsx

@@ -401,6 +401,7 @@ function FieldControl({
   onToast: (message: string, kind?: "ok" | "error") => void;
 }) {
   const raw = node.values[field.key] ?? field.defaultValue;
+  const [picking, setPicking] = useState(false);
 
   const pickPath = async (directories: boolean) => {
     try {
@@ -420,6 +421,27 @@ function FieldControl({
     }
   };
 
+  const pickElement = async () => {
+    setPicking(true);
+    try {
+      const currentFlow = useEditorStore.getState().flow;
+      const nodeIndex = currentFlow?.nodes.findIndex((item) => item.id === node.id) ?? -1;
+      const openNode = currentFlow && nodeIndex > 0
+        ? [...currentFlow.nodes.slice(0, nodeIndex)].reverse().find((item) => {
+            const url = textOf(item.values.url).trim();
+            return Boolean(url && !url.startsWith("${"));
+          })
+        : undefined;
+      const picked = await window.rpa.browser.pickElement({ url: textOf(openNode?.values.url) || undefined });
+      onChange(field.key, picked.selector);
+      onToast(`已拾取 ${picked.tag}${picked.label ? ` · ${picked.label}` : ""}`);
+    } catch (error) {
+      onToast(error instanceof Error ? error.message : String(error), "error");
+    } finally {
+      setPicking(false);
+    }
+  };
+
   if (field.widget === "boolean") {
     return (
       <label className="checkbox-row">
@@ -488,8 +510,8 @@ function FieldControl({
             placeholder="css:#submit / aria:button"
             onChange={(event) => onChange(field.key, event.target.value)}
           />
-          <button type="button" className="mini-btn" disabled title="Day 2 接入浏览器拾取器">
+          <button type="button" className="mini-btn" disabled={picking} onClick={() => void pickElement()} title="从运行浏览器拾取元素">
-            拾取
+            {picking ? "拾取中" : "拾取"}
           </button>
         </div>
       ) : field.widget === "secret" ? (

src/shared/types.ts

@@ -169,6 +169,20 @@ export interface AppInfo {
   fixtureUrl?: string;
 }
 
+export interface ElementPickResult {
+  selector: string;
+  tag: string;
+  id?: string;
+  className?: string;
+  name?: string;
+  type?: string;
+  placeholder?: string;
+  label?: string;
+  text?: string;
+  value?: string;
+  url: string;
+}
+
 export interface RunNodeState {
   nodeId: string;
   status: "idle" | "running" | "succeeded" | "failed" | "skipped" | "cancelled";