feat: 接入浏览器元素拾取

src/main/browser/session.ts

@@ -1,6 +1,7 @@
 import { BrowserWindow } from "electron";
 import { promises as fs } from "node:fs";
 import path from "node:path";
+import type { ElementPickResult } from "../../shared/types";
 import {
   RuntimeError,
   type BrowserElementInfo,
@@ -711,6 +712,155 @@ export class BrowserSession implements BrowserLike {
     return this.win.webContents.getURL();
   }
 
+  async pickElement(timeoutMs = 90000): Promise<import("../../shared/types").ElementPickResult> {
+    const url = this.win.webContents.getURL();
+    if (!url || url === "about:blank") {
+      throw new RuntimeError("no_page", "浏览器还没有打开网页", { recommendation: "先运行或配置「打开网页」，再拾取元素" });
+    }
+    this.win.show();
+    this.win.focus();
+    const result = await this.evaluate<ElementPickResult>(`(() => new Promise((resolve, reject) => {
+      const timeout = ${JSON.stringify(timeoutMs)};
+      if (window.__rpaElementPicker?.cancel) window.__rpaElementPicker.cancel();
+
+      let host = null;
+      let root = null;
+      let box = null;
+      let badge = null;
+      let target = null;
+      let timer = 0;
+      let finished = false;
+
+      const cssEscape = (value) => {
+        if (globalThis.CSS && typeof CSS.escape === 'function') return CSS.escape(String(value));
+        return String(value).replace(/[^a-zA-Z0-9_-]/g, (char) => '\\\\' + char);
+      };
+      const isUnique = (candidate) => {
+        try { return document.querySelectorAll(candidate).length === 1; } catch { return false; }
+      };
+      const cssPath = (node) => {
+        if (node.id) {
+          const candidate = '#' + cssEscape(node.id);
+          if (isUnique(candidate)) return candidate;
+        }
+        const testId = node.getAttribute('data-testid');
+        if (testId) {
+          const candidate = '[data-testid="' + cssEscape(testId) + '"]';
+          if (isUnique(candidate)) return candidate;
+        }
+        const parts = [];
+        let current = node;
+        while (current && current.nodeType === 1) {
+          let part = current.tagName.toLowerCase();
+          if (current.id) {
+            part += '#' + cssEscape(current.id);
+            parts.unshift(part);
+            break;
+          }
+          const parent = current.parentElement;
+          if (parent) {
+            const siblings = [...parent.children].filter((item) => item.tagName === current.tagName);
+            if (siblings.length > 1) part += ':nth-of-type(' + (siblings.indexOf(current) + 1) + ')';
+          }
+          parts.unshift(part);
+          current = parent;
+        }
+        return parts.join(' > ');
+      };
+      const describe = (node) => {
+        const rect = node.getBoundingClientRect();
+        return {
+          selector: 'css:' + cssPath(node),
+          tag: node.tagName.toLowerCase(),
+          id: node.id || undefined,
+          className: node.getAttribute('class') || undefined,
+          name: node.getAttribute('name') || undefined,
+          type: node.getAttribute('type') || undefined,
+          placeholder: node.getAttribute('placeholder') || undefined,
+          label: node.getAttribute('aria-label') || node.getAttribute('placeholder') || node.getAttribute('name') || (node.innerText || '').trim().slice(0, 100) || undefined,
+          text: (node.innerText || node.textContent || '').trim().slice(0, 200) || undefined,
+          value: String(node.value ?? '') || undefined,
+          url: location.href
+        };
+      };
+      const updateBox = () => {
+        if (!target || !box || !badge) return;
+        const rect = target.getBoundingClientRect();
+        box.style.left = Math.max(0, rect.left - 3) + 'px';
+        box.style.top = Math.max(0, rect.top - 3) + 'px';
+        box.style.width = Math.max(4, rect.width + 6) + 'px';
+        box.style.height = Math.max(4, rect.height + 6) + 'px';
+        const tag = target.tagName.toLowerCase();
+        const name = target.getAttribute('aria-label') || target.getAttribute('placeholder') || target.getAttribute('name') || target.id || '';
+        badge.textContent = '拾取：' + tag + (name ? ' · ' + name.slice(0, 60) : '');
+        badge.style.left = Math.max(0, rect.left) + 'px';
+        badge.style.top = Math.max(0, rect.top - 30) + 'px';
+      };
+      const firstElement = (event) => {
+        for (const item of event.composedPath()) if (item instanceof Element) return item;
+        return event.target instanceof Element ? event.target : null;
+      };
+      const move = (event) => {
+        if (finished) return;
+        target = firstElement(event);
+        updateBox();
+      };
+      const click = (event) => {
+        if (finished) return;
+        const selected = firstElement(event);
+        if (!selected) return;
+        event.preventDefault();
+        event.stopPropagation();
+        event.stopImmediatePropagation();
+        cleanup();
+        resolve(describe(selected));
+      };
+      const key = (event) => {
+        if (event.key === 'Escape') {
+          event.preventDefault();
+          event.stopPropagation();
+          cleanup();
+          reject(new Error('已取消元素拾取'));
+        }
+      };
+      const cleanup = () => {
+        if (finished) return;
+        finished = true;
+        clearTimeout(timer);
+        window.removeEventListener('mousemove', move, true);
+        window.removeEventListener('mousedown', click, true);
+        window.removeEventListener('click', click, true);
+        window.removeEventListener('scroll', updateBox, true);
+        window.removeEventListener('resize', updateBox);
+        window.removeEventListener('keydown', key, true);
+        host?.remove();
+        delete window.__rpaElementPicker;
+      };
+
+      host = document.createElement('div');
+      host.style.cssText = 'position:fixed;inset:0;width:100vw;height:100vh;z-index:2147483647;pointer-events:none;';
+      root = host.attachShadow({ mode: 'closed' });
+      box = document.createElement('div');
+      badge = document.createElement('div');
+      box.style.cssText = 'position:fixed;border:2px solid #2563eb;background:rgba(37,99,235,.16);border-radius:3px;pointer-events:none;';
+      badge.style.cssText = 'position:fixed;z-index:2147483647;padding:4px 7px;border-radius:6px;background:#2563eb;color:#fff;font:600 12px/16px system-ui,sans-serif;pointer-events:none;white-space:nowrap;';
+      root.append(box, badge);
+      document.documentElement.append(host);
+      window.addEventListener('mousemove', move, true);
+      window.addEventListener('mousedown', click, true);
+      window.addEventListener('click', click, true);
+      window.addEventListener('scroll', updateBox, true);
+      window.addEventListener('resize', updateBox);
+      window.addEventListener('keydown', key, true);
+      window.__rpaElementPicker = { cancel: () => { cleanup(); reject(new Error('已取消元素拾取')); } };
+      timer = setTimeout(() => {
+        cleanup();
+        reject(new Error('拾取超时，请重新点击拾取'));
+      }, timeout);
+    }))()`);
+    return result;
+  }
+
   async close(): Promise<void> {
     if (!this.win.isDestroyed()) this.win.close();
     globalThis.__rpaBrowserWindow = null;
@@ -726,6 +876,10 @@ export async function browserSession(): Promise<BrowserSession> {
   return BrowserSession.create();
 }
 
+export function hasBrowserSession(): boolean {
+  return Boolean(globalThis.__rpaBrowserWindow && !globalThis.__rpaBrowserWindow.isDestroyed());
+}
+
 export function assertAllowedUrl(url: string): URL {
   return assertUrl(url);
 }

src/main/index.ts

@@ -1,6 +1,7 @@
 import { app, BrowserWindow, dialog, ipcMain, shell } from "electron";
 import path from "node:path";
 import type { AppInfo, FlowDefinition } from "../shared/types";
+import { BrowserSession, hasBrowserSession } from "./browser/session";
 import "./capabilities";
 import { catalogPath, loadCatalog } from "./catalog";
 import { createDemoFlow, createExampleFlows, createFlow, deleteFlow, getFlow, listFlows, saveFlow } from "./flows";
@@ -92,6 +93,18 @@ function registerIpc(): void {
   ipcMain.handle("run:start", (event, flow: FlowDefinition) => startRun(flow, event.sender));
   ipcMain.handle("run:stop", (_event, runId?: string) => stopRun(runId));
   ipcMain.handle("run:capabilities", () => ({ count: capabilityCount(), ids: capabilityIds() }));
+  ipcMain.handle("browser:pick-element", async (_event, options: { url?: string } = {}) => {
+    const url = options.url?.trim();
+    if (!hasBrowserSession() && !url) {
+      throw new Error("请先配置或运行「打开网页」，再拾取元素");
+    }
+    const session = await BrowserSession.create();
+    const currentUrl = await session.currentUrl();
+    if (url && (currentUrl === "about:blank" || currentUrl !== url)) {
+      await session.navigate(url);
+    }
+    return session.pickElement();
+  });
   ipcMain.handle("flows:list", () => listFlows());
   ipcMain.handle("flows:create", (_event, name?: string) => {
     const flow = createFlow(name);