diff --git a/src/main/browser/session.ts b/src/main/browser/session.ts index 0e58595..a5c4988 100644 --- a/src/main/browser/session.ts +++ b/src/main/browser/session.ts @@ -1,6 +1,7 @@ import { BrowserWindow } from "electron"; import { promises as fs } from "node:fs"; import path from "node:path"; +import type { ElementPickResult } from "../../shared/types"; import { RuntimeError, type BrowserElementInfo, @@ -711,6 +712,155 @@ export class BrowserSession implements BrowserLike { return this.win.webContents.getURL(); } + async pickElement(timeoutMs = 90000): Promise { + const url = this.win.webContents.getURL(); + if (!url || url === "about:blank") { + throw new RuntimeError("no_page", "浏览器还没有打开网页", { recommendation: "先运行或配置「打开网页」，再拾取元素" }); + } + this.win.show(); + this.win.focus(); + const result = await this.evaluate(`(() => new Promise((resolve, reject) => { + const timeout = ${JSON.stringify(timeoutMs)}; + if (window.__rpaElementPicker?.cancel) window.__rpaElementPicker.cancel(); + + let host = null; + let root = null; + let box = null; + let badge = null; + let target = null; + let timer = 0; + let finished = false; + + const cssEscape = (value) => { + if (globalThis.CSS && typeof CSS.escape === 'function') return CSS.escape(String(value)); + return String(value).replace(/[^a-zA-Z0-9_-]/g, (char) => '\\\\' + char); + }; + const isUnique = (candidate) => { + try { return document.querySelectorAll(candidate).length === 1; } catch { return false; } + }; + const cssPath = (node) => { + if (node.id) { + const candidate = '#' + cssEscape(node.id); + if (isUnique(candidate)) return candidate; + } + const testId = node.getAttribute('data-testid'); + if (testId) { + const candidate = '[data-testid="' + cssEscape(testId) + '"]'; + if (isUnique(candidate)) return candidate; + } + const parts = []; + let current = node; + while (current && current.nodeType === 1) { + let part = current.tagName.toLowerCase(); + if (current.id) { + part += '#' + cssEscape(current.id); + parts.unshift(part); + break; + } + const parent = current.parentElement; + if (parent) { + const siblings = [...parent.children].filter((item) => item.tagName === current.tagName); + if (siblings.length > 1) part += ':nth-of-type(' + (siblings.indexOf(current) + 1) + ')'; + } + parts.unshift(part); + current = parent; + } + return parts.join(' > '); + }; + const describe = (node) => { + const rect = node.getBoundingClientRect(); + return { + selector: 'css:' + cssPath(node), + tag: node.tagName.toLowerCase(), + id: node.id || undefined, + className: node.getAttribute('class') || undefined, + name: node.getAttribute('name') || undefined, + type: node.getAttribute('type') || undefined, + placeholder: node.getAttribute('placeholder') || undefined, + label: node.getAttribute('aria-label') || node.getAttribute('placeholder') || node.getAttribute('name') || (node.innerText || '').trim().slice(0, 100) || undefined, + text: (node.innerText || node.textContent || '').trim().slice(0, 200) || undefined, + value: String(node.value ?? '') || undefined, + url: location.href + }; + }; + const updateBox = () => { + if (!target || !box || !badge) return; + const rect = target.getBoundingClientRect(); + box.style.left = Math.max(0, rect.left - 3) + 'px'; + box.style.top = Math.max(0, rect.top - 3) + 'px'; + box.style.width = Math.max(4, rect.width + 6) + 'px'; + box.style.height = Math.max(4, rect.height + 6) + 'px'; + const tag = target.tagName.toLowerCase(); + const name = target.getAttribute('aria-label') || target.getAttribute('placeholder') || target.getAttribute('name') || target.id || ''; + badge.textContent = '拾取：' + tag + (name ? ' · ' + name.slice(0, 60) : ''); + badge.style.left = Math.max(0, rect.left) + 'px'; + badge.style.top = Math.max(0, rect.top - 30) + 'px'; + }; + const firstElement = (event) => { + for (const item of event.composedPath()) if (item instanceof Element) return item; + return event.target instanceof Element ? event.target : null; + }; + const move = (event) => { + if (finished) return; + target = firstElement(event); + updateBox(); + }; + const click = (event) => { + if (finished) return; + const selected = firstElement(event); + if (!selected) return; + event.preventDefault(); + event.stopPropagation(); + event.stopImmediatePropagation(); + cleanup(); + resolve(describe(selected)); + }; + const key = (event) => { + if (event.key === 'Escape') { + event.preventDefault(); + event.stopPropagation(); + cleanup(); + reject(new Error('已取消元素拾取')); + } + }; + const cleanup = () => { + if (finished) return; + finished = true; + clearTimeout(timer); + window.removeEventListener('mousemove', move, true); + window.removeEventListener('mousedown', click, true); + window.removeEventListener('click', click, true); + window.removeEventListener('scroll', updateBox, true); + window.removeEventListener('resize', updateBox); + window.removeEventListener('keydown', key, true); + host?.remove(); + delete window.__rpaElementPicker; + }; + + host = document.createElement('div'); + host.style.cssText = 'position:fixed;inset:0;width:100vw;height:100vh;z-index:2147483647;pointer-events:none;'; + root = host.attachShadow({ mode: 'closed' }); + box = document.createElement('div'); + badge = document.createElement('div'); + box.style.cssText = 'position:fixed;border:2px solid #2563eb;background:rgba(37,99,235,.16);border-radius:3px;pointer-events:none;'; + badge.style.cssText = 'position:fixed;z-index:2147483647;padding:4px 7px;border-radius:6px;background:#2563eb;color:#fff;font:600 12px/16px system-ui,sans-serif;pointer-events:none;white-space:nowrap;'; + root.append(box, badge); + document.documentElement.append(host); + window.addEventListener('mousemove', move, true); + window.addEventListener('mousedown', click, true); + window.addEventListener('click', click, true); + window.addEventListener('scroll', updateBox, true); + window.addEventListener('resize', updateBox); + window.addEventListener('keydown', key, true); + window.__rpaElementPicker = { cancel: () => { cleanup(); reject(new Error('已取消元素拾取')); } }; + timer = setTimeout(() => { + cleanup(); + reject(new Error('拾取超时，请重新点击拾取')); + }, timeout); + }))()`); + return result; + } + async close(): Promise { if (!this.win.isDestroyed()) this.win.close(); globalThis.__rpaBrowserWindow = null; @@ -726,6 +876,10 @@ export async function browserSession(): Promise { return BrowserSession.create(); } +export function hasBrowserSession(): boolean { + return Boolean(globalThis.__rpaBrowserWindow && !globalThis.__rpaBrowserWindow.isDestroyed()); +} + export function assertAllowedUrl(url: string): URL { return assertUrl(url); } diff --git a/src/main/index.ts b/src/main/index.ts index c033775..fad8c50 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -1,6 +1,7 @@ import { app, BrowserWindow, dialog, ipcMain, shell } from "electron"; import path from "node:path"; import type { AppInfo, FlowDefinition } from "../shared/types"; +import { BrowserSession, hasBrowserSession } from "./browser/session"; import "./capabilities"; import { catalogPath, loadCatalog } from "./catalog"; import { createDemoFlow, createExampleFlows, createFlow, deleteFlow, getFlow, listFlows, saveFlow } from "./flows"; @@ -92,6 +93,18 @@ function registerIpc(): void { ipcMain.handle("run:start", (event, flow: FlowDefinition) => startRun(flow, event.sender)); ipcMain.handle("run:stop", (_event, runId?: string) => stopRun(runId)); ipcMain.handle("run:capabilities", () => ({ count: capabilityCount(), ids: capabilityIds() })); + ipcMain.handle("browser:pick-element", async (_event, options: { url?: string } = {}) => { + const url = options.url?.trim(); + if (!hasBrowserSession() && !url) { + throw new Error("请先配置或运行「打开网页」，再拾取元素"); + } + const session = await BrowserSession.create(); + const currentUrl = await session.currentUrl(); + if (url && (currentUrl === "about:blank" || currentUrl !== url)) { + await session.navigate(url); + } + return session.pickElement(); + }); ipcMain.handle("flows:list", () => listFlows()); ipcMain.handle("flows:create", (_event, name?: string) => { const flow = createFlow(name); diff --git a/src/preload/index.ts b/src/preload/index.ts index 648452c..4f2af19 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -23,6 +23,10 @@ const api = { save: (options?: { title?: string; defaultPath?: string; filters?: Array<{ name: string; extensions: string[] }> }): Promise => ipcRenderer.invoke("dialog:save", options || {}) }, + browser: { + pickElement: (options?: { url?: string }): Promise => + ipcRenderer.invoke("browser:pick-element", options || {}) + }, run: { start: (flow: FlowDefinition): Promise<{ runId: string }> => ipcRenderer.invoke("run:start", flow), stop: (runId?: string): Promise => ipcRenderer.invoke("run:stop", runId), diff --git a/src/renderer/src/panels/Properties.tsx b/src/renderer/src/panels/Properties.tsx index fd4f057..0542a4a 100644 --- a/src/renderer/src/panels/Properties.tsx +++ b/src/renderer/src/panels/Properties.tsx @@ -401,6 +401,7 @@ function FieldControl({ onToast: (message: string, kind?: "ok" | "error") => void; }) { const raw = node.values[field.key] ?? field.defaultValue; + const [picking, setPicking] = useState(false); const pickPath = async (directories: boolean) => { try { @@ -420,6 +421,27 @@ function FieldControl({ } }; + const pickElement = async () => { + setPicking(true); + try { + const currentFlow = useEditorStore.getState().flow; + const nodeIndex = currentFlow?.nodes.findIndex((item) => item.id === node.id) ?? -1; + const openNode = currentFlow && nodeIndex > 0 + ? [...currentFlow.nodes.slice(0, nodeIndex)].reverse().find((item) => { + const url = textOf(item.values.url).trim(); + return Boolean(url && !url.startsWith("${")); + }) + : undefined; + const picked = await window.rpa.browser.pickElement({ url: textOf(openNode?.values.url) || undefined }); + onChange(field.key, picked.selector); + onToast(`已拾取 ${picked.tag}${picked.label ? ` · ${picked.label}` : ""}`); + } catch (error) { + onToast(error instanceof Error ? error.message : String(error), "error"); + } finally { + setPicking(false); + } + }; + if (field.widget === "boolean") { return (