feat: 接入浏览器元素拾取
This commit is contained in:
yu
parent
9cb4eeab96
commit
7d96badb1f
@ -1,6 +1,7 @@
|
||||
import { BrowserWindow } from "electron";
|
||||
import { promises as fs } from "node:fs";
|
||||
import path from "node:path";
|
||||
import type { ElementPickResult } from "../../shared/types";
|
||||
import {
|
||||
RuntimeError,
|
||||
type BrowserElementInfo,
|
||||
@ -711,6 +712,155 @@ export class BrowserSession implements BrowserLike {
|
||||
return this.win.webContents.getURL();
|
||||
}
|
||||
|
||||
async pickElement(timeoutMs = 90000): Promise<import("../../shared/types").ElementPickResult> {
|
||||
const url = this.win.webContents.getURL();
|
||||
if (!url || url === "about:blank") {
|
||||
throw new RuntimeError("no_page", "浏览器还没有打开网页", { recommendation: "先运行或配置「打开网页」，再拾取元素" });
|
||||
}
|
||||
this.win.show();
|
||||
this.win.focus();
|
||||
const result = await this.evaluate<ElementPickResult>(`(() => new Promise((resolve, reject) => {
|
||||
const timeout = ${JSON.stringify(timeoutMs)};
|
||||
if (window.__rpaElementPicker?.cancel) window.__rpaElementPicker.cancel();
|
||||
|
||||
let host = null;
|
||||
let root = null;
|
||||
let box = null;
|
||||
let badge = null;
|
||||
let target = null;
|
||||
let timer = 0;
|
||||
let finished = false;
|
||||
|
||||
const cssEscape = (value) => {
|
||||
if (globalThis.CSS && typeof CSS.escape === 'function') return CSS.escape(String(value));
|
||||
return String(value).replace(/[^a-zA-Z0-9_-]/g, (char) => '\\\\' + char);
|
||||
};
|
||||
const isUnique = (candidate) => {
|
||||
try { return document.querySelectorAll(candidate).length === 1; } catch { return false; }
|
||||
};
|
||||
const cssPath = (node) => {
|
||||
if (node.id) {
|
||||
const candidate = '#' + cssEscape(node.id);
|
||||
if (isUnique(candidate)) return candidate;
|
||||
}
|
||||
const testId = node.getAttribute('data-testid');
|
||||
if (testId) {
|
||||
const candidate = '[data-testid="' + cssEscape(testId) + '"]';
|
||||
if (isUnique(candidate)) return candidate;
|
||||
}
|
||||
const parts = [];
|
||||
let current = node;
|
||||
while (current && current.nodeType === 1) {
|
||||
let part = current.tagName.toLowerCase();
|
||||
if (current.id) {
|
||||
part += '#' + cssEscape(current.id);
|
||||
parts.unshift(part);
|
||||
break;
|
||||
}
|
||||
const parent = current.parentElement;
|
||||
if (parent) {
|
||||
const siblings = [...parent.children].filter((item) => item.tagName === current.tagName);
|
||||
if (siblings.length > 1) part += ':nth-of-type(' + (siblings.indexOf(current) + 1) + ')';
|
||||
}
|
||||
parts.unshift(part);
|
||||
current = parent;
|
||||
}
|
||||
return parts.join(' > ');
|
||||
};
|
||||
const describe = (node) => {
|
||||
const rect = node.getBoundingClientRect();
|
||||
return {
|
||||
selector: 'css:' + cssPath(node),
|
||||
tag: node.tagName.toLowerCase(),
|
||||
id: node.id || undefined,
|
||||
className: node.getAttribute('class') || undefined,
|
||||
name: node.getAttribute('name') || undefined,
|
||||
type: node.getAttribute('type') || undefined,
|
||||
placeholder: node.getAttribute('placeholder') || undefined,
|
||||
label: node.getAttribute('aria-label') || node.getAttribute('placeholder') || node.getAttribute('name') || (node.innerText || '').trim().slice(0, 100) || undefined,
|
||||
text: (node.innerText || node.textContent || '').trim().slice(0, 200) || undefined,
|
||||
value: String(node.value ?? '') || undefined,
|
||||
url: location.href
|
||||
};
|
||||
};
|
||||
const updateBox = () => {
|
||||
if (!target || !box || !badge) return;
|
||||
const rect = target.getBoundingClientRect();
|
||||
box.style.left = Math.max(0, rect.left - 3) + 'px';
|
||||
box.style.top = Math.max(0, rect.top - 3) + 'px';
|
||||
box.style.width = Math.max(4, rect.width + 6) + 'px';
|
||||
box.style.height = Math.max(4, rect.height + 6) + 'px';
|
||||
const tag = target.tagName.toLowerCase();
|
||||
const name = target.getAttribute('aria-label') || target.getAttribute('placeholder') || target.getAttribute('name') || target.id || '';
|
||||
badge.textContent = '拾取：' + tag + (name ? ' · ' + name.slice(0, 60) : '');
|
||||
badge.style.left = Math.max(0, rect.left) + 'px';
|
||||
badge.style.top = Math.max(0, rect.top - 30) + 'px';
|
||||
};
|
||||
const firstElement = (event) => {
|
||||
for (const item of event.composedPath()) if (item instanceof Element) return item;
|
||||
return event.target instanceof Element ? event.target : null;
|
||||
};
|
||||
const move = (event) => {
|
||||
if (finished) return;
|
||||
target = firstElement(event);
|
||||
updateBox();
|
||||
};
|
||||
const click = (event) => {
|
||||
if (finished) return;
|
||||
const selected = firstElement(event);
|
||||
if (!selected) return;
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
event.stopImmediatePropagation();
|
||||
cleanup();
|
||||
resolve(describe(selected));
|
||||
};
|
||||
const key = (event) => {
|
||||
if (event.key === 'Escape') {
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
cleanup();
|
||||
reject(new Error('已取消元素拾取'));
|
||||
}
|
||||
};
|
||||
const cleanup = () => {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
clearTimeout(timer);
|
||||
window.removeEventListener('mousemove', move, true);
|
||||
window.removeEventListener('mousedown', click, true);
|
||||
window.removeEventListener('click', click, true);
|
||||
window.removeEventListener('scroll', updateBox, true);
|
||||
window.removeEventListener('resize', updateBox);
|
||||
window.removeEventListener('keydown', key, true);
|
||||
host?.remove();
|
||||
delete window.__rpaElementPicker;
|
||||
};
|
||||
|
||||
host = document.createElement('div');
|
||||
host.style.cssText = 'position:fixed;inset:0;width:100vw;height:100vh;z-index:2147483647;pointer-events:none;';
|
||||
root = host.attachShadow({ mode: 'closed' });
|
||||
box = document.createElement('div');
|
||||
badge = document.createElement('div');
|
||||
box.style.cssText = 'position:fixed;border:2px solid #2563eb;background:rgba(37,99,235,.16);border-radius:3px;pointer-events:none;';
|
||||
badge.style.cssText = 'position:fixed;z-index:2147483647;padding:4px 7px;border-radius:6px;background:#2563eb;color:#fff;font:600 12px/16px system-ui,sans-serif;pointer-events:none;white-space:nowrap;';
|
||||
root.append(box, badge);
|
||||
document.documentElement.append(host);
|
||||
window.addEventListener('mousemove', move, true);
|
||||
window.addEventListener('mousedown', click, true);
|
||||
window.addEventListener('click', click, true);
|
||||
window.addEventListener('scroll', updateBox, true);
|
||||
window.addEventListener('resize', updateBox);
|
||||
window.addEventListener('keydown', key, true);
|
||||
window.__rpaElementPicker = { cancel: () => { cleanup(); reject(new Error('已取消元素拾取')); } };
|
||||
timer = setTimeout(() => {
|
||||
cleanup();
|
||||
reject(new Error('拾取超时，请重新点击拾取'));
|
||||
}, timeout);
|
||||
}))()`);
|
||||
return result;
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
if (!this.win.isDestroyed()) this.win.close();
|
||||
globalThis.__rpaBrowserWindow = null;
|
||||
@ -726,6 +876,10 @@ export async function browserSession(): Promise<BrowserSession> {
|
||||
return BrowserSession.create();
|
||||
}
|
||||
|
||||
export function hasBrowserSession(): boolean {
|
||||
return Boolean(globalThis.__rpaBrowserWindow && !globalThis.__rpaBrowserWindow.isDestroyed());
|
||||
}
|
||||
|
||||
export function assertAllowedUrl(url: string): URL {
|
||||
return assertUrl(url);
|
||||
}
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import { app, BrowserWindow, dialog, ipcMain, shell } from "electron";
|
||||
import path from "node:path";
|
||||
import type { AppInfo, FlowDefinition } from "../shared/types";
|
||||
import { BrowserSession, hasBrowserSession } from "./browser/session";
|
||||
import "./capabilities";
|
||||
import { catalogPath, loadCatalog } from "./catalog";
|
||||
import { createDemoFlow, createExampleFlows, createFlow, deleteFlow, getFlow, listFlows, saveFlow } from "./flows";
|
||||
@ -92,6 +93,18 @@ function registerIpc(): void {
|
||||
ipcMain.handle("run:start", (event, flow: FlowDefinition) => startRun(flow, event.sender));
|
||||
ipcMain.handle("run:stop", (_event, runId?: string) => stopRun(runId));
|
||||
ipcMain.handle("run:capabilities", () => ({ count: capabilityCount(), ids: capabilityIds() }));
|
||||
ipcMain.handle("browser:pick-element", async (_event, options: { url?: string } = {}) => {
|
||||
const url = options.url?.trim();
|
||||
if (!hasBrowserSession() && !url) {
|
||||
throw new Error("请先配置或运行「打开网页」，再拾取元素");
|
||||
}
|
||||
const session = await BrowserSession.create();
|
||||
const currentUrl = await session.currentUrl();
|
||||
if (url && (currentUrl === "about:blank" || currentUrl !== url)) {
|
||||
await session.navigate(url);
|
||||
}
|
||||
return session.pickElement();
|
||||
});
|
||||
ipcMain.handle("flows:list", () => listFlows());
|
||||
ipcMain.handle("flows:create", (_event, name?: string) => {
|
||||
const flow = createFlow(name);
|
||||
|
||||
@ -23,6 +23,10 @@ const api = {
|
||||
save: (options?: { title?: string; defaultPath?: string; filters?: Array<{ name: string; extensions: string[] }> }): Promise<string | null> =>
|
||||
ipcRenderer.invoke("dialog:save", options || {})
|
||||
},
|
||||
browser: {
|
||||
pickElement: (options?: { url?: string }): Promise<import("../shared/types").ElementPickResult> =>
|
||||
ipcRenderer.invoke("browser:pick-element", options || {})
|
||||
},
|
||||
run: {
|
||||
start: (flow: FlowDefinition): Promise<{ runId: string }> => ipcRenderer.invoke("run:start", flow),
|
||||
stop: (runId?: string): Promise<boolean> => ipcRenderer.invoke("run:stop", runId),
|
||||
|
||||
@ -401,6 +401,7 @@ function FieldControl({
|
||||
onToast: (message: string, kind?: "ok" | "error") => void;
|
||||
}) {
|
||||
const raw = node.values[field.key] ?? field.defaultValue;
|
||||
const [picking, setPicking] = useState(false);
|
||||
|
||||
const pickPath = async (directories: boolean) => {
|
||||
try {
|
||||
@ -420,6 +421,27 @@ function FieldControl({
|
||||
}
|
||||
};
|
||||
|
||||
const pickElement = async () => {
|
||||
setPicking(true);
|
||||
try {
|
||||
const currentFlow = useEditorStore.getState().flow;
|
||||
const nodeIndex = currentFlow?.nodes.findIndex((item) => item.id === node.id) ?? -1;
|
||||
const openNode = currentFlow && nodeIndex > 0
|
||||
? [...currentFlow.nodes.slice(0, nodeIndex)].reverse().find((item) => {
|
||||
const url = textOf(item.values.url).trim();
|
||||
return Boolean(url && !url.startsWith("${"));
|
||||
})
|
||||
: undefined;
|
||||
const picked = await window.rpa.browser.pickElement({ url: textOf(openNode?.values.url) || undefined });
|
||||
onChange(field.key, picked.selector);
|
||||
onToast(`已拾取 ${picked.tag}${picked.label ? ` · ${picked.label}` : ""}`);
|
||||
} catch (error) {
|
||||
onToast(error instanceof Error ? error.message : String(error), "error");
|
||||
} finally {
|
||||
setPicking(false);
|
||||
}
|
||||
};
|
||||
|
||||
if (field.widget === "boolean") {
|
||||
return (
|
||||
<label className="checkbox-row">
|
||||
@ -488,8 +510,8 @@ function FieldControl({
|
||||
placeholder="css:#submit / aria:button"
|
||||
onChange={(event) => onChange(field.key, event.target.value)}
|
||||
/>
|
||||
<button type="button" className="mini-btn" disabled title="Day 2 接入浏览器拾取器">
|
||||
拾取
|
||||
<button type="button" className="mini-btn" disabled={picking} onClick={() => void pickElement()} title="从运行浏览器拾取元素">
|
||||
{picking ? "拾取中" : "拾取"}
|
||||
</button>
|
||||
</div>
|
||||
) : field.widget === "secret" ? (
|
||||
|
||||
@ -169,6 +169,20 @@ export interface AppInfo {
|
||||
fixtureUrl?: string;
|
||||
}
|
||||
|
||||
export interface ElementPickResult {
|
||||
selector: string;
|
||||
tag: string;
|
||||
id?: string;
|
||||
className?: string;
|
||||
name?: string;
|
||||
type?: string;
|
||||
placeholder?: string;
|
||||
label?: string;
|
||||
text?: string;
|
||||
value?: string;
|
||||
url: string;
|
||||
}
|
||||
|
||||
export interface RunNodeState {
|
||||
nodeId: string;
|
||||
status: "idle" | "running" | "succeeded" | "failed" | "skipped" | "cancelled";
|
||||
|
||||
Loading…
Reference in New Issue
Block a user