diff --git a/scripts/app-system-tests.mjs b/scripts/app-system-tests.mjs index bfe9939..6ed04f3 100644 --- a/scripts/app-system-tests.mjs +++ b/scripts/app-system-tests.mjs @@ -47,9 +47,13 @@ export async function validator() { export async function verifyBundle(catalog = staticCatalog()) { const { manifest, groups } = readBundle(); - const expected = catalog.components.filter(c => c.group === "app" && c.registered).map(c => c.id).sort(); + const expected = catalog.components.filter(c => c.group === "app" && (c.registered || c.implemented)).map(c => c.id).sort(); const actual = groups.flatMap(g => g.entry.components).sort(); - if (groups.length !== 13 || actual.length !== 50 || new Set(actual).size !== 50 || JSON.stringify(actual) !== JSON.stringify(expected)) throw new Error("流程包与当前 50 项验收清单不一致，请重新生成并核对覆盖范围。"); + if (groups.length !== 13 || actual.length !== 50 || new Set(actual).size !== 50 || JSON.stringify(actual) !== JSON.stringify(expected)) { + const expectedOnly = expected.filter(id => !actual.includes(id)); + const actualOnly = actual.filter(id => !expected.includes(id)); + throw new Error(`流程包与当前 50 项验收清单不一致；expectedOnly=${JSON.stringify(expectedOnly)}；actualOnly=${JSON.stringify(actualOnly)}`); + } const validateFlow = await validator(); const validation = {}; for (const { flow, entry } of groups) { @@ -248,6 +252,9 @@ export async function runSafe(outputDirectory, checkUi = false) { const info = await evaluate(page, "window.rpa.app.info()"); if (path.resolve(info.userData).toLowerCase() !== profile.toLowerCase()) throw new Error("隔离用户目录不匹配，已拒绝执行测试。"); const catalog = await evaluate(page, "window.rpa.catalog.get()"); + const runtime = await evaluate(page, "window.rpa.run.capabilities()"); + const registered = new Set(runtime.ids || []); + for (const component of catalog.components) component.registered = registered.has(component.id) || registered.has(component.functionName); const checked = await verifyBundle(catalog); for (const { flow, entry } of checked.groups) { const issues = checked.validation[entry.id]; diff --git a/src/main/browser/external.ts b/src/main/browser/external.ts index cf892f9..900adb1 100644 --- a/src/main/browser/external.ts +++ b/src/main/browser/external.ts @@ -24,6 +24,8 @@ export interface ExternalLaunchOptions { extraArgs?: string[]; /** 独立用户数据目录；缺省使用系统临时目录下的独立 profile */ userDataDir?: string; + /** 首次启动时直接打开的 URL，避免先出现 about:blank */ + initialUrl?: string; } export interface ExternalRemoteOptions { @@ -225,12 +227,13 @@ export async function launchExternalBrowser(options: ExternalLaunchOptions): Pro await fs.mkdir(userDataDir, { recursive: true }); const extraArgs = (options.extraArgs ?? []).filter(Boolean); + const startUrl = options.initialUrl?.trim() || "about:blank"; const args = [ `--remote-debugging-port=${port}`, `--user-data-dir=${userDataDir}`, "--no-first-run", "--no-default-browser-check", - "about:blank", + startUrl, ...extraArgs ]; diff --git a/src/main/browser/session.ts b/src/main/browser/session.ts index 5c383f1..0e22917 100644 --- a/src/main/browser/session.ts +++ b/src/main/browser/session.ts @@ -166,6 +166,16 @@ export class BrowserSession implements BrowserLike { } /** 当前会话是否为外部浏览器 */ + /** 会话底层 CDP 是否已断开（缓存清理用） */ + get closed(): boolean { + return this.transport.closed; + } + + /** 缓存键，外部会话排序用 */ + get sessionKey(): string { + return this.externalKey; + } + get isExternal(): boolean { return this.external !== null; } @@ -1157,6 +1167,10 @@ export class BrowserSession implements BrowserLike { let box = null; let badge = null; let target = null; + let locked = false; + let chain = []; + let chainIndex = 0; + let selected = null; let timer = 0; let finished = false; @@ -1225,18 +1239,24 @@ export class BrowserSession implements BrowserLike { url: location.href }; }; + const nodeName = (node) => node.getAttribute('aria-label') || node.getAttribute('placeholder') || node.getAttribute('name') || node.id || (node.innerText || '').trim().slice(0, 60) || ''; const updateBox = () => { - if (!target || !box || !badge) return; - const rect = target.getBoundingClientRect(); + const node = locked ? selected : target; + if (!node || !box || !badge) return; + const rect = node.getBoundingClientRect(); box.style.left = Math.max(0, rect.left - 3) + 'px'; box.style.top = Math.max(0, rect.top - 3) + 'px'; box.style.width = Math.max(4, rect.width + 6) + 'px'; box.style.height = Math.max(4, rect.height + 6) + 'px'; - const tag = target.tagName.toLowerCase(); - const name = target.getAttribute('aria-label') || target.getAttribute('placeholder') || target.getAttribute('name') || target.id || ''; - badge.textContent = '拾取：' + tag + (name ? ' · ' + name.slice(0, 60) : ''); + const tag = node.tagName.toLowerCase(); + const name = nodeName(node); + if (locked) { + badge.textContent = '已锁定 ' + (chainIndex + 1) + '/' + chain.length + ' · ' + tag + (name ? ' · ' + name.slice(0, 42) : '') + ' · [PageUp/PageDown]调层级 [空格]确认 [Esc]取消'; + } else { + badge.textContent = '拾取 · ' + tag + (name ? ' · ' + name.slice(0, 42) : '') + ' · [空格]锁定 [Esc]取消'; + } badge.style.left = Math.max(0, rect.left) + 'px'; - badge.style.top = Math.max(0, rect.top - 30) + 'px'; + badge.style.top = Math.max(0, rect.top - 34) + 'px'; }; const interactiveSelector = [ 'button', 'a', 'input', 'select', 'textarea', 'summary', 'label', 'option', @@ -1266,20 +1286,48 @@ export class BrowserSession implements BrowserLike { if (!deepest && event.target instanceof Element) deepest = event.target; return closestInteractive(deepest); }; + const buildChain = (node) => { + const result = []; + let current = node; + while (current && current.nodeType === 1) { + result.unshift(current); + const root = current.getRootNode(); + current = root instanceof ShadowRoot ? root.host : null; + } + return result; + }; + const lock = () => { + if (locked || !target) return; + locked = true; + chain = buildChain(target); + chainIndex = Math.max(0, chain.length - 1); + selected = chain[chainIndex] || target; + updateBox(); + }; + const adjustLevel = (delta) => { + if (!locked || !chain.length) return; + const next = Math.min(chain.length - 1, Math.max(0, chainIndex + delta)); + if (next === chainIndex) return; + chainIndex = next; + selected = chain[chainIndex]; + updateBox(); + }; + const confirm = () => { + if (!locked || !selected) return; + cleanup(); + resolve(describe(selected)); + }; const move = (event) => { - if (finished) return; + if (finished || locked) return; target = firstElement(event); updateBox(); }; const click = (event) => { if (finished) return; - const selected = firstElement(event); - if (!selected) return; + // 与桌面拾取对齐：鼠标只用于定位，空格锁定/确认，避免误触发页面按钮。 event.preventDefault(); event.stopPropagation(); event.stopImmediatePropagation(); - cleanup(); - resolve(describe(selected)); }; const key = (event) => { if (event.key === 'Escape') { @@ -1287,6 +1335,25 @@ export class BrowserSession implements BrowserLike { event.stopPropagation(); cleanup(); reject(new Error('已取消元素拾取')); + return; + } + if (event.code === 'Space' && !event.repeat) { + event.preventDefault(); + event.stopPropagation(); + if (locked) confirm(); + else lock(); + return; + } + if (locked && event.key === 'PageUp' && !event.repeat) { + event.preventDefault(); + event.stopPropagation(); + adjustLevel(-1); + return; + } + if (locked && event.key === 'PageDown' && !event.repeat) { + event.preventDefault(); + event.stopPropagation(); + adjustLevel(1); } }; const cleanup = () => { @@ -1317,6 +1384,7 @@ export class BrowserSession implements BrowserLike { window.addEventListener('click', click, true); window.addEventListener('scroll', updateBox, true); window.addEventListener('resize', updateBox); + window.addEventListener('keyup', () => {}, true); window.addEventListener('keydown', key, true); window.__rpaElementPicker = { cancel: () => { cleanup(); reject(new Error('已取消元素拾取')); } }; timer = setTimeout(() => { @@ -1370,6 +1438,16 @@ export async function externalBrowserSession(options: ExternalBrowserOptions): P return BrowserSession.createExternal(options); } +export function listExternalSessions(): BrowserSession[] { + return [...externalSessionCache().values()] + .filter((session) => session.isExternal && !session.closed) + .sort((left, right) => right.sessionKey.localeCompare(left.sessionKey)); +} + +export function latestExternalSession(): BrowserSession | null { + return listExternalSessions()[0] || null; +} + export function hasBrowserSession(): boolean { return Boolean(globalThis.__rpaBrowserWindow && !globalThis.__rpaBrowserWindow.isDestroyed()); } diff --git a/src/main/index.ts b/src/main/index.ts index 8728b93..6c3e9d9 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -1,7 +1,13 @@ import { app, BrowserWindow, dialog, ipcMain, shell } from "electron"; import path from "node:path"; import type { AppInfo, AppSettings, FlowDefinition, FlowSaveOptions, FlowSaveResult } from "../shared/types"; -import { BrowserSession, hasBrowserSession, setBrowserElementWaitMs } from "./browser/session"; +import { + BrowserSession, + externalBrowserSession, + hasBrowserSession, + latestExternalSession, + setBrowserElementWaitMs +} from "./browser/session"; import "./capabilities"; import { catalogPath, loadCatalog } from "./catalog"; import { createDemoFlow, createExampleFlows, createFlow, deleteFlow, getFlow, listFlows, saveFlow } from "./flows"; @@ -103,14 +109,30 @@ function registerIpc(): void { setBrowserElementWaitMs(settings.browserElementWaitMs); return settings; }); - ipcMain.handle("browser:pick-element", async (_event, options: { url?: string } = {}) => { + type BrowserPickOptions = { + url?: string; + browser?: import("./browser/external").ExternalBrowserOptions; + }; + + ipcMain.handle("browser:pick-element", async (_event, options: BrowserPickOptions = {}) => { const url = options.url?.trim(); - if (!hasBrowserSession() && !url) { - throw new Error("请先配置或运行「打开网页」，再拾取元素"); + const browserOptions = options.browser; + let session; + if (browserOptions) { + session = await externalBrowserSession( + browserOptions.mode === "launch" && url ? { ...browserOptions, initialUrl: url } : browserOptions + ); + } else { + session = latestExternalSession(); + } + if (!session) { + if (!hasBrowserSession() && !url) { + throw new Error("请先配置或运行「打开网页」，再拾取元素"); + } + session = await BrowserSession.create(); } - const session = await BrowserSession.create(); const currentUrl = await session.currentUrl(); - if (url && (currentUrl === "about:blank" || currentUrl !== url)) { + if (url && currentUrl !== url) { await session.navigate(url); } return session.pickElement(); diff --git a/src/preload/index.ts b/src/preload/index.ts index 6ba6b75..6878953 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -29,7 +29,7 @@ const api = { ipcRenderer.invoke("dialog:save", options || {}) }, browser: { - pickElement: (options?: { url?: string }): Promise => + pickElement: (options?: { url?: string; browser?: { mode: "launch"; browserType: "chrome" | "edge"; port?: number; extraArgs?: string[] } | { mode: "remote"; host: string; port: number } }): Promise => ipcRenderer.invoke("browser:pick-element", options || {}) }, visionLab: { diff --git a/src/renderer/src/panels/Properties.tsx b/src/renderer/src/panels/Properties.tsx index 08be64e..967fb96 100644 --- a/src/renderer/src/panels/Properties.tsx +++ b/src/renderer/src/panels/Properties.tsx @@ -438,6 +438,49 @@ function PropertyTab({ ); } +interface FileFilterOption { + name: string; + extensions: string[]; +} + +/** + * 按组件与字段上下文推导文件选择对话框的类型过滤器。 + * 先匹配字段本身的含义（字段键/标签），再匹配组件类别；无法识别时退回「全部文件」。 + */ +function fileFiltersFor(componentId: string, field: CatalogField): FileFilterOption[] { + const haystack = `${componentId} ${field.key} ${field.label ?? ""}`; + const filters: FileFilterOption[] = []; + if (/excel|bivartable|databook|工作簿|表格/i.test(haystack)) { + filters.push({ name: "Excel 表格", extensions: ["xlsx", "xlsm", "xls", "csv"] }); + } + if (/word|文档/i.test(haystack)) { + filters.push({ name: "Word 文档", extensions: ["docx", "doc"] }); + } + if (/pdf/i.test(haystack)) { + filters.push({ name: "PDF 文档", extensions: ["pdf"] }); + } + if (/image|ocr|CV_|vision|图片|图像|截图|模板|照片/i.test(haystack)) { + filters.push({ name: "图片文件", extensions: ["png", "jpg", "jpeg", "bmp", "gif", "webp"] }); + } + if (/zip|compress|archive|压缩/i.test(haystack)) { + filters.push({ name: "压缩文件", extensions: ["zip", "rar", "7z", "gz"] }); + } + if (/audio|speech|音频|语音|录音/i.test(haystack)) { + filters.push({ name: "音频文件", extensions: ["wav", "mp3", "flac", "ogg"] }); + } + if (/video|视频/i.test(haystack)) { + filters.push({ name: "视频文件", extensions: ["mp4", "avi", "mkv", "mov"] }); + } + if (/json/i.test(haystack)) { + filters.push({ name: "JSON 文件", extensions: ["json"] }); + } + if (!filters.length) { + return [{ name: "全部文件", extensions: ["*"] }]; + } + filters.push({ name: "全部文件", extensions: ["*"] }); + return filters; +} + function FieldControl({ field, node, @@ -486,12 +529,7 @@ function FieldControl({ const selected = await window.rpa.dialog.open({ title: field.label, directories, - filters: directories - ? undefined - : [ - { name: "Excel", extensions: ["xlsx", "xlsm", "xls"] }, - { name: "全部文件", extensions: ["*"] } - ] + filters: directories ? undefined : fileFiltersFor(node.componentId, field) }); if (selected) onChange(field.key, selected); } catch (error) { @@ -506,12 +544,24 @@ function FieldControl({ const currentFlow = useEditorStore.getState().flow; const nodeIndex = currentFlow?.nodes.findIndex((item) => item.id === node.id) ?? -1; const openNode = currentFlow && nodeIndex > 0 - ? [...currentFlow.nodes.slice(0, nodeIndex)].reverse().find((item) => { - const url = textOf(item.values.url).trim(); - return Boolean(url && !url.startsWith("${")); - }) + ? [...currentFlow.nodes.slice(0, nodeIndex)].reverse().find((item) => /^WebBrowser_Create/.test(item.componentId)) : undefined; - const picked = await window.rpa.browser.pickElement({ url: textOf(openNode?.values.url) || undefined }); + let browser; + if (openNode) { + const type = textOf(openNode.values.type).trim().toLowerCase(); + const isRemote = ["1", "true", "yes"].includes(textOf(openNode.values.isRemote).trim().toLowerCase()); + const port = Number(textOf(openNode.values.port) || 9222) || 9222; + if (isRemote) { + browser = { mode: "remote" as const, host: textOf(openNode.values.ipAddress) || "127.0.0.1", port }; + } else if (type === "chrome" || type === "edge") { + const extraArgs = textOf(openNode.values.arguments).split(/\s+/).map((item) => item.trim()).filter(Boolean); + browser = { mode: "launch" as const, browserType: type === "edge" ? "edge" as const : "chrome" as const, port, extraArgs }; + } + } + const picked = await window.rpa.browser.pickElement({ + url: textOf(openNode?.values.url).trim() || undefined, + browser + }); onChange(field.key, picked.selector); onToast(`已拾取 ${picked.tag}${picked.label ? ` · ${picked.label}` : ""}`); } catch (error) { @@ -652,8 +702,8 @@ function FieldControl({ ) : ( - )}