import { BrowserWindow } from "electron"; import { promises as fs } from "node:fs"; import path from "node:path"; import { RuntimeError, type BrowserElementRegion, type BrowserLike } from "../engine/types"; const DEFAULT_DOMAINS = ["127.0.0.1", "localhost", "example.com"]; function allowedDomains(): string[] { const extra = (process.env.RPA_ALLOWED_DOMAINS || "") .split(",") .map((item) => item.trim()) .filter(Boolean); return [...DEFAULT_DOMAINS, ...extra]; } function assertUrl(raw: string): URL { let url: URL; try { url = new URL(raw); } catch { throw new RuntimeError("blocked_url", `无效的 URL:${raw}`, { recommendation: "填写完整的 http/https 地址" }); } if (url.protocol !== "http:" && url.protocol !== "https:") { throw new RuntimeError("blocked_url", `只允许 http/https,当前为 ${url.protocol}`); } const host = url.hostname; const allowed = allowedDomains().some((domain) => host === domain || host.endsWith(`.${domain}`)); if (!allowed) { throw new RuntimeError("blocked_url", `域名不在白名单:${host}`, { recommendation: `在设置中把 ${host} 加入白名单(或设置 RPA_ALLOWED_DOMAINS)` }); } return url; } interface ElementInfo { tag: string; text: string; value: string; visible: boolean; disabled: boolean; rect: { x: number; y: number; width: number; height: number }; count: number; } function selectorSpec(selector: unknown): { kind: string; value: string; role?: string; name?: string; attr?: string } { if (selector && typeof selector === "object") { const record = selector as Record; if (record.kind && record.value) { return { kind: String(record.kind), value: String(record.value), role: record.role as string, name: record.name as string }; } const common = record.commonProps as Record | undefined; if (common?.cssSelector) { return { kind: "css", value: String(common.cssSelector) }; } if (record.selector && Array.isArray(record.selector)) { const last = [...record.selector].reverse().find((item) => (item as Record)?.name); if (last) { const item = last as Record; const props = (item.props as Array>) || []; const id = props.find((prop) => prop.name === "id")?.textValue; const cls = props.find((prop) => prop.name === "class")?.textValue; const tag = String(item.name); if (id) return { kind: "css", value: `${tag}#${id}` }; if (cls) return { kind: "css", value: `${tag}.${String(cls).split(/\s+/).join(".")}` }; return { kind: "css", value: tag }; } } } const text = String(selector ?? "").trim(); if (!text) throw new RuntimeError("validation_error", "元素选择器为空", { recommendation: "使用 css:#id、text:提交 或 aria:button=提交" }); if (text.startsWith("css:")) return { kind: "css", value: text.slice(4) }; if (text.startsWith("testid:")) return { kind: "css", value: `[data-testid="${text.slice(7).replace(/"/g, '\\"')}"]` }; if (text.startsWith("text:")) return { kind: "text", value: text.slice(5) }; if (text.startsWith("aria:")) { const rest = text.slice(5); const [role, name] = rest.split("="); return { kind: "aria", value: role, role, name: name || "" }; } return { kind: "css", value: text }; } export class BrowserSession implements BrowserLike { private win: BrowserWindow; private constructor(win: BrowserWindow) { this.win = win; } static async create(): Promise { if (globalThis.__rpaBrowserWindow && !globalThis.__rpaBrowserWindow.isDestroyed()) { return new BrowserSession(globalThis.__rpaBrowserWindow); } const win = new BrowserWindow({ width: 1200, height: 820, show: process.env.RPA_E2E !== "1", title: "RPA 运行浏览器", backgroundColor: "#ffffff", webPreferences: { contextIsolation: true, nodeIntegration: false, sandbox: true } }); globalThis.__rpaBrowserWindow = win; win.on("closed", () => { globalThis.__rpaBrowserWindow = null; }); await win.loadURL("about:blank"); if (!win.webContents.debugger.isAttached()) { win.webContents.debugger.attach("1.3"); } return new BrowserSession(win); } private async cdp(method: string, params: Record = {}): Promise { if (this.win.isDestroyed()) { throw new RuntimeError("tab_unavailable", "运行浏览器已关闭", { recommendation: "重新运行流程" }); } try { if (!this.win.webContents.debugger.isAttached()) this.win.webContents.debugger.attach("1.3"); return (await this.win.webContents.debugger.sendCommand(method, params)) as T; } catch (error) { if (/destroyed|Target closed|not attached/i.test((error as Error).message)) { throw new RuntimeError("tab_unavailable", "浏览器连接已失效", { detail: (error as Error).message }); } throw new RuntimeError("browser_error", (error as Error).message, { recommendation: "重试或检查页面状态" }); } } private async evaluate(expression: string): Promise { const result = await this.cdp<{ result?: { value?: T }; exceptionDetails?: { exception?: { description?: string }; text?: string } }>( "Runtime.evaluate", { expression, returnByValue: true, awaitPromise: true } ); if (result.exceptionDetails) { throw new RuntimeError( "evaluate_error", result.exceptionDetails.exception?.description || result.exceptionDetails.text || "页面脚本执行失败" ); } return result.result?.value as T; } private async withElement(selector: unknown, operation: string): Promise { const spec = selectorSpec(selector); const result = await this.evaluate<{ found: boolean; value?: T }>(`(() => { const spec = ${JSON.stringify(spec)}; const find = () => { if (spec.kind === 'text') { const target = spec.value; const matches = [...document.querySelectorAll('*')].filter((node) => { const text = (node.innerText || node.textContent || '').trim(); return text === target || (text.includes(target) && text.length <= target.length + 20); }); return matches.sort((a, b) => (a.innerText || '').length - (b.innerText || '').length)[0] || null; } if (spec.kind === 'aria') { const role = spec.role || spec.value; const name = spec.name || ''; return [...document.querySelectorAll('[role="' + role + '"], ' + role)].find((node) => { const accessibleName = node.getAttribute('aria-label') || node.innerText || ''; return !name || accessibleName.includes(name); }) || null; } return document.querySelector(spec.value); }; const el = find(); if (!el) return { found: false }; return { found: true, value: (() => { ${operation} })() }; })()`); if (!result.found) { throw new RuntimeError("not_found", `Element not found: ${spec.kind}:${spec.value}`); } return result.value as T; } private regionOperation(): string { return ` const rect = el.getBoundingClientRect(); const x = rect.x; const y = rect.y; return { x, y, width: rect.width, height: rect.height, left: rect.left, top: rect.top, right: rect.right, bottom: rect.bottom, centerX: x + rect.width / 2, centerY: y + rect.height / 2, pageX: x + window.scrollX, pageY: y + window.scrollY }; `; } async navigate(url: string): Promise<{ url: string; title: string }> { const target = assertUrl(url); await this.cdp("Page.enable"); await this.win.loadURL(target.toString()); return { url: this.win.webContents.getURL(), title: this.win.webContents.getTitle() }; } async locate(selector: unknown): Promise { const spec = selectorSpec(selector); const expression = `(() => { const spec = ${JSON.stringify(spec)}; const visible = (nodes) => { const result = []; for (const el of nodes) { const style = getComputedStyle(el); const rect = el.getBoundingClientRect(); if (rect.width <= 0 || rect.height <= 0) continue; if (style.display === 'none' || style.visibility === 'hidden' || Number(style.opacity) === 0) continue; result.push(el); } return result; }; let matches = []; if (spec.kind === 'text') { const target = spec.value; const all = [...document.querySelectorAll('*')].filter((node) => { const text = (node.innerText || node.textContent || '').trim(); return text === target || (text.includes(target) && text.length <= target.length + 20); }); matches = visible(all.sort((a, b) => (a.innerText || '').length - (b.innerText || '').length)); } else if (spec.kind === 'aria') { const role = spec.role || spec.value; const name = spec.name || ''; const all = [...document.querySelectorAll('[role="' + role + '"], ' + role)]; matches = visible(all.filter((node) => { const acc = node.getAttribute('aria-label') || node.innerText || ''; return !name || acc.includes(name); })); } else { matches = visible([...document.querySelectorAll(spec.value)]); } const el = matches[0] || null; if (!el) return null; el.scrollIntoView({ block: 'center', inline: 'center' }); const rect = el.getBoundingClientRect(); return { tag: el.tagName.toLowerCase(), text: (el.innerText || el.textContent || '').trim().slice(0, 200), value: el.value ?? '', visible: rect.width > 0 && rect.height > 0, disabled: Boolean(el.disabled), count: matches.length, rect: { x: rect.x, y: rect.y, width: rect.width, height: rect.height } }; })()`; const info = await this.evaluate(expression); if (!info) { throw new RuntimeError("not_found", `未找到元素:${spec.kind}:${spec.value}`, { recommendation: "检查选择器,或先用拾取器重新采集" }); } if (info.count > 1) { throw new RuntimeError("ambiguous", `选择器匹配到 ${info.count} 个可见元素:${spec.kind}:${spec.value}`, { recommendation: "使用更稳定的 data-testid、aria-label 或唯一 CSS 选择器" }); } return { ...info, selector: `${spec.kind}:${spec.value}` }; } private async mouseClick(x: number, y: number): Promise { const point = { x: Math.round(x), y: Math.round(y) }; await this.cdp("Input.dispatchMouseEvent", { type: "mouseMoved", ...point, button: "none", buttons: 0 }); await new Promise((resolve) => setTimeout(resolve, 35)); await this.cdp("Input.dispatchMouseEvent", { type: "mousePressed", ...point, button: "left", buttons: 1, clickCount: 1 }); await new Promise((resolve) => setTimeout(resolve, 25)); await this.cdp("Input.dispatchMouseEvent", { type: "mouseReleased", ...point, button: "left", buttons: 0, clickCount: 1 }); } async click(selector: unknown): Promise { const info = await this.locate(selector); if (!info.visible) { throw new RuntimeError("not_found", `元素不可见:${info.selector}`, { recommendation: "等待元素出现或检查页面状态" }); } if (info.disabled) { throw new RuntimeError("not_found", `元素已禁用:${info.selector}`); } const clickable = await this.withElement(selector, ` const rect = el.getBoundingClientRect(); const hit = document.elementFromPoint(rect.left + rect.width / 2, rect.top + rect.height / 2); return Boolean(hit && (hit === el || el.contains(hit) || hit.contains(el))); `); if (!clickable) throw new RuntimeError("not_found", `目标元素被其他内容遮挡:${info.selector}`, { recommendation: "关闭遮挡弹窗或等待页面稳定" }); const clickToken = `rpa_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 7)}`; await this.withElement(selector, ` const token = ${JSON.stringify(clickToken)}; el.addEventListener('click', () => { el.dataset.rpaClickToken = token; }, { once: true }); return true; `); await this.mouseClick(info.rect.x + info.rect.width / 2, info.rect.y + info.rect.height / 2); await new Promise((resolve) => setTimeout(resolve, 180)); const clicked = await this.withElement(selector, ` const matched = el.dataset.rpaClickToken === ${JSON.stringify(clickToken)}; delete el.dataset.rpaClickToken; return matched; `); if (!clicked) throw new RuntimeError("not_found", `目标未收到可信点击:${info.selector}`, { recommendation: "检查元素是否被遮挡或重新拾取元素" }); return { selector: info.selector, tag: info.tag, text: info.text }; } async type(selector: unknown, text: string, clearFirst = true): Promise { const info = await this.locate(selector); await this.mouseClick(info.rect.x + info.rect.width / 2, info.rect.y + info.rect.height / 2); if (clearFirst) { await this.cdp("Input.dispatchKeyEvent", { type: "rawKeyDown", key: "a", code: "KeyA", windowsVirtualKeyCode: 65, modifiers: 2 }); await this.cdp("Input.dispatchKeyEvent", { type: "keyUp", key: "a", code: "KeyA", windowsVirtualKeyCode: 65, modifiers: 2 }); await new Promise((resolve) => setTimeout(resolve, 30)); } await this.cdp("Input.insertText", { text }); await new Promise((resolve) => setTimeout(resolve, 120)); const expected = clearFirst ? text : `${info.value || ""}${text}`; const actual = await this.evaluate(`(() => { const el = document.activeElement; return el ? (el.value ?? el.textContent ?? '') : ''; })()`); if (actual !== expected) throw new RuntimeError("evaluate_error", `输入校验失败:期望「${expected}」,实际「${actual}」`, { recommendation: "检查输入框状态或重新拾取元素" }); return { selector: info.selector, value: actual }; } async getText(selector: unknown, attribute?: string): Promise { const info = await this.locate(selector); if (attribute) { const value = await this.evaluate(`(() => { const spec = ${JSON.stringify(selectorSpec(selector))}; const el = document.querySelector(spec.value); return el ? el.getAttribute(${JSON.stringify(attribute)}) : null; })()`); return value ?? ""; } return info.value || info.text; } async exists(selector: unknown): Promise { try { await this.locate(selector); return true; } catch { return false; } } async select(selector: unknown, value: string): Promise { const info = await this.locate(selector); const applied = await this.evaluate(`(() => { const spec = ${JSON.stringify(selectorSpec(selector))}; const el = document.querySelector(spec.value); if (!el) return false; el.focus(); el.value = ${JSON.stringify(value)}; el.dispatchEvent(new Event('input', { bubbles: true })); el.dispatchEvent(new Event('change', { bubbles: true })); return true; })()`); if (!applied) throw new RuntimeError("not_found", `下拉框未找到:${info.selector}`); return { selector: info.selector, value }; } async runJs(script: string): Promise { return this.evaluate(script); } async download(selector: unknown, folder: string, filename = "", timeoutMs = 10000): Promise { const targetFolder = path.resolve(folder); await fs.mkdir(targetFolder, { recursive: true }); return new Promise((resolve, reject) => { let settled = false; const browserSession = this.win.webContents.session; const onDownload = (_event: Electron.Event, item: Electron.DownloadItem) => { const target = path.join(targetFolder, filename || item.getFilename()); item.setSavePath(target); item.once("done", (_doneEvent, state) => { if (state === "completed") finish(undefined, target); else finish(new RuntimeError("browser_error", `下载未完成:${state}`)); }); }; const finish = (error?: Error, filePath?: string) => { if (settled) return; settled = true; clearTimeout(timer); browserSession.removeListener("will-download", onDownload); error ? reject(error) : resolve(filePath || ""); }; const timer = setTimeout(() => finish(new RuntimeError("timeout", `下载超时(${timeoutMs}ms)`)), timeoutMs); browserSession.once("will-download", onDownload); void this.click(selector).catch((error) => finish(error instanceof Error ? error : new Error(String(error)))); }); } async screenshot(filePath: string): Promise { const result = await this.cdp<{ data: string }>("Page.captureScreenshot", { format: "png" }); const target = path.resolve(filePath); await fs.mkdir(path.dirname(target), { recursive: true }); await fs.writeFile(target, Buffer.from(result.data, "base64")); return target; } async getAttribute(selector: unknown, name: string): Promise { return this.withElement(selector, `return el.getAttribute(${JSON.stringify(name)}) ?? '';`); } async setAttribute(selector: unknown, name: string, value: string): Promise { return this.withElement(selector, ` el.setAttribute(${JSON.stringify(name)}, ${JSON.stringify(value)}); return { name: ${JSON.stringify(name)}, value: el.getAttribute(${JSON.stringify(name)}) ?? '' }; `); } async getHtml(selector: unknown): Promise { return this.withElement(selector, "return el.outerHTML;"); } async getChecked(selector: unknown): Promise { return this.withElement(selector, ` if (!('checked' in el)) throw new Error('The target element has no checked state'); return Boolean(el.checked); `); } async setChecked(selector: unknown, checked: boolean): Promise { return this.withElement(selector, ` if (!('checked' in el)) throw new Error('The target element has no checked state'); const next = ${checked ? "true" : "false"}; if (Boolean(el.checked) !== next) { el.checked = next; el.dispatchEvent(new Event('input', { bubbles: true })); el.dispatchEvent(new Event('change', { bubbles: true })); } return Boolean(el.checked); `); } async isEnabled(selector: unknown, includeHidden = false): Promise { return this.withElement(selector, ` const style = getComputedStyle(el); const rect = el.getBoundingClientRect(); const visible = rect.width > 0 && rect.height > 0 && style.display !== 'none' && style.visibility !== 'hidden' && Number(style.opacity) !== 0; const enabled = !Boolean(el.disabled) && el.getAttribute('aria-disabled') !== 'true' && style.pointerEvents !== 'none'; return enabled && (${includeHidden ? "true" : "false"} || visible); `); } async scrollIntoView(selector: unknown): Promise { return this.withElement(selector, ` el.scrollIntoView({ block: 'center', inline: 'center' }); ${this.regionOperation()} `); } async getRegion(selector: unknown): Promise { return this.withElement(selector, this.regionOperation()); } async hover(selector: unknown): Promise { const region = await this.scrollIntoView(selector); await this.cdp("Input.dispatchMouseEvent", { type: "mouseMoved", x: Math.round(region.centerX), y: Math.round(region.centerY), button: "none" }); await new Promise((resolve) => setTimeout(resolve, 120)); return region; } async screenshotElement(selector: unknown, filePath: string, captureBeyondViewport = true): Promise { const region = await this.scrollIntoView(selector); if (region.width <= 0 || region.height <= 0) { throw new RuntimeError("not_found", "The target element has no screenshot area"); } await this.cdp("Page.enable"); const result = await this.cdp<{ data: string }>("Page.captureScreenshot", { format: "png", captureBeyondViewport, clip: { x: region.pageX, y: region.pageY, width: region.width, height: region.height, scale: 1 } }); const target = path.resolve(filePath); await fs.mkdir(path.dirname(target), { recursive: true }); await fs.writeFile(target, Buffer.from(result.data, "base64")); return target; } async currentUrl(): Promise { return this.win.webContents.getURL(); } async close(): Promise { if (!this.win.isDestroyed()) this.win.close(); globalThis.__rpaBrowserWindow = null; } } declare global { // eslint-disable-next-line no-var var __rpaBrowserWindow: BrowserWindow | null | undefined; } export async function browserSession(): Promise { return BrowserSession.create(); } export function assertAllowedUrl(url: string): URL { return assertUrl(url); }