From 6adf7729873d194cb02c1c4d99bc770d89debc5f Mon Sep 17 00:00:00 2001 From: yu <191972707@qq.com> Date: Tue, 27 Oct 2026 13:50:03 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=E6=9C=AC=E5=9C=B0?= =?UTF-8?q?=E6=BC=94=E7=A4=BA=E6=B5=81=E7=A8=8B=E4=B8=8E=E7=BD=91=E9=A1=B5?= =?UTF-8?q?DOM=E8=83=BD=E5=8A=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 3 +- docs/02-实施方案.md | 2 +- resources/fixture/index.html | 34 ++++ src/main/browser/session.ts | 132 ++++++++++++++- src/main/capabilities/browser-dom.test.ts | 125 ++++++++++++++ src/main/capabilities/browser-dom.ts | 154 ++++++++++++++++++ src/main/capabilities/index.ts | 1 + src/main/engine/types.ts | 25 +++ src/main/fixture/server.test.ts | 19 +++ src/main/fixture/server.ts | 69 ++++++++ src/main/flows.test.ts | 12 +- src/main/flows.ts | 21 +++ src/main/index.ts | 18 +- src/preload/index.ts | 1 + src/renderer/src/App.tsx | 2 +- src/renderer/src/pages/SettingsPage.tsx | 2 +- .../src/validation/flowValidation.test.ts | 18 ++ src/renderer/src/validation/flowValidation.ts | 12 ++ src/shared/types.ts | 1 + 19 files changed, 642 insertions(+), 9 deletions(-) create mode 100644 resources/fixture/index.html create mode 100644 src/main/capabilities/browser-dom.test.ts create mode 100644 src/main/capabilities/browser-dom.ts create mode 100644 src/main/fixture/server.test.ts create mode 100644 src/main/fixture/server.ts diff --git a/README.md b/README.md index 0cecf19..db458d3 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ docs 现状分析、实施方案 ## 边界 -- 1016 项目录中当前有 363 项通过能力注册表或运行时容器标记为可执行，覆盖控制/触发、字符串/集合/INI/二维表格、文件、浏览器元素常用变体、HTTP、Excel/CSV与表格辅助、系统路径与环境变量、进程管理、桌面键鼠/窗口对象/剪贴板/对话框、PDF、Word、邮件发送、屏幕截图、离线图像查找与点击；其余灰标「待接入」。运行前检查会提前定位待接入组件、必填参数、多出口与断链节点。 +- 1016 项目录中当前有 398 项通过能力注册表或运行时容器标记为可执行，覆盖控制/触发、字符串/集合/INI/二维表格、文件、浏览器元素常用及 DOM 高级能力、HTTP、Excel/CSV与表格辅助、系统路径与环境变量、进程管理、桌面键鼠/窗口对象/剪贴板/对话框、PDF、Word、邮件发送、屏幕截图、离线图像查找与点击；其余灰标「待接入」。运行前检查会提前定位待接入组件、必填参数、多出口与断链节点。 +- 内置本地演示站默认监听 `127.0.0.1:17863`；新用户首次启动会获得“本地网页订单演示”任务，可离线验证网页输入、点击和文本读取。 - 不包含厂商代码、二进制、模型或前端资产；目录是从厂商元数据派生的静态定义。 - 运行期不访问云服务；HTTP/邮件属于用户显式配置的外部系统。 diff --git a/docs/02-实施方案.md b/docs/02-实施方案.md index eb7ba94..7ef8b48 100644 --- a/docs/02-实施方案.md +++ b/docs/02-实施方案.md @@ -4,7 +4,7 @@ > 依据：`D:\code\Agent-Employee`（只读）、`C:\rpa-demo`（只读）、2026-10-26 需求确认 > 本方案是执行基线：范围、契约、排期、验收、降级顺序都在这里；实施中变更必须回写本文件。 -> **2026-10-27 当前状态**：计划点名的代表能力已全部进入注册表或运行时容器，覆盖控制/触发、字符串/集合/INI/二维表格、文件、浏览器元素常用变体、HTTP、Excel/CSV与表格辅助、系统路径与环境变量、进程管理、桌面键鼠/窗口对象/剪贴板/对话框、PDF、Word、邮件发送、屏幕截图、离线图像查找与点击；启动后目录显示 **363 项可执行 / 1016 项总目录**。当前 13 个测试文件共 89 条测试，覆盖计划能力审计、浏览器参数路由与变体、Excel/PDF/Word/SMTP/模板匹配、文件与数据扩展、系统/进程/窗口解析、直线/IF/循环/重试/跳过、主出口与异常出口语义、运行记录回写和运行前检查。设置页可查看分类覆盖和处理器 ID 数量。未通过实际场景验证的变体仍不计入最终交付数量。 +> **2026-10-27 当前状态**：计划点名的代表能力已全部进入注册表或运行时容器，覆盖控制/触发、字符串/集合/INI/二维表格、文件、浏览器元素常用及 DOM 高级能力、HTTP、Excel/CSV与表格辅助、系统路径与环境变量、进程管理、桌面键鼠/窗口对象/剪贴板/对话框、PDF、Word、邮件发送、屏幕截图、离线图像查找与点击；启动后目录显示 **398 项可执行 / 1016 项总目录**。当前 15 个测试文件共 99 条测试，覆盖计划能力审计、浏览器参数路由/变体/DOM、Excel/PDF/Word/SMTP/模板匹配、文件与数据扩展、系统/进程/窗口解析、直线/IF/循环/重试/跳过、主出口与异常出口语义、运行记录回写、运行前检查、本地演示站与示例流程。设置页可查看分类覆盖和处理器 ID 数量。未通过实际场景验证的变体仍不计入最终交付数量。 --- diff --git a/resources/fixture/index.html b/resources/fixture/index.html new file mode 100644 index 0000000..54cc1a0 --- /dev/null +++ b/resources/fixture/index.html @@ -0,0 +1,34 @@ + + + + + + RPA Studio 本地订单夹具 + + + +
+

本地订单录入

+

该页面由 RPA Studio 在本机提供，不访问互联网，用于验证输入、点击、读取和下载。

+
+ + +
+
下载示例 CSV
+
等待提交
+
客户金额状态
+
+ + + diff --git a/src/main/browser/session.ts b/src/main/browser/session.ts index 096e707..90ba211 100644 --- a/src/main/browser/session.ts +++ b/src/main/browser/session.ts @@ -1,7 +1,7 @@ import { BrowserWindow } from "electron"; import { promises as fs } from "node:fs"; import path from "node:path"; -import { RuntimeError, type BrowserLike } from "../engine/types"; +import { RuntimeError, type BrowserElementRegion, type BrowserLike } from "../engine/types"; const DEFAULT_DOMAINS = ["127.0.0.1", "localhost", "example.com"]; @@ -142,6 +142,53 @@ export class BrowserSession implements BrowserLike { return result.result?.value as T; } + private async withElement(selector: unknown, operation: string): Promise { + const spec = selectorSpec(selector); + const result = await this.evaluate<{ found: boolean; value?: T }>(`(() => { + const spec = ${JSON.stringify(spec)}; + const find = () => { + if (spec.kind === 'text') { + const target = spec.value; + const matches = [...document.querySelectorAll('*')].filter((node) => { + const text = (node.innerText || node.textContent || '').trim(); + return text === target || (text.includes(target) && text.length <= target.length + 20); + }); + return matches.sort((a, b) => (a.innerText || '').length - (b.innerText || '').length)[0] || null; + } + if (spec.kind === 'aria') { + const role = spec.role || spec.value; + const name = spec.name || ''; + return [...document.querySelectorAll('[role="' + role + '"], ' + role)].find((node) => { + const accessibleName = node.getAttribute('aria-label') || node.innerText || ''; + return !name || accessibleName.includes(name); + }) || null; + } + return document.querySelector(spec.value); + }; + const el = find(); + if (!el) return { found: false }; + return { found: true, value: (() => { ${operation} })() }; + })()`); + if (!result.found) { + throw new RuntimeError("not_found", `Element not found: ${spec.kind}:${spec.value}`); + } + return result.value as T; + } + + private regionOperation(): string { + return ` + const rect = el.getBoundingClientRect(); + const x = rect.x; + const y = rect.y; + return { + x, y, width: rect.width, height: rect.height, + left: rect.left, top: rect.top, right: rect.right, bottom: rect.bottom, + centerX: x + rect.width / 2, centerY: y + rect.height / 2, + pageX: x + window.scrollX, pageY: y + window.scrollY + }; + `; + } + async navigate(url: string): Promise<{ url: string; title: string }> { const target = assertUrl(url); await this.cdp("Page.enable"); @@ -318,6 +365,89 @@ export class BrowserSession implements BrowserLike { return target; } + async getAttribute(selector: unknown, name: string): Promise { + return this.withElement(selector, `return el.getAttribute(${JSON.stringify(name)}) ?? '';`); + } + + async setAttribute(selector: unknown, name: string, value: string): Promise { + return this.withElement(selector, ` + el.setAttribute(${JSON.stringify(name)}, ${JSON.stringify(value)}); + return { name: ${JSON.stringify(name)}, value: el.getAttribute(${JSON.stringify(name)}) ?? '' }; + `); + } + + async getHtml(selector: unknown): Promise { + return this.withElement(selector, "return el.outerHTML;"); + } + + async getChecked(selector: unknown): Promise { + return this.withElement(selector, ` + if (!('checked' in el)) throw new Error('The target element has no checked state'); + return Boolean(el.checked); + `); + } + + async setChecked(selector: unknown, checked: boolean): Promise { + return this.withElement(selector, ` + if (!('checked' in el)) throw new Error('The target element has no checked state'); + const next = ${checked ? "true" : "false"}; + if (Boolean(el.checked) !== next) { + el.checked = next; + el.dispatchEvent(new Event('input', { bubbles: true })); + el.dispatchEvent(new Event('change', { bubbles: true })); + } + return Boolean(el.checked); + `); + } + + async isEnabled(selector: unknown, includeHidden = false): Promise { + return this.withElement(selector, ` + const style = getComputedStyle(el); + const rect = el.getBoundingClientRect(); + const visible = rect.width > 0 && rect.height > 0 && style.display !== 'none' && + style.visibility !== 'hidden' && Number(style.opacity) !== 0; + const enabled = !Boolean(el.disabled) && el.getAttribute('aria-disabled') !== 'true' && style.pointerEvents !== 'none'; + return enabled && (${includeHidden ? "true" : "false"} || visible); + `); + } + + async scrollIntoView(selector: unknown): Promise { + return this.withElement(selector, ` + el.scrollIntoView({ block: 'center', inline: 'center' }); + ${this.regionOperation()} + `); + } + + async getRegion(selector: unknown): Promise { + return this.withElement(selector, this.regionOperation()); + } + + async hover(selector: unknown): Promise { + const region = await this.scrollIntoView(selector); + await this.cdp("Input.dispatchMouseEvent", { + type: "mouseMoved", x: Math.round(region.centerX), y: Math.round(region.centerY), button: "none" + }); + await new Promise((resolve) => setTimeout(resolve, 120)); + return region; + } + + async screenshotElement(selector: unknown, filePath: string, captureBeyondViewport = true): Promise { + const region = await this.scrollIntoView(selector); + if (region.width <= 0 || region.height <= 0) { + throw new RuntimeError("not_found", "The target element has no screenshot area"); + } + await this.cdp("Page.enable"); + const result = await this.cdp<{ data: string }>("Page.captureScreenshot", { + format: "png", + captureBeyondViewport, + clip: { x: region.pageX, y: region.pageY, width: region.width, height: region.height, scale: 1 } + }); + const target = path.resolve(filePath); + await fs.mkdir(path.dirname(target), { recursive: true }); + await fs.writeFile(target, Buffer.from(result.data, "base64")); + return target; + } + async currentUrl(): Promise { return this.win.webContents.getURL(); } diff --git a/src/main/capabilities/browser-dom.test.ts b/src/main/capabilities/browser-dom.test.ts new file mode 100644 index 0000000..b88de2a --- /dev/null +++ b/src/main/capabilities/browser-dom.test.ts @@ -0,0 +1,125 @@ +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { CatalogComponent, FlowNode } from "../../shared/types"; +import { getHandler } from "../engine/registry"; +import { RuntimeError, type BrowserElementRegion, type BrowserLike, type RunContext } from "../engine/types"; +import { BROWSER_DOM_IDS } from "./browser-dom"; + +const component = {} as CatalogComponent; +const selector = { kind: "css", value: "#target" }; +const region: BrowserElementRegion = { + x: 10, y: 20, width: 100, height: 40, + left: 10, top: 20, right: 110, bottom: 60, + centerX: 60, centerY: 40, pageX: 10, pageY: 120 +}; + +function node(componentId: string): FlowNode { + return { id: `dom_${componentId}`, componentId, position: { x: 0, y: 0 }, values: {}, options: {} }; +} + +function fakeBrowser(): BrowserLike { + return { + navigate: vi.fn(async (url: string) => ({ url, title: "Fake page" })), + click: vi.fn(), type: vi.fn(), getText: vi.fn(), exists: vi.fn(), select: vi.fn(), runJs: vi.fn(), + download: vi.fn(), screenshot: vi.fn(), currentUrl: vi.fn(), close: vi.fn(), + getAttribute: vi.fn(async () => "ready"), + setAttribute: vi.fn(async (_selector, name, value) => ({ name, value })), + getHtml: vi.fn(async () => ''), + getChecked: vi.fn(async () => true), + setChecked: vi.fn(async (_selector, checked) => checked), + isEnabled: vi.fn(async () => true), + scrollIntoView: vi.fn(async () => region), + getRegion: vi.fn(async () => region), + hover: vi.fn(async () => region), + screenshotElement: vi.fn(async (_selector, target) => target) + }; +} + +function context(browser: BrowserLike): RunContext { + return { + runId: "dom-run", flowId: "dom-flow", flow: {} as RunContext["flow"], vars: {}, secrets: {}, + signal: new AbortController().signal, evidenceDir: "C:/temp/evidence", browser, + emit: vi.fn(), log: vi.fn(), componentOf: vi.fn() + }; +} + +async function invoke(ctx: RunContext, componentId: string, args: Record): Promise { + const handler = getHandler(componentId); + expect(handler, `${componentId} should be registered`).toBeTypeOf("function"); + return handler!(ctx, node(componentId), args, component); +} + +describe("browser DOM component registration", () => { + it("registers real catalog IDs without Ground variants", () => { + const catalog = JSON.parse(readFileSync(resolve("resources/catalog.json"), "utf8")) as { + components: Array<{ id: string }>; + }; + const catalogIds = new Set(catalog.components.map((item) => item.id)); + expect(BROWSER_DOM_IDS.length).toBeGreaterThanOrEqual(12); + expect(BROWSER_DOM_IDS.some((id) => id.includes("Ground"))).toBe(false); + expect(BROWSER_DOM_IDS.filter((id) => !catalogIds.has(id))).toEqual([]); + expect(BROWSER_DOM_IDS.every((id) => typeof getHandler(id) === "function")).toBe(true); + }); +}); + +describe("browser DOM capability routing", () => { + let browser: BrowserLike; + let ctx: RunContext; + + beforeEach(() => { + browser = fakeBrowser(); + ctx = context(browser); + }); + + it("gets and sets attributes and HTML", async () => { + await expect(invoke(ctx, "Element_GetAttribute_v3", { uiSelector: selector, attrName: "data-state" })).resolves.toBe("ready"); + expect(browser.getAttribute).toHaveBeenCalledWith(selector, "data-state"); + await invoke(ctx, "Element_SetAttribute_v2", { uiSelector: selector, attrName: "role", attrValue: "switch" }); + expect(browser.setAttribute).toHaveBeenCalledWith(selector, "role", "switch"); + await expect(invoke(ctx, "Element_GetHtml", { selector })).resolves.toContain("button"); + }); + + it("gets, sets and operates checkbox states", async () => { + await expect(invoke(ctx, "Element_GetCheck", { uiSelector: selector })).resolves.toBe(true); + await expect(invoke(ctx, "Element_SetCheck", { uiSelector: selector, isChecked: "0" })).resolves.toBe(false); + expect(browser.setChecked).toHaveBeenCalledWith(selector, false); + await invoke(ctx, "Element_CheckBoxOperation_v3", { uiSelector: selector, operation_type: "1" }); + expect(browser.setChecked).toHaveBeenLastCalledWith(selector, true); + await invoke(ctx, "Element_CheckBoxOperation_v3", { uiSelector: selector, operation_type: "2" }); + expect(browser.setChecked).toHaveBeenLastCalledWith(selector, false); + await expect(invoke(ctx, "Element_CheckBoxOperation_v3", { uiSelector: selector, operation_type: "3" })).resolves.toBe(true); + }); + + it("checks enabled state and honors hidden-element mode", async () => { + await expect(invoke(ctx, "Element_GetButtonEnable_v3", { + uiSelector: selector, identify_hidden_element: "1" + })).resolves.toBe(true); + expect(browser.isEnabled).toHaveBeenCalledWith(selector, true); + }); + + it("scrolls, reads regions and hovers", async () => { + await expect(invoke(ctx, "Element_ScrollIntoView_v2", { uiSelector: selector })).resolves.toEqual(region); + await expect(invoke(ctx, "Element_Region2_v3", { uiSelector: selector, position: "3" })).resolves.toEqual({ ...region, position: "3" }); + await expect(invoke(ctx, "Element_MouseTo_V4", { uiSelector: selector })).resolves.toEqual(region); + expect(browser.scrollIntoView).toHaveBeenCalledWith(selector); + expect(browser.getRegion).toHaveBeenCalledWith(selector); + expect(browser.hover).toHaveBeenCalledWith(selector); + }); + + it("captures element screenshots with viewport semantics", async () => { + const result = await invoke(ctx, "Element_Shot_V2", { + uiSelector: selector, filePath: "C:/temp/evidence", filename: "target.png", isUseVerifyCode: "1" + }); + expect(result).toMatch(/target\.png$/); + expect(browser.screenshotElement).toHaveBeenCalledWith(selector, expect.stringMatching(/target\.png$/), false); + }); + + it("rejects missing selectors, attributes and designated browser objects", async () => { + await expect(invoke(ctx, "Element_GetHtml", {})).rejects.toBeInstanceOf(RuntimeError); + await expect(invoke(ctx, "Element_GetAttribute", { uiSelector: selector })).rejects.toBeInstanceOf(RuntimeError); + await expect(invoke(ctx, "Element_SetAttribute_v2", { + uiSelector: selector, attrName: "role", attrValue: "button", designated_info: "1" + })).rejects.toMatchObject({ code: "validation_error" }); + }); +}); diff --git a/src/main/capabilities/browser-dom.ts b/src/main/capabilities/browser-dom.ts new file mode 100644 index 0000000..80c6add --- /dev/null +++ b/src/main/capabilities/browser-dom.ts @@ -0,0 +1,154 @@ +import path from "node:path"; +import { browserSession } from "../browser/session"; +import { register } from "../engine/registry"; +import { RuntimeError, type BrowserLike, type RunContext } from "../engine/types"; + +export const BROWSER_DOM_IDS = [ + "Element_GetAttribute", "Element_GetAttribute_v1", "Element_GetAttribute_v2", "Element_GetAttribute_v3", + "Element_SetAttribute", "Element_SetAttribute_v1", "Element_SetAttribute_v2", "Element_GetHtml", + "Element_GetCheck", "Element_SetCheck", "Element_CheckBoxOperation", "Element_CheckBoxOperation_v1", + "Element_CheckBoxOperation_v2", "Element_CheckBoxOperation_v3", "Element_GetButtonEnable", + "Element_GetButtonEnable_v1", "Element_GetButtonEnable_v2", "Element_GetButtonEnable_v3", + "Element_GetButtonEnable_v4", "Element_ScrollIntoView", "Element_ScrollIntoView_v1", + "Element_ScrollIntoView_v2", "Element_Region", "Element_Region2", "Element_Region2_v1", + "Element_Region2_v2", "Element_Region2_v3", "Element_MouseTo", "Element_MouseTo_V1", + "Element_MouseTo_V2", "Element_MouseTo_V3", "Element_MouseTo_V4", "Element_Shot", + "Element_Shot_V1", "Element_Shot_V2" +] as const; + +async function session(ctx: RunContext): Promise { + if (!ctx.browser) ctx.browser = await browserSession(); + return ctx.browser; +} + +function selectorOf(args: Record): unknown { + const selector = args.uiSelector ?? args.selector ?? args.target; + if (selector === undefined || selector === null || selector === "") { + throw new RuntimeError("validation_error", "元素选择器为空", { recommendation: "请拾取或填写网页元素" }); + } + return selector; +} + +function enabled(value: unknown): boolean { + return value === true || value === 1 || String(value).toLowerCase() === "true" || String(value) === "1"; +} + +function requiredText(args: Record, keys: string[], label: string): string { + for (const key of keys) { + const value = args[key]; + if (value !== undefined && value !== null && String(value).trim()) return String(value); + } + throw new RuntimeError("validation_error", `${label}为空`); +} + +function assertUnifiedBrowser(args: Record): void { + if (enabled(args.designated_info)) { + throw new RuntimeError("validation_error", "当前能力不支持指定其他网页或窗口对象", { + recommendation: "关闭指定网页/窗口对象，使用当前运行浏览器" + }); + } +} + +function requireMethod(browser: BrowserLike, key: K): NonNullable { + const method = browser[key]; + if (typeof method !== "function") throw new RuntimeError("browser_error", `浏览器不支持 DOM 方法：${String(key)}`); + return method.bind(browser) as NonNullable; +} + +register(["Element_GetAttribute", "Element_GetAttribute_v1", "Element_GetAttribute_v2", "Element_GetAttribute_v3"], async (ctx, node, args) => { + assertUnifiedBrowser(args); + const browser = await session(ctx); + const name = requiredText(args, ["attrName", "attribute", "name"], "属性名"); + const value = await requireMethod(browser, "getAttribute")(selectorOf(args), name); + ctx.log(`已获取属性 ${name}`, "info", node.id); + return value; +}); + +register(["Element_SetAttribute", "Element_SetAttribute_v1", "Element_SetAttribute_v2"], async (ctx, node, args) => { + assertUnifiedBrowser(args); + const browser = await session(ctx); + const name = requiredText(args, ["attrName", "attribute", "name"], "属性名"); + const value = String(args.attrValue ?? args.value ?? ""); + const result = await requireMethod(browser, "setAttribute")(selectorOf(args), name, value); + ctx.log(`已设置属性 ${name}`, "info", node.id); + return result; +}); + +register("Element_GetHtml", async (ctx, node, args) => { + const browser = await session(ctx); + const html = await requireMethod(browser, "getHtml")(selectorOf(args)); + ctx.log("已获取元素源码", "info", node.id); + return html; +}); + +register("Element_GetCheck", async (ctx, node, args) => { + const browser = await session(ctx); + const checked = await requireMethod(browser, "getChecked")(selectorOf(args)); + ctx.log(`勾选状态：${checked}`, "info", node.id); + return checked; +}); + +register("Element_SetCheck", async (ctx, node, args) => { + const browser = await session(ctx); + const checked = await requireMethod(browser, "setChecked")(selectorOf(args), enabled(args.isChecked)); + ctx.log(`已设置勾选状态：${checked}`, "info", node.id); + return checked; +}); + +register(["Element_CheckBoxOperation", "Element_CheckBoxOperation_v1", "Element_CheckBoxOperation_v2", "Element_CheckBoxOperation_v3"], async (ctx, node, args) => { + assertUnifiedBrowser(args); + const browser = await session(ctx); + const selector = selectorOf(args); + const operation = String(args.operation_type ?? "1"); + const modern = /_v[23]$/.test(node.componentId); + if ((modern && operation === "3") || (!modern && operation === "2")) { + return requireMethod(browser, "getChecked")(selector); + } + const checked = modern ? operation === "1" : enabled(args.isChecked); + return requireMethod(browser, "setChecked")(selector, checked); +}); + +register(["Element_GetButtonEnable", "Element_GetButtonEnable_v1", "Element_GetButtonEnable_v2", "Element_GetButtonEnable_v3", "Element_GetButtonEnable_v4"], async (ctx, node, args) => { + assertUnifiedBrowser(args); + const browser = await session(ctx); + const result = await requireMethod(browser, "isEnabled")(selectorOf(args), enabled(args.identify_hidden_element)); + ctx.log(`元素可点击：${result}`, "info", node.id); + return result; +}); + +register(["Element_ScrollIntoView", "Element_ScrollIntoView_v1", "Element_ScrollIntoView_v2"], async (ctx, node, args) => { + assertUnifiedBrowser(args); + const browser = await session(ctx); + const region = await requireMethod(browser, "scrollIntoView")(selectorOf(args)); + ctx.log("元素已滚动到可视区域", "info", node.id); + return region; +}); + +register(["Element_Region", "Element_Region2", "Element_Region2_v1", "Element_Region2_v2", "Element_Region2_v3"], async (ctx, node, args) => { + assertUnifiedBrowser(args); + const browser = await session(ctx); + const region = await requireMethod(browser, "getRegion")(selectorOf(args)); + ctx.log("已获取元素区域", "info", node.id); + return { ...region, position: String(args.position ?? "3") }; +}); + +register(["Element_MouseTo", "Element_MouseTo_V1", "Element_MouseTo_V2", "Element_MouseTo_V3", "Element_MouseTo_V4"], async (ctx, node, args) => { + assertUnifiedBrowser(args); + const browser = await session(ctx); + const region = await requireMethod(browser, "hover")(selectorOf(args)); + ctx.log("鼠标已移动到元素", "info", node.id); + return region; +}); + +register(["Element_Shot", "Element_Shot_V1", "Element_Shot_V2"], async (ctx, node, args) => { + assertUnifiedBrowser(args); + const browser = await session(ctx); + const folder = String(args.filePath ?? args.folder ?? ctx.evidenceDir); + const rawName = String(args.filename ?? `${node.id}.png`); + const filename = rawName.replace(/\$\{randomString\}/g, `${Date.now()}_${Math.random().toString(36).slice(2, 8)}`); + const target = path.resolve(folder, filename); + const captureBeyondViewport = node.componentId !== "Element_Shot_V2" || !enabled(args.isUseVerifyCode); + const result = await requireMethod(browser, "screenshotElement")(selectorOf(args), target, captureBeyondViewport); + ctx.log(`元素截图已保存：${result}`, "info", node.id); + return result; +}); diff --git a/src/main/capabilities/index.ts b/src/main/capabilities/index.ts index 22a492f..3e252dd 100644 --- a/src/main/capabilities/index.ts +++ b/src/main/capabilities/index.ts @@ -4,6 +4,7 @@ import "./http"; import "./data"; import "./browser"; import "./browser-extended"; +import "./browser-dom"; import "./office"; import "./desktop"; import "./pdf"; diff --git a/src/main/engine/types.ts b/src/main/engine/types.ts index 9e273b2..e890752 100644 --- a/src/main/engine/types.ts +++ b/src/main/engine/types.ts @@ -10,10 +10,35 @@ export interface BrowserLike { runJs(script: string): Promise; download(selector: unknown, folder: string, filename?: string, timeoutMs?: number): Promise; screenshot(path: string): Promise; + getAttribute?(selector: unknown, name: string): Promise; + setAttribute?(selector: unknown, name: string, value: string): Promise; + getHtml?(selector: unknown): Promise; + getChecked?(selector: unknown): Promise; + setChecked?(selector: unknown, checked: boolean): Promise; + isEnabled?(selector: unknown, includeHidden?: boolean): Promise; + scrollIntoView?(selector: unknown): Promise; + getRegion?(selector: unknown): Promise; + hover?(selector: unknown): Promise; + screenshotElement?(selector: unknown, path: string, captureBeyondViewport?: boolean): Promise; currentUrl(): Promise; close(): Promise; } +export interface BrowserElementRegion { + x: number; + y: number; + width: number; + height: number; + left: number; + top: number; + right: number; + bottom: number; + centerX: number; + centerY: number; + pageX: number; + pageY: number; +} + export interface RunContext { runId: string; flowId: string; diff --git a/src/main/fixture/server.test.ts b/src/main/fixture/server.test.ts new file mode 100644 index 0000000..57f8ab1 --- /dev/null +++ b/src/main/fixture/server.test.ts @@ -0,0 +1,19 @@ +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +vi.mock("electron", () => ({ app: { getAppPath: () => path.resolve(".") } })); + +import { startFixtureServer, stopFixtureServer } from "./server"; + +afterEach(async () => stopFixtureServer()); + +describe("local fixture server", () => { + it("serves the offline form and downloadable CSV", async () => { + const url = await startFixtureServer(0); + const html = await (await fetch(url)).text(); + expect(html).toContain("本地订单录入"); + expect(html).toContain('id="submit"'); + const csv = await (await fetch(new URL("/download.csv", url))).text(); + expect(csv).toContain("name,amount,status"); + }); +}); diff --git a/src/main/fixture/server.ts b/src/main/fixture/server.ts new file mode 100644 index 0000000..893a66b --- /dev/null +++ b/src/main/fixture/server.ts @@ -0,0 +1,69 @@ +import { app } from "electron"; +import fs from "node:fs"; +import http, { type Server } from "node:http"; +import path from "node:path"; + +let server: Server | null = null; +let currentUrl = ""; + +function fixtureRoot(): string { + const candidates = [ + process.resourcesPath ? path.join(process.resourcesPath, "fixture") : "", + path.join(app.getAppPath(), "resources", "fixture"), + path.join(__dirname, "../../resources/fixture") + ].filter(Boolean); + const found = candidates.find((candidate) => fs.existsSync(path.join(candidate, "index.html"))); + if (!found) throw new Error(`fixture index.html not found; tried ${candidates.join(", ")}`); + return found; +} + +export async function startFixtureServer(preferredPort = 17863): Promise { + if (server && currentUrl) return currentUrl; + const root = fixtureRoot(); + server = http.createServer((request, response) => { + const url = new URL(request.url || "/", "http://127.0.0.1"); + if (url.pathname === "/download.csv") { + response.writeHead(200, { "Content-Type": "text/csv; charset=utf-8", "Content-Disposition": "attachment; filename=orders.csv" }); + response.end("name,amount,status\r\nRPA,128,created\r\n"); + return; + } + if (url.pathname !== "/" && url.pathname !== "/index.html") { + response.writeHead(404).end("Not found"); + return; + } + response.writeHead(200, { "Content-Type": "text/html; charset=utf-8", "Cache-Control": "no-store" }); + fs.createReadStream(path.join(root, "index.html")).pipe(response); + }); + + const attempts = preferredPort === 0 ? [0] : Array.from({ length: 10 }, (_, index) => preferredPort + index); + for (const port of attempts) { + try { + await new Promise((resolve, reject) => { + const onError = (error: Error) => { server?.removeListener("listening", onListening); reject(error); }; + const onListening = () => { server?.removeListener("error", onError); resolve(); }; + server!.once("error", onError); + server!.once("listening", onListening); + server!.listen(port, "127.0.0.1"); + }); + const address = server.address(); + const activePort = typeof address === "object" && address ? address.port : port; + currentUrl = `http://127.0.0.1:${activePort}/`; + return currentUrl; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EADDRINUSE") throw error; + } + } + throw new Error("无法启动本地演示站：17863-17872 端口均被占用"); +} + +export async function stopFixtureServer(): Promise { + const active = server; + server = null; + currentUrl = ""; + if (!active) return; + await new Promise((resolve) => active.close(() => resolve())); +} + +export function fixtureUrl(): string { + return currentUrl; +} diff --git a/src/main/flows.test.ts b/src/main/flows.test.ts index 8811a05..8c31b85 100644 --- a/src/main/flows.test.ts +++ b/src/main/flows.test.ts @@ -11,7 +11,7 @@ vi.mock("electron", () => ({ } })); -import { createFlow, listFlows, saveFlow } from "./flows"; +import { createDemoFlow, createFlow, listFlows, saveFlow } from "./flows"; describe("flow summaries", () => { beforeEach(async () => { @@ -45,4 +45,14 @@ describe("flow summaries", () => { lastRunError: "测试失败原因" })); }); + + it("creates a connected local browser demo flow", () => { + const flow = createDemoFlow("http://127.0.0.1:17863/"); + expect(flow.name).toBe("本地网页订单演示"); + expect(flow.nodes.map((node) => node.componentId)).toEqual([ + "Trigger.Manual", "WebBrowser_CreateV3", "Element_SetValue_V6", "Element_SetValue_V6", "Element_Click_V6", "Element_GetText_V5" + ]); + expect(flow.edges).toHaveLength(5); + expect(flow.nodes.find((node) => node.id === "node_result")?.options.outputVar).toBe("订单结果"); + }); }); diff --git a/src/main/flows.ts b/src/main/flows.ts index 8f8f643..992d864 100644 --- a/src/main/flows.ts +++ b/src/main/flows.ts @@ -71,6 +71,27 @@ export function createFlow(name = "未命名流程"): FlowDefinition { }; } +export function createDemoFlow(url: string): FlowDefinition { + const flow = createFlow("本地网页订单演示"); + flow.description = "打开本地演示站，填写客户和金额，提交订单并读取结果。"; + flow.nodes = [ + flow.nodes[0], + { id: "node_open", componentId: "WebBrowser_CreateV3", position: { x: 180, y: 250 }, values: { type: "chrome", url, isRemote: "0", openNewTab: "1", waitingload: "1" }, options: { timeoutMs: 30000 } }, + { id: "node_name", componentId: "Element_SetValue_V6", position: { x: 180, y: 390 }, values: { uiSelector: "css:#name", designated_info: "0", designated_obj: "", text: "RPA 客户", input_type: "0", isEnter: "0", interval: "0" }, options: { timeoutMs: 10000 } }, + { id: "node_amount", componentId: "Element_SetValue_V6", position: { x: 180, y: 530 }, values: { uiSelector: "css:#amount", designated_info: "0", designated_obj: "", text: "128", input_type: "0", isEnter: "0", interval: "0" }, options: { timeoutMs: 10000 } }, + { id: "node_submit", componentId: "Element_Click_V6", position: { x: 180, y: 670 }, values: { uiSelector: "css:#submit", designated_info: "0", designated_obj: "", clickway: "0", clickfunc: "0", operation: "1" }, options: { timeoutMs: 10000 } }, + { id: "node_result", componentId: "Element_GetText_V5", position: { x: 180, y: 810 }, values: { choose: "0", uiSelector: "css:#result", cvSelector: "", designated_info: "0", designated_obj: "", attr_type: "default", attr_name: "" }, options: { timeoutMs: 10000, outputVar: "订单结果" } } + ]; + flow.edges = [ + { id: "edge_start_open", source: "node_start", target: "node_open" }, + { id: "edge_open_name", source: "node_open", target: "node_name" }, + { id: "edge_name_amount", source: "node_name", target: "node_amount" }, + { id: "edge_amount_submit", source: "node_amount", target: "node_submit" }, + { id: "edge_submit_result", source: "node_submit", target: "node_result" } + ]; + return flow; +} + export function listFlows(): FlowSummary[] { const base = dir(); const rows: FlowSummary[] = []; diff --git a/src/main/index.ts b/src/main/index.ts index 8952465..d0f1ae1 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -3,9 +3,10 @@ import path from "node:path"; import type { AppInfo, FlowDefinition } from "../shared/types"; import "./capabilities"; import { catalogPath, loadCatalog } from "./catalog"; -import { createFlow, deleteFlow, getFlow, listFlows, saveFlow } from "./flows"; +import { createDemoFlow, createFlow, deleteFlow, getFlow, listFlows, saveFlow } from "./flows"; import { capabilityCount, capabilityIds } from "./engine/registry"; import { closeBrowserSession, startRun, stopRun } from "./run/manager"; +import { fixtureUrl, startFixtureServer, stopFixtureServer } from "./fixture/server"; let mainWindow: BrowserWindow | null = null; @@ -57,7 +58,8 @@ function registerIpc(): void { node: process.versions.node, userData: app.getPath("userData"), catalogPath: catalogPath(), - packaged: app.isPackaged + packaged: app.isPackaged, + fixtureUrl: fixtureUrl() })); ipcMain.handle("catalog:get", () => loadCatalog()); @@ -69,6 +71,10 @@ function registerIpc(): void { const flow = createFlow(name); return saveFlow(flow); }); + ipcMain.handle("flows:create-demo", () => { + const url = fixtureUrl(); + return saveFlow(url ? createDemoFlow(url) : createFlow("未命名任务")); + }); ipcMain.handle("flows:get", (_event, id: string) => getFlow(id)); ipcMain.handle("flows:save", (_event, flow: FlowDefinition) => saveFlow(flow)); ipcMain.handle("flows:delete", (_event, id: string) => { @@ -107,7 +113,12 @@ if (!gotLock) { mainWindow.focus(); }); - app.whenReady().then(() => { + app.whenReady().then(async () => { + try { + await startFixtureServer(); + } catch (error) { + console.warn("local fixture server unavailable", error); + } registerIpc(); createWindow(); app.on("activate", () => { @@ -121,5 +132,6 @@ if (!gotLock) { app.on("before-quit", () => { void closeBrowserSession(); + void stopFixtureServer(); }); } diff --git a/src/preload/index.ts b/src/preload/index.ts index 70ca23d..71f139a 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -11,6 +11,7 @@ const api = { flows: { list: (): Promise => ipcRenderer.invoke("flows:list"), create: (name?: string): Promise => ipcRenderer.invoke("flows:create", name), + createDemo: (): Promise => ipcRenderer.invoke("flows:create-demo"), get: (id: string): Promise => ipcRenderer.invoke("flows:get", id), save: (flow: FlowDefinition): Promise => ipcRenderer.invoke("flows:save", flow), delete: (id: string): Promise => ipcRenderer.invoke("flows:delete", id) diff --git a/src/renderer/src/App.tsx b/src/renderer/src/App.tsx index 233471a..3653588 100644 --- a/src/renderer/src/App.tsx +++ b/src/renderer/src/App.tsx @@ -81,7 +81,7 @@ export function App() { setCatalog(loaded); let availableFlows = await window.rpa.flows.list(); if (!availableFlows.length) { - await window.rpa.flows.create("网页订单录入"); + await window.rpa.flows.createDemo(); availableFlows = await window.rpa.flows.list(); } if (!cancelled) setFlows(availableFlows); diff --git a/src/renderer/src/pages/SettingsPage.tsx b/src/renderer/src/pages/SettingsPage.tsx index 4e60c9e..9214b01 100644 --- a/src/renderer/src/pages/SettingsPage.tsx +++ b/src/renderer/src/pages/SettingsPage.tsx @@ -61,7 +61,7 @@ export function SettingsPage({ catalog }: { catalog: Catalog }) { {tab === "browser" && ( <> -
默认允许
127.0.0.1、localhost、example.com
扩展方式
环境变量 RPA_ALLOWED_DOMAINS
格式
逗号分隔域名，不含协议和路径
 +
默认允许
127.0.0.1、localhost、example.com
本地演示站
{info?.fixtureUrl || "未启动"}
扩展方式
环境变量 RPA_ALLOWED_DOMAINS
格式
逗号分隔域名，不含协议和路径

导航只允许 HTTP/HTTPS。未列入白名单的域名会返回 blocked_url；文件协议、脚本协议和任意本地路径不会由浏览器导航组件打开。

 )} diff --git a/src/renderer/src/validation/flowValidation.test.ts b/src/renderer/src/validation/flowValidation.test.ts index bb6218c..00876f0 100644 --- a/src/renderer/src/validation/flowValidation.test.ts +++ b/src/renderer/src/validation/flowValidation.test.ts @@ -34,4 +34,22 @@ describe("flow validation", () => { definition.edges.push({ id: "second", source: "start", target: "orphan" }); expect(validateFlow(definition, catalog)).toContainEqual(expect.objectContaining({ code: "multiple_normal_edges", nodeId: "start" })); }); + + it("does not require inactive designated browser fields or optional screenshot output", () => { + const conditionalCatalog = structuredClone(catalog); + conditionalCatalog.components.push({ + id: "Element.Input", name: "输入", kind: "vendor", group: "web", implemented: true, slots: [], + fields: [{ key: "designated_obj", label: "指定对象", required: true, widget: "text", defaultValue: "" }] + } as never, { + id: "Image_Screenshot", name: "截图", kind: "vendor", group: "vision", implemented: true, slots: [], + fields: [{ key: "image_path", label: "输出路径", required: true, widget: "file", defaultValue: "" }] + } as never); + const definition = flow(); + definition.nodes = [ + { id: "input", componentId: "Element.Input", position: { x: 0, y: 0 }, values: { designated_info: "0" }, options: {} }, + { id: "shot", componentId: "Image_Screenshot", position: { x: 0, y: 100 }, values: {}, options: {} } + ]; + definition.edges = []; + expect(validateFlow(definition, conditionalCatalog).filter((issue) => issue.code === "required")).toEqual([]); + }); }); diff --git a/src/renderer/src/validation/flowValidation.ts b/src/renderer/src/validation/flowValidation.ts index 427b63b..cf22491 100644 --- a/src/renderer/src/validation/flowValidation.ts +++ b/src/renderer/src/validation/flowValidation.ts @@ -11,6 +11,17 @@ function empty(value: unknown): boolean { return value === undefined || value === null || (typeof value === "string" && value.trim() === ""); } +function truthy(value: unknown): boolean { + return [true, 1, "1", "true", "True", "是", "开启"].includes(value as never); +} + +function requiredInContext(componentId: string, key: string, values: Record): boolean { + if (key === "designated_obj" && !truthy(values.designated_info)) return false; + if (key === "cvSelector" && String(values.choose ?? "0") !== "1") return false; + if (componentId === "Image_Screenshot" && ["image_path", "outImagePath", "outImageName"].includes(key)) return false; + return true; +} + export function validateFlow(flow: FlowDefinition, catalog: Catalog): FlowValidationIssue[] { const issues: FlowValidationIssue[] = []; const componentMap = new Map(catalog.components.map((component) => [component.id, component])); @@ -29,6 +40,7 @@ export function validateFlow(flow: FlowDefinition, catalog: Catalog): FlowValida issues.push({ severity: "error", code: "not_implemented", nodeId: node.id, message: `「${node.label || component.name}」尚未接入执行器` }); } for (const field of component.fields.filter((item) => item.required && item.widget !== "hidden" && item.widget !== "region")) { + if (!requiredInContext(component.id, field.key, node.values)) continue; const current = Object.prototype.hasOwnProperty.call(node.values, field.key) ? node.values[field.key] : field.defaultValue; if (empty(current)) issues.push({ severity: "error", code: "required", nodeId: node.id, message: `「${node.label || component.name}」缺少必填参数：${field.label}` }); } diff --git a/src/shared/types.ts b/src/shared/types.ts index f7d139d..5292fbc 100644 --- a/src/shared/types.ts +++ b/src/shared/types.ts @@ -166,6 +166,7 @@ export interface AppInfo { userData: string; catalogPath: string; packaged: boolean; + fixtureUrl?: string; } export interface RunNodeState {