feat: 增加本地演示流程与网页DOM能力

This commit is contained in:
yu 2026-10-27 13:50:03 +08:00
parent d4799a4285
commit 6adf772987
19 changed files with 642 additions and 9 deletions

View File

@ -50,6 +50,7 @@ docs 现状分析、实施方案
## 边界
- 1016 项目录中当前有 363 项通过能力注册表或运行时容器标记为可执行,覆盖控制/触发、字符串/集合/INI/二维表格、文件、浏览器元素常用变体、HTTP、Excel/CSV与表格辅助、系统路径与环境变量、进程管理、桌面键鼠/窗口对象/剪贴板/对话框、PDF、Word、邮件发送、屏幕截图、离线图像查找与点击;其余灰标「待接入」。运行前检查会提前定位待接入组件、必填参数、多出口与断链节点。
- 1016 项目录中当前有 398 项通过能力注册表或运行时容器标记为可执行,覆盖控制/触发、字符串/集合/INI/二维表格、文件、浏览器元素常用及 DOM 高级能力、HTTP、Excel/CSV与表格辅助、系统路径与环境变量、进程管理、桌面键鼠/窗口对象/剪贴板/对话框、PDF、Word、邮件发送、屏幕截图、离线图像查找与点击;其余灰标「待接入」。运行前检查会提前定位待接入组件、必填参数、多出口与断链节点。
- 内置本地演示站默认监听 `127.0.0.1:17863`;新用户首次启动会获得“本地网页订单演示”任务,可离线验证网页输入、点击和文本读取。
- 不包含厂商代码、二进制、模型或前端资产;目录是从厂商元数据派生的静态定义。
- 运行期不访问云服务;HTTP/邮件属于用户显式配置的外部系统。

View File

@ -4,7 +4,7 @@
> 依据:`D:\code\Agent-Employee`(只读)、`C:\rpa-demo`(只读)、2026-10-26 需求确认
> 本方案是执行基线:范围、契约、排期、验收、降级顺序都在这里;实施中变更必须回写本文件。
> **2026-10-27 当前状态**:计划点名的代表能力已全部进入注册表或运行时容器,覆盖控制/触发、字符串/集合/INI/二维表格、文件、浏览器元素常用变体、HTTP、Excel/CSV与表格辅助、系统路径与环境变量、进程管理、桌面键鼠/窗口对象/剪贴板/对话框、PDF、Word、邮件发送、屏幕截图、离线图像查找与点击;启动后目录显示 **363 项可执行 / 1016 项总目录**。当前 13 个测试文件共 89 条测试,覆盖计划能力审计、浏览器参数路由与变体、Excel/PDF/Word/SMTP/模板匹配、文件与数据扩展、系统/进程/窗口解析、直线/IF/循环/重试/跳过、主出口与异常出口语义、运行记录回写和运行前检查。设置页可查看分类覆盖和处理器 ID 数量。未通过实际场景验证的变体仍不计入最终交付数量。
> **2026-10-27 当前状态**:计划点名的代表能力已全部进入注册表或运行时容器,覆盖控制/触发、字符串/集合/INI/二维表格、文件、浏览器元素常用及 DOM 高级能力、HTTP、Excel/CSV与表格辅助、系统路径与环境变量、进程管理、桌面键鼠/窗口对象/剪贴板/对话框、PDF、Word、邮件发送、屏幕截图、离线图像查找与点击;启动后目录显示 **398 项可执行 / 1016 项总目录**。当前 15 个测试文件共 99 条测试,覆盖计划能力审计、浏览器参数路由/变体/DOM、Excel/PDF/Word/SMTP/模板匹配、文件与数据扩展、系统/进程/窗口解析、直线/IF/循环/重试/跳过、主出口与异常出口语义、运行记录回写、运行前检查、本地演示站与示例流程。设置页可查看分类覆盖和处理器 ID 数量。未通过实际场景验证的变体仍不计入最终交付数量。
---

View File

@ -0,0 +1,34 @@
<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<title>RPA Studio 本地订单夹具</title>
<style>
*{box-sizing:border-box}body{margin:0;background:#f3f6f9;color:#172033;font-family:"Microsoft YaHei UI",sans-serif}main{width:min(760px,calc(100% - 40px));margin:48px auto;background:#fff;border:1px solid #e1e7ee;border-radius:14px;padding:28px;box-shadow:0 10px 35px rgba(23,32,51,.08)}h1{font-size:22px;margin:0 0 6px}p{color:#788494;font-size:13px;margin:0 0 24px}.grid{display:grid;grid-template-columns:1fr 1fr;gap:16px}label{display:flex;flex-direction:column;gap:7px;font-size:12px;font-weight:600}input{height:40px;border:1px solid #dbe3eb;border-radius:8px;padding:0 11px;font:inherit}button,a{height:40px;border:0;border-radius:8px;padding:0 16px;display:inline-flex;align-items:center;justify-content:center;font:inherit;text-decoration:none;cursor:pointer}button{background:#2264e5;color:#fff}.actions{margin-top:18px;display:flex;gap:9px}.secondary{background:#f2f5f8;color:#4e5a6b}.result{margin-top:22px;padding:14px;border-radius:9px;background:#f5f7fa;min-height:48px;color:#4e5a6b}.result.success{background:#e9f7ee;color:#067647}table{width:100%;margin-top:20px;border-collapse:collapse;font-size:12px}th,td{padding:10px;border-bottom:1px solid #edf1f5;text-align:left}
</style>
</head>
<body>
<main>
<h1>本地订单录入</h1>
<p>该页面由 RPA Studio 在本机提供,不访问互联网,用于验证输入、点击、读取和下载。</p>
<div class="grid">
<label>客户名称<input id="name" placeholder="请输入客户名称"></label>
<label>订单金额<input id="amount" type="number" placeholder="请输入金额"></label>
</div>
<div class="actions"><button id="submit" type="button">创建订单</button><a class="secondary" id="download" href="/download.csv">下载示例 CSV</a></div>
<div class="result" id="result">等待提交</div>
<table><thead><tr><th>客户</th><th>金额</th><th>状态</th></tr></thead><tbody id="rows"></tbody></table>
</main>
<script>
document.querySelector('#submit').addEventListener('click',()=>{
const name=document.querySelector('#name').value.trim();
const amount=document.querySelector('#amount').value.trim();
const result=document.querySelector('#result');
if(!name||!amount){result.className='result';result.textContent='请完整填写客户名称和订单金额';return;}
result.className='result success';result.textContent=`订单创建成功:${name},金额 ${amount}`;
const row=document.createElement('tr');row.innerHTML=`<td>${name}</td><td>${amount}</td><td>已创建</td>`;document.querySelector('#rows').appendChild(row);
});
</script>
</body>
</html>

View File

@ -1,7 +1,7 @@
import { BrowserWindow } from "electron";
import { promises as fs } from "node:fs";
import path from "node:path";
import { RuntimeError, type BrowserLike } from "../engine/types";
import { RuntimeError, type BrowserElementRegion, type BrowserLike } from "../engine/types";
const DEFAULT_DOMAINS = ["127.0.0.1", "localhost", "example.com"];
@ -142,6 +142,53 @@ export class BrowserSession implements BrowserLike {
return result.result?.value as T;
}
private async withElement<T>(selector: unknown, operation: string): Promise<T> {
const spec = selectorSpec(selector);
const result = await this.evaluate<{ found: boolean; value?: T }>(`(() => {
const spec = ${JSON.stringify(spec)};
const find = () => {
if (spec.kind === 'text') {
const target = spec.value;
const matches = [...document.querySelectorAll('*')].filter((node) => {
const text = (node.innerText || node.textContent || '').trim();
return text === target || (text.includes(target) && text.length <= target.length + 20);
});
return matches.sort((a, b) => (a.innerText || '').length - (b.innerText || '').length)[0] || null;
}
if (spec.kind === 'aria') {
const role = spec.role || spec.value;
const name = spec.name || '';
return [...document.querySelectorAll('[role="' + role + '"], ' + role)].find((node) => {
const accessibleName = node.getAttribute('aria-label') || node.innerText || '';
return !name || accessibleName.includes(name);
}) || null;
}
return document.querySelector(spec.value);
};
const el = find();
if (!el) return { found: false };
return { found: true, value: (() => { ${operation} })() };
})()`);
if (!result.found) {
throw new RuntimeError("not_found", `Element not found: ${spec.kind}:${spec.value}`);
}
return result.value as T;
}
private regionOperation(): string {
return `
const rect = el.getBoundingClientRect();
const x = rect.x;
const y = rect.y;
return {
x, y, width: rect.width, height: rect.height,
left: rect.left, top: rect.top, right: rect.right, bottom: rect.bottom,
centerX: x + rect.width / 2, centerY: y + rect.height / 2,
pageX: x + window.scrollX, pageY: y + window.scrollY
};
`;
}
async navigate(url: string): Promise<{ url: string; title: string }> {
const target = assertUrl(url);
await this.cdp("Page.enable");
@ -318,6 +365,89 @@ export class BrowserSession implements BrowserLike {
return target;
}
async getAttribute(selector: unknown, name: string): Promise<string> {
return this.withElement<string>(selector, `return el.getAttribute(${JSON.stringify(name)}) ?? '';`);
}
async setAttribute(selector: unknown, name: string, value: string): Promise<unknown> {
return this.withElement(selector, `
el.setAttribute(${JSON.stringify(name)}, ${JSON.stringify(value)});
return { name: ${JSON.stringify(name)}, value: el.getAttribute(${JSON.stringify(name)}) ?? '' };
`);
}
async getHtml(selector: unknown): Promise<string> {
return this.withElement<string>(selector, "return el.outerHTML;");
}
async getChecked(selector: unknown): Promise<boolean> {
return this.withElement<boolean>(selector, `
if (!('checked' in el)) throw new Error('The target element has no checked state');
return Boolean(el.checked);
`);
}
async setChecked(selector: unknown, checked: boolean): Promise<boolean> {
return this.withElement<boolean>(selector, `
if (!('checked' in el)) throw new Error('The target element has no checked state');
const next = ${checked ? "true" : "false"};
if (Boolean(el.checked) !== next) {
el.checked = next;
el.dispatchEvent(new Event('input', { bubbles: true }));
el.dispatchEvent(new Event('change', { bubbles: true }));
}
return Boolean(el.checked);
`);
}
async isEnabled(selector: unknown, includeHidden = false): Promise<boolean> {
return this.withElement<boolean>(selector, `
const style = getComputedStyle(el);
const rect = el.getBoundingClientRect();
const visible = rect.width > 0 && rect.height > 0 && style.display !== 'none' &&
style.visibility !== 'hidden' && Number(style.opacity) !== 0;
const enabled = !Boolean(el.disabled) && el.getAttribute('aria-disabled') !== 'true' && style.pointerEvents !== 'none';
return enabled && (${includeHidden ? "true" : "false"} || visible);
`);
}
async scrollIntoView(selector: unknown): Promise<BrowserElementRegion> {
return this.withElement<BrowserElementRegion>(selector, `
el.scrollIntoView({ block: 'center', inline: 'center' });
${this.regionOperation()}
`);
}
async getRegion(selector: unknown): Promise<BrowserElementRegion> {
return this.withElement<BrowserElementRegion>(selector, this.regionOperation());
}
async hover(selector: unknown): Promise<BrowserElementRegion> {
const region = await this.scrollIntoView(selector);
await this.cdp("Input.dispatchMouseEvent", {
type: "mouseMoved", x: Math.round(region.centerX), y: Math.round(region.centerY), button: "none"
});
await new Promise((resolve) => setTimeout(resolve, 120));
return region;
}
async screenshotElement(selector: unknown, filePath: string, captureBeyondViewport = true): Promise<string> {
const region = await this.scrollIntoView(selector);
if (region.width <= 0 || region.height <= 0) {
throw new RuntimeError("not_found", "The target element has no screenshot area");
}
await this.cdp("Page.enable");
const result = await this.cdp<{ data: string }>("Page.captureScreenshot", {
format: "png",
captureBeyondViewport,
clip: { x: region.pageX, y: region.pageY, width: region.width, height: region.height, scale: 1 }
});
const target = path.resolve(filePath);
await fs.mkdir(path.dirname(target), { recursive: true });
await fs.writeFile(target, Buffer.from(result.data, "base64"));
return target;
}
async currentUrl(): Promise<string> {
return this.win.webContents.getURL();
}

View File

@ -0,0 +1,125 @@
import { readFileSync } from "node:fs";
import { resolve } from "node:path";
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { CatalogComponent, FlowNode } from "../../shared/types";
import { getHandler } from "../engine/registry";
import { RuntimeError, type BrowserElementRegion, type BrowserLike, type RunContext } from "../engine/types";
import { BROWSER_DOM_IDS } from "./browser-dom";
const component = {} as CatalogComponent;
const selector = { kind: "css", value: "#target" };
const region: BrowserElementRegion = {
x: 10, y: 20, width: 100, height: 40,
left: 10, top: 20, right: 110, bottom: 60,
centerX: 60, centerY: 40, pageX: 10, pageY: 120
};
function node(componentId: string): FlowNode {
return { id: `dom_${componentId}`, componentId, position: { x: 0, y: 0 }, values: {}, options: {} };
}
function fakeBrowser(): BrowserLike {
return {
navigate: vi.fn(async (url: string) => ({ url, title: "Fake page" })),
click: vi.fn(), type: vi.fn(), getText: vi.fn(), exists: vi.fn(), select: vi.fn(), runJs: vi.fn(),
download: vi.fn(), screenshot: vi.fn(), currentUrl: vi.fn(), close: vi.fn(),
getAttribute: vi.fn(async () => "ready"),
setAttribute: vi.fn(async (_selector, name, value) => ({ name, value })),
getHtml: vi.fn(async () => '<button id="target">Run</button>'),
getChecked: vi.fn(async () => true),
setChecked: vi.fn(async (_selector, checked) => checked),
isEnabled: vi.fn(async () => true),
scrollIntoView: vi.fn(async () => region),
getRegion: vi.fn(async () => region),
hover: vi.fn(async () => region),
screenshotElement: vi.fn(async (_selector, target) => target)
};
}
function context(browser: BrowserLike): RunContext {
return {
runId: "dom-run", flowId: "dom-flow", flow: {} as RunContext["flow"], vars: {}, secrets: {},
signal: new AbortController().signal, evidenceDir: "C:/temp/evidence", browser,
emit: vi.fn(), log: vi.fn(), componentOf: vi.fn()
};
}
async function invoke(ctx: RunContext, componentId: string, args: Record<string, unknown>): Promise<unknown> {
const handler = getHandler(componentId);
expect(handler, `${componentId} should be registered`).toBeTypeOf("function");
return handler!(ctx, node(componentId), args, component);
}
describe("browser DOM component registration", () => {
it("registers real catalog IDs without Ground variants", () => {
const catalog = JSON.parse(readFileSync(resolve("resources/catalog.json"), "utf8")) as {
components: Array<{ id: string }>;
};
const catalogIds = new Set(catalog.components.map((item) => item.id));
expect(BROWSER_DOM_IDS.length).toBeGreaterThanOrEqual(12);
expect(BROWSER_DOM_IDS.some((id) => id.includes("Ground"))).toBe(false);
expect(BROWSER_DOM_IDS.filter((id) => !catalogIds.has(id))).toEqual([]);
expect(BROWSER_DOM_IDS.every((id) => typeof getHandler(id) === "function")).toBe(true);
});
});
describe("browser DOM capability routing", () => {
let browser: BrowserLike;
let ctx: RunContext;
beforeEach(() => {
browser = fakeBrowser();
ctx = context(browser);
});
it("gets and sets attributes and HTML", async () => {
await expect(invoke(ctx, "Element_GetAttribute_v3", { uiSelector: selector, attrName: "data-state" })).resolves.toBe("ready");
expect(browser.getAttribute).toHaveBeenCalledWith(selector, "data-state");
await invoke(ctx, "Element_SetAttribute_v2", { uiSelector: selector, attrName: "role", attrValue: "switch" });
expect(browser.setAttribute).toHaveBeenCalledWith(selector, "role", "switch");
await expect(invoke(ctx, "Element_GetHtml", { selector })).resolves.toContain("button");
});
it("gets, sets and operates checkbox states", async () => {
await expect(invoke(ctx, "Element_GetCheck", { uiSelector: selector })).resolves.toBe(true);
await expect(invoke(ctx, "Element_SetCheck", { uiSelector: selector, isChecked: "0" })).resolves.toBe(false);
expect(browser.setChecked).toHaveBeenCalledWith(selector, false);
await invoke(ctx, "Element_CheckBoxOperation_v3", { uiSelector: selector, operation_type: "1" });
expect(browser.setChecked).toHaveBeenLastCalledWith(selector, true);
await invoke(ctx, "Element_CheckBoxOperation_v3", { uiSelector: selector, operation_type: "2" });
expect(browser.setChecked).toHaveBeenLastCalledWith(selector, false);
await expect(invoke(ctx, "Element_CheckBoxOperation_v3", { uiSelector: selector, operation_type: "3" })).resolves.toBe(true);
});
it("checks enabled state and honors hidden-element mode", async () => {
await expect(invoke(ctx, "Element_GetButtonEnable_v3", {
uiSelector: selector, identify_hidden_element: "1"
})).resolves.toBe(true);
expect(browser.isEnabled).toHaveBeenCalledWith(selector, true);
});
it("scrolls, reads regions and hovers", async () => {
await expect(invoke(ctx, "Element_ScrollIntoView_v2", { uiSelector: selector })).resolves.toEqual(region);
await expect(invoke(ctx, "Element_Region2_v3", { uiSelector: selector, position: "3" })).resolves.toEqual({ ...region, position: "3" });
await expect(invoke(ctx, "Element_MouseTo_V4", { uiSelector: selector })).resolves.toEqual(region);
expect(browser.scrollIntoView).toHaveBeenCalledWith(selector);
expect(browser.getRegion).toHaveBeenCalledWith(selector);
expect(browser.hover).toHaveBeenCalledWith(selector);
});
it("captures element screenshots with viewport semantics", async () => {
const result = await invoke(ctx, "Element_Shot_V2", {
uiSelector: selector, filePath: "C:/temp/evidence", filename: "target.png", isUseVerifyCode: "1"
});
expect(result).toMatch(/target\.png$/);
expect(browser.screenshotElement).toHaveBeenCalledWith(selector, expect.stringMatching(/target\.png$/), false);
});
it("rejects missing selectors, attributes and designated browser objects", async () => {
await expect(invoke(ctx, "Element_GetHtml", {})).rejects.toBeInstanceOf(RuntimeError);
await expect(invoke(ctx, "Element_GetAttribute", { uiSelector: selector })).rejects.toBeInstanceOf(RuntimeError);
await expect(invoke(ctx, "Element_SetAttribute_v2", {
uiSelector: selector, attrName: "role", attrValue: "button", designated_info: "1"
})).rejects.toMatchObject({ code: "validation_error" });
});
});

View File

@ -0,0 +1,154 @@
import path from "node:path";
import { browserSession } from "../browser/session";
import { register } from "../engine/registry";
import { RuntimeError, type BrowserLike, type RunContext } from "../engine/types";
export const BROWSER_DOM_IDS = [
"Element_GetAttribute", "Element_GetAttribute_v1", "Element_GetAttribute_v2", "Element_GetAttribute_v3",
"Element_SetAttribute", "Element_SetAttribute_v1", "Element_SetAttribute_v2", "Element_GetHtml",
"Element_GetCheck", "Element_SetCheck", "Element_CheckBoxOperation", "Element_CheckBoxOperation_v1",
"Element_CheckBoxOperation_v2", "Element_CheckBoxOperation_v3", "Element_GetButtonEnable",
"Element_GetButtonEnable_v1", "Element_GetButtonEnable_v2", "Element_GetButtonEnable_v3",
"Element_GetButtonEnable_v4", "Element_ScrollIntoView", "Element_ScrollIntoView_v1",
"Element_ScrollIntoView_v2", "Element_Region", "Element_Region2", "Element_Region2_v1",
"Element_Region2_v2", "Element_Region2_v3", "Element_MouseTo", "Element_MouseTo_V1",
"Element_MouseTo_V2", "Element_MouseTo_V3", "Element_MouseTo_V4", "Element_Shot",
"Element_Shot_V1", "Element_Shot_V2"
] as const;
async function session(ctx: RunContext): Promise<BrowserLike> {
if (!ctx.browser) ctx.browser = await browserSession();
return ctx.browser;
}
function selectorOf(args: Record<string, unknown>): unknown {
const selector = args.uiSelector ?? args.selector ?? args.target;
if (selector === undefined || selector === null || selector === "") {
throw new RuntimeError("validation_error", "元素选择器为空", { recommendation: "请拾取或填写网页元素" });
}
return selector;
}
function enabled(value: unknown): boolean {
return value === true || value === 1 || String(value).toLowerCase() === "true" || String(value) === "1";
}
function requiredText(args: Record<string, unknown>, keys: string[], label: string): string {
for (const key of keys) {
const value = args[key];
if (value !== undefined && value !== null && String(value).trim()) return String(value);
}
throw new RuntimeError("validation_error", `${label}为空`);
}
function assertUnifiedBrowser(args: Record<string, unknown>): void {
if (enabled(args.designated_info)) {
throw new RuntimeError("validation_error", "当前能力不支持指定其他网页或窗口对象", {
recommendation: "关闭指定网页/窗口对象,使用当前运行浏览器"
});
}
}
function requireMethod<K extends keyof BrowserLike>(browser: BrowserLike, key: K): NonNullable<BrowserLike[K]> {
const method = browser[key];
if (typeof method !== "function") throw new RuntimeError("browser_error", `浏览器不支持 DOM 方法:${String(key)}`);
return method.bind(browser) as NonNullable<BrowserLike[K]>;
}
register(["Element_GetAttribute", "Element_GetAttribute_v1", "Element_GetAttribute_v2", "Element_GetAttribute_v3"], async (ctx, node, args) => {
assertUnifiedBrowser(args);
const browser = await session(ctx);
const name = requiredText(args, ["attrName", "attribute", "name"], "属性名");
const value = await requireMethod(browser, "getAttribute")(selectorOf(args), name);
ctx.log(`已获取属性 ${name}`, "info", node.id);
return value;
});
register(["Element_SetAttribute", "Element_SetAttribute_v1", "Element_SetAttribute_v2"], async (ctx, node, args) => {
assertUnifiedBrowser(args);
const browser = await session(ctx);
const name = requiredText(args, ["attrName", "attribute", "name"], "属性名");
const value = String(args.attrValue ?? args.value ?? "");
const result = await requireMethod(browser, "setAttribute")(selectorOf(args), name, value);
ctx.log(`已设置属性 ${name}`, "info", node.id);
return result;
});
register("Element_GetHtml", async (ctx, node, args) => {
const browser = await session(ctx);
const html = await requireMethod(browser, "getHtml")(selectorOf(args));
ctx.log("已获取元素源码", "info", node.id);
return html;
});
register("Element_GetCheck", async (ctx, node, args) => {
const browser = await session(ctx);
const checked = await requireMethod(browser, "getChecked")(selectorOf(args));
ctx.log(`勾选状态:${checked}`, "info", node.id);
return checked;
});
register("Element_SetCheck", async (ctx, node, args) => {
const browser = await session(ctx);
const checked = await requireMethod(browser, "setChecked")(selectorOf(args), enabled(args.isChecked));
ctx.log(`已设置勾选状态:${checked}`, "info", node.id);
return checked;
});
register(["Element_CheckBoxOperation", "Element_CheckBoxOperation_v1", "Element_CheckBoxOperation_v2", "Element_CheckBoxOperation_v3"], async (ctx, node, args) => {
assertUnifiedBrowser(args);
const browser = await session(ctx);
const selector = selectorOf(args);
const operation = String(args.operation_type ?? "1");
const modern = /_v[23]$/.test(node.componentId);
if ((modern && operation === "3") || (!modern && operation === "2")) {
return requireMethod(browser, "getChecked")(selector);
}
const checked = modern ? operation === "1" : enabled(args.isChecked);
return requireMethod(browser, "setChecked")(selector, checked);
});
register(["Element_GetButtonEnable", "Element_GetButtonEnable_v1", "Element_GetButtonEnable_v2", "Element_GetButtonEnable_v3", "Element_GetButtonEnable_v4"], async (ctx, node, args) => {
assertUnifiedBrowser(args);
const browser = await session(ctx);
const result = await requireMethod(browser, "isEnabled")(selectorOf(args), enabled(args.identify_hidden_element));
ctx.log(`元素可点击:${result}`, "info", node.id);
return result;
});
register(["Element_ScrollIntoView", "Element_ScrollIntoView_v1", "Element_ScrollIntoView_v2"], async (ctx, node, args) => {
assertUnifiedBrowser(args);
const browser = await session(ctx);
const region = await requireMethod(browser, "scrollIntoView")(selectorOf(args));
ctx.log("元素已滚动到可视区域", "info", node.id);
return region;
});
register(["Element_Region", "Element_Region2", "Element_Region2_v1", "Element_Region2_v2", "Element_Region2_v3"], async (ctx, node, args) => {
assertUnifiedBrowser(args);
const browser = await session(ctx);
const region = await requireMethod(browser, "getRegion")(selectorOf(args));
ctx.log("已获取元素区域", "info", node.id);
return { ...region, position: String(args.position ?? "3") };
});
register(["Element_MouseTo", "Element_MouseTo_V1", "Element_MouseTo_V2", "Element_MouseTo_V3", "Element_MouseTo_V4"], async (ctx, node, args) => {
assertUnifiedBrowser(args);
const browser = await session(ctx);
const region = await requireMethod(browser, "hover")(selectorOf(args));
ctx.log("鼠标已移动到元素", "info", node.id);
return region;
});
register(["Element_Shot", "Element_Shot_V1", "Element_Shot_V2"], async (ctx, node, args) => {
assertUnifiedBrowser(args);
const browser = await session(ctx);
const folder = String(args.filePath ?? args.folder ?? ctx.evidenceDir);
const rawName = String(args.filename ?? `${node.id}.png`);
const filename = rawName.replace(/\$\{randomString\}/g, `${Date.now()}_${Math.random().toString(36).slice(2, 8)}`);
const target = path.resolve(folder, filename);
const captureBeyondViewport = node.componentId !== "Element_Shot_V2" || !enabled(args.isUseVerifyCode);
const result = await requireMethod(browser, "screenshotElement")(selectorOf(args), target, captureBeyondViewport);
ctx.log(`元素截图已保存:${result}`, "info", node.id);
return result;
});

View File

@ -4,6 +4,7 @@ import "./http";
import "./data";
import "./browser";
import "./browser-extended";
import "./browser-dom";
import "./office";
import "./desktop";
import "./pdf";

View File

@ -10,10 +10,35 @@ export interface BrowserLike {
runJs(script: string): Promise<unknown>;
download(selector: unknown, folder: string, filename?: string, timeoutMs?: number): Promise<string>;
screenshot(path: string): Promise<string>;
getAttribute?(selector: unknown, name: string): Promise<string>;
setAttribute?(selector: unknown, name: string, value: string): Promise<unknown>;
getHtml?(selector: unknown): Promise<string>;
getChecked?(selector: unknown): Promise<boolean>;
setChecked?(selector: unknown, checked: boolean): Promise<boolean>;
isEnabled?(selector: unknown, includeHidden?: boolean): Promise<boolean>;
scrollIntoView?(selector: unknown): Promise<BrowserElementRegion>;
getRegion?(selector: unknown): Promise<BrowserElementRegion>;
hover?(selector: unknown): Promise<BrowserElementRegion>;
screenshotElement?(selector: unknown, path: string, captureBeyondViewport?: boolean): Promise<string>;
currentUrl(): Promise<string>;
close(): Promise<void>;
}
export interface BrowserElementRegion {
x: number;
y: number;
width: number;
height: number;
left: number;
top: number;
right: number;
bottom: number;
centerX: number;
centerY: number;
pageX: number;
pageY: number;
}
export interface RunContext {
runId: string;
flowId: string;

View File

@ -0,0 +1,19 @@
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
vi.mock("electron", () => ({ app: { getAppPath: () => path.resolve(".") } }));
import { startFixtureServer, stopFixtureServer } from "./server";
afterEach(async () => stopFixtureServer());
describe("local fixture server", () => {
it("serves the offline form and downloadable CSV", async () => {
const url = await startFixtureServer(0);
const html = await (await fetch(url)).text();
expect(html).toContain("本地订单录入");
expect(html).toContain('id="submit"');
const csv = await (await fetch(new URL("/download.csv", url))).text();
expect(csv).toContain("name,amount,status");
});
});

View File

@ -0,0 +1,69 @@
import { app } from "electron";
import fs from "node:fs";
import http, { type Server } from "node:http";
import path from "node:path";
let server: Server | null = null;
let currentUrl = "";
function fixtureRoot(): string {
const candidates = [
process.resourcesPath ? path.join(process.resourcesPath, "fixture") : "",
path.join(app.getAppPath(), "resources", "fixture"),
path.join(__dirname, "../../resources/fixture")
].filter(Boolean);
const found = candidates.find((candidate) => fs.existsSync(path.join(candidate, "index.html")));
if (!found) throw new Error(`fixture index.html not found; tried ${candidates.join(", ")}`);
return found;
}
export async function startFixtureServer(preferredPort = 17863): Promise<string> {
if (server && currentUrl) return currentUrl;
const root = fixtureRoot();
server = http.createServer((request, response) => {
const url = new URL(request.url || "/", "http://127.0.0.1");
if (url.pathname === "/download.csv") {
response.writeHead(200, { "Content-Type": "text/csv; charset=utf-8", "Content-Disposition": "attachment; filename=orders.csv" });
response.end("name,amount,status\r\nRPA,128,created\r\n");
return;
}
if (url.pathname !== "/" && url.pathname !== "/index.html") {
response.writeHead(404).end("Not found");
return;
}
response.writeHead(200, { "Content-Type": "text/html; charset=utf-8", "Cache-Control": "no-store" });
fs.createReadStream(path.join(root, "index.html")).pipe(response);
});
const attempts = preferredPort === 0 ? [0] : Array.from({ length: 10 }, (_, index) => preferredPort + index);
for (const port of attempts) {
try {
await new Promise<void>((resolve, reject) => {
const onError = (error: Error) => { server?.removeListener("listening", onListening); reject(error); };
const onListening = () => { server?.removeListener("error", onError); resolve(); };
server!.once("error", onError);
server!.once("listening", onListening);
server!.listen(port, "127.0.0.1");
});
const address = server.address();
const activePort = typeof address === "object" && address ? address.port : port;
currentUrl = `http://127.0.0.1:${activePort}/`;
return currentUrl;
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "EADDRINUSE") throw error;
}
}
throw new Error("无法启动本地演示站:17863-17872 端口均被占用");
}
export async function stopFixtureServer(): Promise<void> {
const active = server;
server = null;
currentUrl = "";
if (!active) return;
await new Promise<void>((resolve) => active.close(() => resolve()));
}
export function fixtureUrl(): string {
return currentUrl;
}

View File

@ -11,7 +11,7 @@ vi.mock("electron", () => ({
}
}));
import { createFlow, listFlows, saveFlow } from "./flows";
import { createDemoFlow, createFlow, listFlows, saveFlow } from "./flows";
describe("flow summaries", () => {
beforeEach(async () => {
@ -45,4 +45,14 @@ describe("flow summaries", () => {
lastRunError: "测试失败原因"
}));
});
it("creates a connected local browser demo flow", () => {
const flow = createDemoFlow("http://127.0.0.1:17863/");
expect(flow.name).toBe("本地网页订单演示");
expect(flow.nodes.map((node) => node.componentId)).toEqual([
"Trigger.Manual", "WebBrowser_CreateV3", "Element_SetValue_V6", "Element_SetValue_V6", "Element_Click_V6", "Element_GetText_V5"
]);
expect(flow.edges).toHaveLength(5);
expect(flow.nodes.find((node) => node.id === "node_result")?.options.outputVar).toBe("订单结果");
});
});

View File

@ -71,6 +71,27 @@ export function createFlow(name = "未命名流程"): FlowDefinition {
};
}
export function createDemoFlow(url: string): FlowDefinition {
const flow = createFlow("本地网页订单演示");
flow.description = "打开本地演示站,填写客户和金额,提交订单并读取结果。";
flow.nodes = [
flow.nodes[0],
{ id: "node_open", componentId: "WebBrowser_CreateV3", position: { x: 180, y: 250 }, values: { type: "chrome", url, isRemote: "0", openNewTab: "1", waitingload: "1" }, options: { timeoutMs: 30000 } },
{ id: "node_name", componentId: "Element_SetValue_V6", position: { x: 180, y: 390 }, values: { uiSelector: "css:#name", designated_info: "0", designated_obj: "", text: "RPA 客户", input_type: "0", isEnter: "0", interval: "0" }, options: { timeoutMs: 10000 } },
{ id: "node_amount", componentId: "Element_SetValue_V6", position: { x: 180, y: 530 }, values: { uiSelector: "css:#amount", designated_info: "0", designated_obj: "", text: "128", input_type: "0", isEnter: "0", interval: "0" }, options: { timeoutMs: 10000 } },
{ id: "node_submit", componentId: "Element_Click_V6", position: { x: 180, y: 670 }, values: { uiSelector: "css:#submit", designated_info: "0", designated_obj: "", clickway: "0", clickfunc: "0", operation: "1" }, options: { timeoutMs: 10000 } },
{ id: "node_result", componentId: "Element_GetText_V5", position: { x: 180, y: 810 }, values: { choose: "0", uiSelector: "css:#result", cvSelector: "", designated_info: "0", designated_obj: "", attr_type: "default", attr_name: "" }, options: { timeoutMs: 10000, outputVar: "订单结果" } }
];
flow.edges = [
{ id: "edge_start_open", source: "node_start", target: "node_open" },
{ id: "edge_open_name", source: "node_open", target: "node_name" },
{ id: "edge_name_amount", source: "node_name", target: "node_amount" },
{ id: "edge_amount_submit", source: "node_amount", target: "node_submit" },
{ id: "edge_submit_result", source: "node_submit", target: "node_result" }
];
return flow;
}
export function listFlows(): FlowSummary[] {
const base = dir();
const rows: FlowSummary[] = [];

View File

@ -3,9 +3,10 @@ import path from "node:path";
import type { AppInfo, FlowDefinition } from "../shared/types";
import "./capabilities";
import { catalogPath, loadCatalog } from "./catalog";
import { createFlow, deleteFlow, getFlow, listFlows, saveFlow } from "./flows";
import { createDemoFlow, createFlow, deleteFlow, getFlow, listFlows, saveFlow } from "./flows";
import { capabilityCount, capabilityIds } from "./engine/registry";
import { closeBrowserSession, startRun, stopRun } from "./run/manager";
import { fixtureUrl, startFixtureServer, stopFixtureServer } from "./fixture/server";
let mainWindow: BrowserWindow | null = null;
@ -57,7 +58,8 @@ function registerIpc(): void {
node: process.versions.node,
userData: app.getPath("userData"),
catalogPath: catalogPath(),
packaged: app.isPackaged
packaged: app.isPackaged,
fixtureUrl: fixtureUrl()
}));
ipcMain.handle("catalog:get", () => loadCatalog());
@ -69,6 +71,10 @@ function registerIpc(): void {
const flow = createFlow(name);
return saveFlow(flow);
});
ipcMain.handle("flows:create-demo", () => {
const url = fixtureUrl();
return saveFlow(url ? createDemoFlow(url) : createFlow("未命名任务"));
});
ipcMain.handle("flows:get", (_event, id: string) => getFlow(id));
ipcMain.handle("flows:save", (_event, flow: FlowDefinition) => saveFlow(flow));
ipcMain.handle("flows:delete", (_event, id: string) => {
@ -107,7 +113,12 @@ if (!gotLock) {
mainWindow.focus();
});
app.whenReady().then(() => {
app.whenReady().then(async () => {
try {
await startFixtureServer();
} catch (error) {
console.warn("local fixture server unavailable", error);
}
registerIpc();
createWindow();
app.on("activate", () => {
@ -121,5 +132,6 @@ if (!gotLock) {
app.on("before-quit", () => {
void closeBrowserSession();
void stopFixtureServer();
});
}

View File

@ -11,6 +11,7 @@ const api = {
flows: {
list: (): Promise<FlowSummary[]> => ipcRenderer.invoke("flows:list"),
create: (name?: string): Promise<FlowDefinition> => ipcRenderer.invoke("flows:create", name),
createDemo: (): Promise<FlowDefinition> => ipcRenderer.invoke("flows:create-demo"),
get: (id: string): Promise<FlowDefinition> => ipcRenderer.invoke("flows:get", id),
save: (flow: FlowDefinition): Promise<FlowDefinition> => ipcRenderer.invoke("flows:save", flow),
delete: (id: string): Promise<boolean> => ipcRenderer.invoke("flows:delete", id)

View File

@ -81,7 +81,7 @@ export function App() {
setCatalog(loaded);
let availableFlows = await window.rpa.flows.list();
if (!availableFlows.length) {
await window.rpa.flows.create("网页订单录入");
await window.rpa.flows.createDemo();
availableFlows = await window.rpa.flows.list();
}
if (!cancelled) setFlows(availableFlows);

View File

@ -61,7 +61,7 @@ export function SettingsPage({ catalog }: { catalog: Catalog }) {
{tab === "browser" && (
<>
<SettingCard icon={Globe2} title="浏览器域名白名单"><dl className="kv"><dt>默认允许</dt><dd>127.0.0.1、localhost、example.com</dd><dt>扩展方式</dt><dd>环境变量 RPA_ALLOWED_DOMAINS</dd><dt>格式</dt><dd>逗号分隔域名,不含协议和路径</dd></dl></SettingCard>
<SettingCard icon={Globe2} title="浏览器域名白名单"><dl className="kv"><dt>默认允许</dt><dd>127.0.0.1、localhost、example.com</dd><dt>本地演示站</dt><dd>{info?.fixtureUrl || "未启动"}</dd><dt>扩展方式</dt><dd>环境变量 RPA_ALLOWED_DOMAINS</dd><dt>格式</dt><dd>逗号分隔域名,不含协议和路径</dd></dl></SettingCard>
<SettingCard icon={ShieldCheck} title="限制说明"><p className="field-hint setting-copy">导航只允许 HTTP/HTTPS。未列入白名单的域名会返回 blocked_url;文件协议、脚本协议和任意本地路径不会由浏览器导航组件打开。</p></SettingCard>
</>
)}

View File

@ -34,4 +34,22 @@ describe("flow validation", () => {
definition.edges.push({ id: "second", source: "start", target: "orphan" });
expect(validateFlow(definition, catalog)).toContainEqual(expect.objectContaining({ code: "multiple_normal_edges", nodeId: "start" }));
});
it("does not require inactive designated browser fields or optional screenshot output", () => {
const conditionalCatalog = structuredClone(catalog);
conditionalCatalog.components.push({
id: "Element.Input", name: "输入", kind: "vendor", group: "web", implemented: true, slots: [],
fields: [{ key: "designated_obj", label: "指定对象", required: true, widget: "text", defaultValue: "" }]
} as never, {
id: "Image_Screenshot", name: "截图", kind: "vendor", group: "vision", implemented: true, slots: [],
fields: [{ key: "image_path", label: "输出路径", required: true, widget: "file", defaultValue: "" }]
} as never);
const definition = flow();
definition.nodes = [
{ id: "input", componentId: "Element.Input", position: { x: 0, y: 0 }, values: { designated_info: "0" }, options: {} },
{ id: "shot", componentId: "Image_Screenshot", position: { x: 0, y: 100 }, values: {}, options: {} }
];
definition.edges = [];
expect(validateFlow(definition, conditionalCatalog).filter((issue) => issue.code === "required")).toEqual([]);
});
});

View File

@ -11,6 +11,17 @@ function empty(value: unknown): boolean {
return value === undefined || value === null || (typeof value === "string" && value.trim() === "");
}
function truthy(value: unknown): boolean {
return [true, 1, "1", "true", "True", "是", "开启"].includes(value as never);
}
function requiredInContext(componentId: string, key: string, values: Record<string, unknown>): boolean {
if (key === "designated_obj" && !truthy(values.designated_info)) return false;
if (key === "cvSelector" && String(values.choose ?? "0") !== "1") return false;
if (componentId === "Image_Screenshot" && ["image_path", "outImagePath", "outImageName"].includes(key)) return false;
return true;
}
export function validateFlow(flow: FlowDefinition, catalog: Catalog): FlowValidationIssue[] {
const issues: FlowValidationIssue[] = [];
const componentMap = new Map(catalog.components.map((component) => [component.id, component]));
@ -29,6 +40,7 @@ export function validateFlow(flow: FlowDefinition, catalog: Catalog): FlowValida
issues.push({ severity: "error", code: "not_implemented", nodeId: node.id, message: `「${node.label || component.name}」尚未接入执行器` });
}
for (const field of component.fields.filter((item) => item.required && item.widget !== "hidden" && item.widget !== "region")) {
if (!requiredInContext(component.id, field.key, node.values)) continue;
const current = Object.prototype.hasOwnProperty.call(node.values, field.key) ? node.values[field.key] : field.defaultValue;
if (empty(current)) issues.push({ severity: "error", code: "required", nodeId: node.id, message: `「${node.label || component.name}」缺少必填参数:${field.label}` });
}

View File

@ -166,6 +166,7 @@ export interface AppInfo {
userData: string;
catalogPath: string;
packaged: boolean;
fixtureUrl?: string;
}
export interface RunNodeState {