Compare commits
211 Commits
codex/fix-
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5f5e8bcb0e | ||
|
|
594377ab79 | ||
|
|
41975baf5a | ||
|
|
08e798c087 | ||
|
|
43ae705629 | ||
|
|
10d125ce9d | ||
|
|
b4c89fd7d3 | ||
|
|
751dc04713 | ||
|
|
7a553fb270 | ||
|
|
4f1fbc07dc | ||
|
|
62652b4fca | ||
|
|
a766902a18 | ||
|
|
81ce4bc9b8 | ||
|
|
3a0ff42ac6 | ||
|
|
fb6e8ca0bf | ||
|
|
fec0028bd5 | ||
|
|
0f498afff2 | ||
|
|
e62a408a0d | ||
|
|
f0defc2199 | ||
|
|
107fbe0cee | ||
|
|
07a76b42b7 | ||
|
|
65343da311 | ||
|
|
33b85c081e | ||
|
|
407aa385d2 | ||
|
|
a01667bdb3 | ||
|
|
1b952e3bbd | ||
|
|
4615fcab10 | ||
|
|
93915dfe8a | ||
|
|
2bcda3a558 | ||
|
|
4b5c73b752 | ||
|
|
b18d5b1f0b | ||
|
|
d99be8f4ab | ||
|
|
1f804971f4 | ||
|
|
7fa1c4038c | ||
|
|
9bb5a44b95 | ||
|
|
5e8fc9686f | ||
|
|
1a20dd72d1 | ||
|
|
1da00827ac | ||
|
|
9b76dc6987 | ||
|
|
d972a18bda | ||
|
|
b14ad0b89b | ||
|
|
b4aaba4dbe | ||
|
|
93b79366f7 | ||
|
|
7f5ef2f5df | ||
|
|
a8f75c6a55 | ||
|
|
0fcb1ecc0e | ||
|
|
51a6178bfc | ||
|
|
8414c4f355 | ||
|
|
a7709aefbb | ||
|
|
dbe98947bc | ||
|
|
609c46e06f | ||
|
|
79bba4296f | ||
|
|
5a3d4cbdbd | ||
|
|
8fb3721b0f | ||
|
|
ec25db2207 | ||
|
|
487b46420b | ||
|
|
230e3c79ec | ||
|
|
de6165e76e | ||
|
|
5ef797dcc0 | ||
|
|
902ee0cec4 | ||
|
|
b59d906161 | ||
|
|
e25121660d | ||
|
|
9e12431db8 | ||
|
|
124f11bd6b | ||
|
|
0db3e59f75 | ||
|
|
624d13f2ef | ||
|
|
47b7a43be2 | ||
|
|
36b268f74c | ||
|
|
21a400e172 | ||
|
|
4af6bd0693 | ||
|
|
3e4ded9f5f | ||
|
|
6a27ede6d9 | ||
|
|
20d224a46b | ||
|
|
cf1e560802 | ||
|
|
309898045c | ||
|
|
15fcf66972 | ||
|
|
150d7ce9c9 | ||
|
|
91d998b2ea | ||
|
|
5250bf1e35 | ||
|
|
2ae1010f67 | ||
|
|
8e711d502e | ||
|
|
ea3ff0b891 | ||
|
|
02af3478a7 | ||
|
|
b929f86525 | ||
|
|
d638d44c79 | ||
|
|
abb3d26acd | ||
|
|
61a7f60a0e | ||
|
|
f95a24458e | ||
|
|
22a7ee1595 | ||
|
|
3a9a8b63c2 | ||
|
|
a8084dca26 | ||
|
|
b54311d467 | ||
|
|
e691b9fa78 | ||
|
|
3decced268 | ||
|
|
caa8e19332 | ||
|
|
4139400dbd | ||
|
|
f35f0ac979 | ||
|
|
1c5d981e38 | ||
|
|
b242b9c4fe | ||
|
|
1ff6f59990 | ||
|
|
143fcae34a | ||
|
|
bb03699cce | ||
|
|
f390db9c94 | ||
|
|
e1f162aa82 | ||
|
|
3a9b4fa3c7 | ||
|
|
8d6f0ff755 | ||
|
|
46348413c1 | ||
|
|
6b05a65664 | ||
|
|
158104661c | ||
|
|
fb71c0551b | ||
|
|
58aae7c92f | ||
|
|
cb349ec979 | ||
|
|
d4de0ec245 | ||
|
|
d5d7df8880 | ||
|
|
58ea001e7d | ||
|
|
f0fca2ebfe | ||
|
|
c90584670d | ||
|
|
c973209418 | ||
|
|
9ec301b081 | ||
|
|
7a067173a5 | ||
|
|
4fc1209c93 | ||
|
|
5b1b41663f | ||
|
|
8fdc62d98d | ||
|
|
4486c57536 | ||
|
|
04ac6130a0 | ||
|
|
8176bf7c75 | ||
|
|
8dc2b60c6d | ||
|
|
b4bcee8c15 | ||
|
|
9e9617256d | ||
|
|
6bf8e90c63 | ||
|
|
41b999115e | ||
|
|
4a3e6e38c7 | ||
|
|
b6829cbaab | ||
|
|
0d429e5175 | ||
|
|
38b2199d4b | ||
|
|
9bfa4ac107 | ||
|
|
bc4b349771 | ||
|
|
0651bc995c | ||
|
|
f163c8d0b6 | ||
|
|
fe0c6c2790 | ||
|
|
6b54decfb0 | ||
|
|
346e2ad342 | ||
|
|
d4f27cdaab | ||
|
|
6027895d4d | ||
|
|
12b97315f2 | ||
|
|
241c598b10 | ||
|
|
4824ac85f2 | ||
|
|
b5d6bef369 | ||
|
|
3922447689 | ||
|
|
e7669acd16 | ||
|
|
68e16282de | ||
|
|
9868f5d51c | ||
|
|
38af534f70 | ||
|
|
52423d46a4 | ||
|
|
0960be88e5 | ||
|
|
82765a7f19 | ||
|
|
817ea21c9a | ||
|
|
257fdc7358 | ||
|
|
5e6b7cd878 | ||
|
|
0641c5b91b | ||
|
|
b40c15d5b2 | ||
|
|
22054fed37 | ||
|
|
d98185d1cf | ||
|
|
5aa113ea0e | ||
|
|
c93543be1c | ||
|
|
58c23591a5 | ||
|
|
dfc5041a67 | ||
|
|
afa32133ec | ||
|
|
bf5ad56063 | ||
|
|
3bc8b60e83 | ||
|
|
f0b9d1beec | ||
|
|
f9b980ab5c | ||
|
|
22e3c191a8 | ||
|
|
32ac14a36f | ||
|
|
f8859fed77 | ||
|
|
541908961b | ||
|
|
03e3d31a83 | ||
|
|
bf70cf92e5 | ||
|
|
d7aff54bc6 | ||
|
|
56d00fba49 | ||
|
|
5795c39a6f | ||
|
|
1bb28e3368 | ||
|
|
29f3443110 | ||
|
|
88d5e61c5b | ||
|
|
ba17d6dc2a | ||
|
|
342145cc46 | ||
|
|
c11e1d1527 | ||
|
|
e5f64c21e2 | ||
|
|
1ddd1f1b35 | ||
|
|
b1fc8c25b8 | ||
|
|
1c85a90f1f | ||
|
|
9b9c6f17e4 | ||
|
|
d39c77324a | ||
|
|
927b18551f | ||
|
|
0132871885 | ||
|
|
d22468e29f | ||
|
|
e956ec9055 | ||
|
|
ee60435e2b | ||
|
|
510f0d041d | ||
|
|
95fb07e876 | ||
|
|
66169590de | ||
|
|
a1c65fb42f | ||
|
|
657bc6d7d0 | ||
|
|
ae4f4a2460 | ||
|
|
9a5f86c92a | ||
|
|
85dcb94056 | ||
|
|
e390b1b1eb | ||
|
|
84d73f1b86 | ||
|
|
23d292fdff | ||
|
|
32269a5760 | ||
|
|
adf5f3f7cf |
2
.gitignore
vendored
2
.gitignore
vendored
@ -12,3 +12,5 @@ test-output/
|
|||||||
web-capability-audit.txt
|
web-capability-audit.txt
|
||||||
|
|
||||||
*.traineddata
|
*.traineddata
|
||||||
|
|
||||||
|
temp/
|
||||||
|
|||||||
32
.workbuddy/memory/2026-11-04.md
Normal file
32
.workbuddy/memory/2026-11-04.md
Normal file
@ -0,0 +1,32 @@
|
|||||||
|
# 2026-11-04
|
||||||
|
|
||||||
|
## 合并 origin/main(psText 非 ASCII 修复)
|
||||||
|
- 本地重构(rpa-core + driver 抽象)与远端 2 个旧路径修复提交冲突,3 文件已解:取本地 driver 骨架,把 psText 修复语义移植到新位置(powershell.ts 新增导出、windows-driver.ts 委托、desktop.ts 保持删除、compat-audit.test 迁入 rpa-core)。合并提交 `3a9a8b6`,测试全绿。
|
||||||
|
|
||||||
|
## 排查+修复外部浏览器 CDP 无响应(重要根因)
|
||||||
|
- 现象:e2e / 应用中 Element_SetValue_V6 在外部 Chrome 的 input 上 10s 超时;裸脚本、裸 Electron 复现均正常;换 Edge、退出 Clash Verge(含 clash-verge-service)均无效——Clash 不是元凶(已道歉)。
|
||||||
|
- 取证路径:TCP 层应用↔Chrome 连接真实 ESTABLISHED(重试泄漏 11 条 socket);自写 TCP MITM 代理 + external.ts 加 RPA_EXT_PORT/RPA_EXT_LAUNCH_PORT 诊断开关(启停端口分离 + wsURL 改写)抓到决定性证据:**应用每条 ws 握手后 0 字节发出**,且握手请求带 permessage-deflate。
|
||||||
|
- 根因:ws 打进主进程 bundle 时,可选依赖 bufferutil 的打包解析产物破坏 mask fallback;permessage-deflate 异步路径把 `bufferUtil.mask is not a function` 静默吞掉 → 帧 0 字节无报错。
|
||||||
|
- 修复(提交 `22a7ee1`):
|
||||||
|
- electron.vite.config:externalizeDepsPlugin({ include: ["ws"] }),运行时 require
|
||||||
|
- package.json:ws 提升到根依赖(electron-builder 收集进 asar,21595 条 node_modules)
|
||||||
|
- transport.ts:显式 perMessageDeflate: false(双保险)
|
||||||
|
- package.mjs verifyLaunch:剥离 ELECTRON_RUN_AS_NODE / NODE_OPTIONS(WorkBuddy 沙箱 shell 带 ELECTRON_RUN_AS_NODE=1,曾把打包应用带进纯 Node 模式 → "bad option" 退出码 9 假故障)
|
||||||
|
- flows.ts:RPA_DEMO_BROWSER 强制指定浏览器 + Chrome 缺装回退 Edge
|
||||||
|
- 验证:349 测试 + e2e + portable/NSIS 双冒烟全绿;release/dist 产出免安装目录、zip、安装包、SHA256SUMS。
|
||||||
|
- 教训:electron-builder 的 asar list 输出用反斜杠路径,grep 判缺失时别用 `/`;mouse/keyboard 真实冒烟测试运行期间桌面不能有人操作。
|
||||||
|
|
||||||
|
## 其他
|
||||||
|
- 用户确认删除了 compat-audit 测试产物 .compat-report.txt(vitest 跑 compat-audit 会重新生成,未跟踪)。
|
||||||
|
- 用户询问改造进度:阶段 0~5 已全部完成,只剩阶段 6(迁入 Agent-Employee + Bun 冒烟 + 可选 PowerShell worker)。
|
||||||
|
|
||||||
|
## 任务分享/导入功能 + 只读文件保存 EPERM 修复(提交 `f95a244`)
|
||||||
|
- 用户报错 flows:save EPERM(read_gateway_api_keys.json.tmp rename 失败):根因是外部复制导入的任务文件带 Windows 只读属性,rename 覆盖只读文件在 Windows 抛 EPERM。已当场清掉用户文件只读属性和残留 .tmp。
|
||||||
|
- saveFlow 加固为 writeFlowFile:rename 失败(EPERM/EACCES/EBUSY)→ 清只读属性+删除后 rename → copyFileSync 覆盖兜底,仍失败给友好占用提示。
|
||||||
|
- 新增分享/导入:exportFlowToPath/importFlowFromPath(.rpaflow.json = FlowDefinition JSON);IPC flows:export/flows:import(弹对话框、多选导入、失败逐个汇报);preload flows.export/import;TasksPage 头部「导入任务」+ 每行「分享」按钮;导入 id 冲突生成 imported_* 新 id 绝不覆盖。
|
||||||
|
- 测试 7/7 绿(新增导出往返/id 冲突/非法文件/只读覆盖 4 例),typecheck、build 绿。
|
||||||
|
|
||||||
|
## 元素拾取 Chrome 启动失败(exit 0)修复(提交 `ea3ff0b`)
|
||||||
|
- 根因:上次运行遗留的 Chrome 占用专用 profile(rpa-studio-external-*)时,再次启动的 Chrome 转交旧实例后自杀(exit 0),报「Chrome 启动失败(进程已退出,代码 0)」。
|
||||||
|
- 修复:launchExternalBrowser 检测子进程立即退出 → 按命令行里的 user-data-dir 过滤杀残留进程 → 重试一次;启动参数加 --hide-crash-restore-bubble 防恢复气泡遮挡拾取。
|
||||||
|
- 验证:故意拉起无调试端口的同 profile Chrome 后跑 e2e,自动清理重试生效,演示流程 succeeded。
|
||||||
301
.workbuddy/memory/2026-11-05.md
Normal file
301
.workbuddy/memory/2026-11-05.md
Normal file
@ -0,0 +1,301 @@
|
|||||||
|
|
||||||
|
## CC Switch 流程「复制失败」修复
|
||||||
|
- 根因:流程用 WebBrowser_RunJsInIeBySelector 执行 button.click(),合成点击无 user activation,网页剪贴板 API 被拒 → 页面提示"复制失败",但节点返回 true 看似成功;Clipboard_GetText 读到的是残留旧内容(窗口标题)。
|
||||||
|
- 修复:改写 %APPDATA%\rpa-studio\flows\imported_cc_switch_mw2tyjfu_lmq9.json,全流程去 JS:
|
||||||
|
- loop_click_copy → Element_Click_V6(CDP Input.dispatchMouseEvent 可信点击),选择器 css:table tbody tr:nth-of-type(${index+1}) button[title="复制到剪贴板"],retryCount 2
|
||||||
|
- node_read_endpoint(JS) → Element_GetText_V5(css:body, pageText) + 新增 Regex_Find 节点(pattern: API\s*端点[^\n]*?(https?://[^\s|]+),index 1 → keyEndpoint),边已重连
|
||||||
|
- 引擎插值确认:runtime 对所有 values 通用 interpolate,${index} 表达式在 uiSelector 中可用。
|
||||||
|
|
||||||
|
## dev 启动 bufferutil 报错修复
|
||||||
|
- 现象:dev 启动弹 "Could not resolve 'bufferutil' imported by 'ws'",主进程崩溃。
|
||||||
|
- 根因:package.json 根依赖里 ws 声明丢失(疑似合并冲突解决时被覆盖),externalizeDepsPlugin 默认只外部化 dependencies,ws 被打进 bundle,rollup 为其可选依赖 bufferutil 生成顶层 throw 存根。
|
||||||
|
- 修复:package.json dependencies 补 "ws": "^8.18.0" + npm install 同步 lockfile;重启 dev 正常(out/main/index.js 从 793KB 降到 671KB,bundle 内 require("ws") 运行时解析)。
|
||||||
|
- 教训:ws 必须保持在根 dependencies,否则 ws 打包坑(存根 throw + permessage-deflate)在 dev/打包都会复发。
|
||||||
|
|
||||||
|
## CC Switch 流程端点提取修复(第 2 轮)
|
||||||
|
- 复制修复验证成功:可信点击后剪贴板读到真实密钥 sk-de6855...。
|
||||||
|
- Regex_Find 返回空:页面文本 "API 端点" 与 URL 之间隔多个换行,[^\n]*? 不跨行 → 改为 [\s\S]*? 跨行非贪婪,用真实 pageText 自测通过。
|
||||||
|
- sw_click_tab 选择器 type=Button 太严 → 去掉类型限定,改为 window=CC Switch;name=Codex。
|
||||||
|
|
||||||
|
## 流程修改被应用覆盖的坑(重要)
|
||||||
|
- 现象:直接改 %APPDATA% 下的 flow JSON 后运行,修改被回滚——应用在运行流程时会把内存中的流程副本 flows:save 回磁盘,覆盖外部修改。
|
||||||
|
- 正确姿势:改 flow JSON 必须先关闭应用 → 改完回读验证 → 再启动应用(内存副本刷新为磁盘版)。
|
||||||
|
- 端点正则最终版:API\s*端点[\s\S]*?(https?://[^\s|]+)(index 1);sw_click_tab 选择器放宽为 window=CC Switch;name=Codex。已落盘+自测通过,应用已重启加载。
|
||||||
|
|
||||||
|
## 新流程:淘宝 · 手机搜索比价(走真实流程迭代模式)
|
||||||
|
- 用户反馈:不用手搓 CDP 走查,应用本身有浏览器接管+拾取器+运行记录,按 CC Switch 的模式「建草稿→跑→看 run.json 迭代」即可。
|
||||||
|
- 已建 %APPDATA%\rpa-studio\flows\flow_taobao_phone_price.json(18 节点):Edge 打开淘宝→搜手机→等结果(40s 给手动登录/滑块)→分列抓标题/价格/店铺/销量→DataService_LargeModel 比价分析→Bivartable 逐行写入(A-D 列)+AI 结论(F2)。
|
||||||
|
- 选择器为猜测值(input[name=q]、button.btn-search、div[class*=doubleCardWrapper]、[class*=title/price/shopName/realSales]),节点标了【拾取确认】,跑挂了用拾取器重采。
|
||||||
|
|
||||||
|
## 拾取器跟随页面跳转(已提交 5250bf1)
|
||||||
|
- 用户痛点:目标页面要在运行后才能到达,拾取器只认打开时的页面。
|
||||||
|
- 重构 pickElement:拾取脚本改由 Page.addScriptToEvaluateOnNewDocument 随新文档自动重注入,主进程每 400ms 轮询 window.__rpaPickResult;页面跳转不再失效,拾取窗口里可翻页/搜索/过滑块后继续拾取。Esc 取消→cancelled,超时→timeout。
|
||||||
|
- 定位根因:早上的 bufferutil 崩溃是合并 2ae1010 把 config 的 include:["ws"] 和 package.json 的 ws 依赖都覆盖丢了,已双双恢复。
|
||||||
|
- 教训:electron-vite dev 不会自动重编译主进程改动,改代码后需重启 dev。
|
||||||
|
|
||||||
|
## 拾取器跟随新标签页(第 2 轮)
|
||||||
|
- 痛点:淘宝点搜索会开新标签页,拾取附着在旧标签页失效。
|
||||||
|
- 方案:pickElement 每 400ms 轮询 /json/list,pages[0](前台标签页)变化时 activateTab 切附着 + Page.enable + 重注册 addScriptToEvaluateOnNewDocument + 重注入;finally 对所有 tabTransports 清残留、移除全部注入。
|
||||||
|
- 已提交,typecheck + 25 单测通过,dev 已重启生效。
|
||||||
|
|
||||||
|
## 拾取器支持钻入子元素(第 3 轮)
|
||||||
|
- 痛点:淘宝结果整卡是一层 <a> 链接,标题/价格是其子元素;层级调整只能向上走祖先链,选不到内部标题。
|
||||||
|
- 方案:悬停记录 deepestOf(光标下最深元素),锁定时把「锁定元素→最深元素」的子级路径并入 chain;PageUp 向上升/PageDown 向下钻。空格锁定→PageDown 几次到标题→空格确认。
|
||||||
|
- 已提交;typecheck + 25 单测过;dev 已重启。
|
||||||
|
|
||||||
|
## 拾取器悬停自动钻入(第 4 轮)
|
||||||
|
- 反馈:PageDown 未锁定时会翻页(要先空格锁定,太隐蔽);用户建议悬停自动钻。
|
||||||
|
- 实现:进入新交互区域吸附交互元素,悬停 300ms 自动钻入光标最深元素,钻入后跟随光标在内部移动;PageUp/PageDown 未锁定时直接锁定(PageDown=最深处,PageUp=交互元素),锁定后上下调层。
|
||||||
|
- 提交 + dev 重启生效。
|
||||||
|
|
||||||
|
## 拾取器泛化动态 id(第 5 轮)
|
||||||
|
- 痛点:淘宝卡片根节点是唯一 id(item_id_1087512368188),cssPath 直接用 #id,选择器只命中一张卡。
|
||||||
|
- 方案:stable() 对以 ≥6 位数字结尾的 id,若 [id^="前缀"] 命中 >1 元素则优先泛化采用;普通 id 行为不变。两处 cssPath(pickElement 注入脚本 + elementReferenceHelpers)同步修改。
|
||||||
|
- 生成结果示例:[id^="item_id_"] > div:nth-of-type(1) > ... 每张卡都能命中。
|
||||||
|
- 提交后 dev 重启生效。
|
||||||
|
|
||||||
|
## 拾取器残留守卫 bug(第 6 轮,根因级)
|
||||||
|
- 现象:泛化 id 已提交但用户拾取仍出 #item_id_ 唯一 id。
|
||||||
|
- 排查:产物含新代码、真实页面 [id^=item_id_] 有 46 个、正则正确 → 锁定环境问题。
|
||||||
|
- 根因:11:13 强杀 electron 时页面挂着未结束的拾取,__rpaPickerActive.active 永久为 true;引导脚本守卫 if(active) return 使新脚本永不生效,一直由旧代码残留监听器响应。
|
||||||
|
- 修复:守卫改强制接管(cancelHard+清状态+删宿主),pickElement 注入前同步清残留。
|
||||||
|
- 验证:本地复现页两轮 E2E(CDP 合成鼠标+两段式空格:空格一次=锁定、两次=确认),无残留/有残留均输出 css:[id^="item_id_"]...。25 单测过,已提交,dev 重启生效。
|
||||||
|
- 环境教训:本机手工拉起的测试 Edge 容易被用户随手关掉,测试要"拉起+跑"一条命令内完成;ECONNREFUSED 不一定是端口死,也可能是竞态。
|
||||||
|
|
||||||
|
## 泛化阈值时序 bug(第 7 轮,最终修复)
|
||||||
|
- 现象:全新应用+全新浏览器仍出 #item_id_ 唯一 id;现场检查 9222 实例:46 卡、正则/条件全对。
|
||||||
|
- 根因:浏览器 11:45:05 刚拉起用户就拾取,页面渐进渲染只有 1 张卡,泛化条件"命中>1"不成立回退唯一 id。
|
||||||
|
- 修复:动态数字 id(≥6 位数字结尾)命中 >=1 即泛化(此时与原 id 等价,零风险)。单卡页面 E2E 验证输出 css:[id^="item_id_"]...
|
||||||
|
- 已提交,dev 重启生效。拾取器共 7 轮迭代:跳转重注入→跟前台标签页→钻入子元素→悬停自动钻→泛化动态id→强制接管残留→泛化阈值。
|
||||||
|
|
||||||
|
## 模板字面量吃反斜杠(第 8 轮,真正根因)
|
||||||
|
- 现象:阈值修复后用户重拾仍出唯一 id。
|
||||||
|
- 根因:PICKER_BOOTSTRAP_SOURCE 是 TS 模板字面量,编译时 \\d 被当无效转义吞掉反斜杠,运行时正则变成字面量 d{6,} 永不匹配 → 泛化从未生效。此前 E2E 直接注入源码原文(反斜杠还在)绕过了编译步骤,所以全绿——教训:验证必须用 bundle 里提取的真实运行时串。
|
||||||
|
- 修复:正则改 [0-9]{6,}(无转义需求);从 bundle 提取真实脚本重跑 E2E 输出 css:[id^="item_id_"]...,应用已重启生效。
|
||||||
|
- 通用教训:模板字符串内的正则永远别用 \\d 类转义,用 [0-9] 或双写反斜杠。
|
||||||
|
|
||||||
|
## 淘宝流程四列选择器定稿(真实页面验证)
|
||||||
|
- 用户拾取标题成功(泛化 id 生效)。价格/店铺/销量三个猜的选择器 0 命中:卡片是 <a> 不是 div,且新版类名变了。
|
||||||
|
- 真实页面探测+验证(46 卡):标题 [id^="item_id_"] > div:nth-of-type(1) > ... > span(46);价格 [id^="item_id_"] [class*="innerPriceWrapper"](46,纯数字);店铺 [class*="shopNameText"](46,注意 shopName 会双匹配 92);销量 [class*="realSales"](31——广告卡无销量,列尾可能少量错位)。
|
||||||
|
- 已按"关应用→改流程→重启"写入三个 uiSelector。表头 tableRows、AI 提示词、写入循环均已配好。
|
||||||
|
|
||||||
|
## 淘宝流程运行前检查 12 错误修复
|
||||||
|
- 原因:用户在应用里拾取/编辑后应用保存流程,把我之前补的 web/cssSelector/xpath 必填字段覆盖掉了;且拾取把 node_read_titles 的 xpath 写成了 ${searchReady}。
|
||||||
|
- 校验依据:resources/catalog.json 中 Element_SimilarElementList_V2 必填 = uiSelector/designated_obj/web/cssSelector/xpath/parent_selector/attr_name;运行时 selectorOf 默认优先 uiSelector,cssSelector/xpath 仅为校验用。
|
||||||
|
- 修复:关应用→四个节点补 web=当前浏览器、cssSelector=uiSelector 去 css: 前缀、xpath=/→回读校验齐备→重启。注意 ${searchReady} 是 node_wait_search 的输出变量(node_input 的 designated_obj 在用),xpath 不能指向它。
|
||||||
|
|
||||||
|
## AI 提示词字段 + 循环变量警告修复
|
||||||
|
- AI 节点错误:DataService_LargeModel 必填字段是 hint_word,此前误写到 input(校验拦截);已关应用→迁移字段→重启。
|
||||||
|
- 循环变量警告:校验器 definedVars 不认循环容器运行时注入的 item/index/value(引擎 ctx.vars 注入),已在 flowValidation.ts 加 LOOP_VARS 映射(计数循环支持自定义索引名),25 校验测试过,已提交。
|
||||||
|
- 淘宝流程选择器/必填字段确认仍齐备(updatedAt 04:29 未再被覆盖)。
|
||||||
|
|
||||||
|
## 淘宝流程:搜索后加延迟+关弹窗+等结果
|
||||||
|
- 用户要求:点搜索后加延迟、加一个点关闭按钮的节点。
|
||||||
|
- 发现点搜索原本直连抓取节点(无任何等待)。已插入 3 节点:Basic_Delay_V2(3s) → logic_control_try_catch(尝试关闭登录弹窗,try_fn 内 Element_Click_V6 通用 close 选择器、is_verify=0、超时 6s,弹窗没出现则吞错继续) → Element_WaitElement(css:[id^="item_id_"], 40s,自适应等结果)。
|
||||||
|
- 槽位机制备忘:容器子节点用 parentId + slot 字段(try_catch 的槽是 try_fn/except_fn,数组循环是 array_for_fn)。
|
||||||
|
- 关应用→改→重启完成,21 节点。
|
||||||
|
|
||||||
|
## 运行时跟随前台标签页(关键修复)
|
||||||
|
- 现象:运行记录里 loop_click_close 报"未找到元素"(用户明明拾取过),四个抓取节点 2-3ms "成功"实为在旧标签页抓空,AI 报"输入文本为空"。
|
||||||
|
- 根因:拾取器做了前台标签页跟随,但运行链路没有——点搜索开新标签页后,元素操作仍附着旧页。
|
||||||
|
- 修复:session.evaluate 入口加 followForegroundTab()(/json/list 查前台页,变化则 activateTab,400ms 节流防高频轮询放大)。监听器只走 cdp 无递归风险。
|
||||||
|
- 25 单测过,已提交,dev 重启生效。注意:需要后台页操作的场景应显式用「切换标签页」能力。
|
||||||
|
|
||||||
|
## 搜索按钮选择器修正(真实页面探测)
|
||||||
|
- 点搜索按钮失败:猜的 css:#J_TSearchForm > div:nth-of-type(2) > button 层级不对。
|
||||||
|
- 连 61619 实例探首页:表单 #J_TSearchForm 还在,按钮是 BUTTON.btn-search(DIV.search-button 内),新选择器 css:#J_TSearchForm button.btn-search 命中 1 个、文本"搜索"。
|
||||||
|
- 淘宝首页当前 front tab 已是 s.taobao.com/search(说明上次运行其实点进去了或手动操作过)。已关应用→改流程(uiSelector+cssSelector 同步)→重启。
|
||||||
|
|
||||||
|
## 点搜索"未找到元素"= 页面渲染时序问题
|
||||||
|
- 现象:选择器已验证正确(btn-search 命中 1),运行时却 17s 找不到;事后连上同一标签页按钮存在、readyState complete。
|
||||||
|
- 根因:淘宝首页 #q 输入框是静态 HTML 早出,.btn-search 按钮 JS 渲染慢;点击重试 3×5s 仍未出。
|
||||||
|
- 修复:node_wait_search 从等 #q 改为等 #J_TSearchForm button.btn-search(等按钮=整块渲染好),timeout 30s。重启完成。
|
||||||
|
|
||||||
|
## 「点到了还报错」澄清 + 搜索按钮选择器再次回滚
|
||||||
|
- 用户看到"点击了搜索按钮"是自己在 Edge 窗口里手动点的;流程节点本身因选择器不匹配从未点到,等 17s 后报 not_found。页面跳转≠流程节点成功。
|
||||||
|
- node_click_search 的 uiSelector 被应用内重新拾取/编辑覆盖回结构化路径 #J_TSearchForm > div:nth-of-type(2) > button(label/cssSelector/position 混合状态证实),结构路径在冷启动新页面渲染顺序变化后匹配不上。
|
||||||
|
- 修复:uiSelector 与 cssSelector 统一为稳健的 #J_TSearchForm button.btn-search(按 class、不依赖层级序号),关应用→改→重启完成。
|
||||||
|
- 教训:拾取器生成的结构化路径对动态渲染页面不稳,同类站点节点优先用 class 特征选择器;改完流程后让用户避免对已修节点重拾。
|
||||||
|
|
||||||
|
## 修复:点击后页面跳转被误报为点击失败(13:15)
|
||||||
|
- 用户澄清:搜索按钮点击是流程节点点的(非手动),页面也跳了但节点报错
|
||||||
|
- 根因:click() 点击后 180ms 回读 el.dataset.rpaClickToken 验证"可信点击",但按钮触发跳转 → 同页 context 销毁/新标签页被前台跟随切走 → 回读必然失败 → 误报"目标未收到可信点击"(还带满超时重试,失败得慢)
|
||||||
|
- 修复(session.ts):回读改为快速尝试(1 次 + 400ms 后 1 次),失败后用 navigationEvidence(beforeTargets, beforeUrl) 判定:新开标签页 / URL 变化 / evaluate 抛 context destroyed → 视为点击已生效,返回 { navigated: true }
|
||||||
|
- browser.ts:日志加"(页面已跳转,点击生效)"
|
||||||
|
- tsc 通过;dev 已热重启生效
|
||||||
|
|
||||||
|
## 修复:多标签环境前台跟随切错页 + dev 未重编译坑(13:21-13:28)
|
||||||
|
- 用户重跑仍失败:run_mw33knx1 run.json 显示 node_wait_search 成功(searchReady=true)但 0.5s 后 node_click_search "Element not found: #J_TSearchForm button.btn-search"
|
||||||
|
- 根因 1:followForegroundTab 在每次元素操作前无条件切到 pages[0],浏览器有多个旧标签页时切错页;node_open openNewTab=1 新开的页不是 pages[0]
|
||||||
|
- 修复(session.ts):evaluate 加 follow 参数(默认 true 兼容);withElementOnce / locate 改两段式——先在当前激活页找,找不到才 followForegroundTab 后重试一次
|
||||||
|
- 根因 2:electron-vite dev 一直没重编译(out/main 还是 13:13 产物),13:15 的修复实际未生效;重启时还踩坑 ELECTRON_RUN_AS_NODE=1(WorkBuddy 继承)导致 electron.app undefined → env -u 清除后正常
|
||||||
|
- 教训:验证代码生效要看 out/main 产物内容,不能只看进程 PID;重启 dev 必须用 env -u ELECTRON_RUN_AS_NODE
|
||||||
|
- 新 dev 任务 id: acdZmO
|
||||||
|
|
||||||
|
## 修复:相似元素列表全重复 + AI 不认 hint_word(13:29-13:35)
|
||||||
|
- run_mw33t9g5:点搜索/关弹窗/等结果全过,抓到 46/46/46/29 条,但每列值全相同(46 个标题同一句直播广告、价格全 4799)
|
||||||
|
- 根因 A:动态 id 泛化使 46 张卡的引用选择器全为 [id^="item_id_"] > ... 相同串 → getSimilarElements 46 个引用全命中第一张卡
|
||||||
|
- 修复:BrowserElementReference 加 index?; selectorSpec 透传; getSimilarElements/getChildren 生成引用时算 querySelectorAll().indexOf(node) 写入 index; withElementOnce 与 locate 的 css/xpath find 按 index 取(index>0 用 querySelectorAll[n])
|
||||||
|
- 根因 B:ai-caps.ts inputText() 候选键不含 hint_word("输入文本为空")→ 加入 hint_word
|
||||||
|
- electron-vite dev 确认不自动重编译主进程,每次改完必须重启 dev(env -u ELECTRON_RUN_AS_NODE npm run dev);验证看 out/main/index.js 内容
|
||||||
|
- 新 dev 任务 id: HtSyND
|
||||||
|
|
||||||
|
## 修复:about:blank 残留页 + try_catch 捕获语义不明(13:41-13:47)
|
||||||
|
- 用户问:为啥浏览器有 about:blank 页;关闭按钮明明"已经关了"却一直报找不到
|
||||||
|
- about:blank 根因:外部浏览器启动参数固定带 about:blank 初始页(external.ts startUrl 兜底),openNewTab 再开新 tab 后空白页永远残留(还是之前前台跟随切错页的元凶之一)
|
||||||
|
- 修复(session.ts openNewTab):新开 tab 前先找 about:blank/chrome://newtab/edge://newtab 空白页,有则 activateTab+navigate 复用,没有才 openNewPageTarget
|
||||||
|
- 关闭按钮:try_catch 兜底本来就该"找不到=没弹窗=继续",但引擎只把错误写进 ctx.vars.error 无日志说明,用户误以为故障
|
||||||
|
- 修复(runtime.ts try_catch catch 块):加 warn 日志「子步骤未成功,已捕获并继续执行:<原因>」
|
||||||
|
- dev 重启任务 id: L9eDv1
|
||||||
|
|
||||||
|
## 关闭按钮选择器漂移 + 关键教训:electron 内存代码≠磁盘产物(13:47-13:56)
|
||||||
|
- run_mw348pyc(13:39,AI 503 前一步)数据仍全重复 → 发现 electron-vite dev 只重编译不重启 electron:13:27 启动的进程内存里一直是旧代码,13:35 我 grep 磁盘产物验证是假象;13:47 重启后 index 修复才首次真正进内存
|
||||||
|
- 教训:验证修复生效必须重启 electron 进程本身,grep 磁盘产物不够
|
||||||
|
- 关闭按钮"刚才能点现在点不到":html > body > div:nth-of-type(4) > img 结构序号随 body 动态内容漂移
|
||||||
|
- 修复:flow 文件 loop_click_close 改为 css:img[src*="close"](try_catch 兜底,找不到就跳过);拾取器 stable() 增强——IMG 元素优先用 img[src$="文件名"] 特征(无反斜杠,模板字面量安全)
|
||||||
|
- flow 运行时每次 getFlow 读磁盘,改文件无需重启;但用户在画布点保存会覆盖
|
||||||
|
- AI 503 是服务端暂时不可用,重试即可
|
||||||
|
- dev 任务 id: uKXMoA
|
||||||
|
|
||||||
|
## 真凶落网:NodeList 没有 indexOf(14:00-14:11)
|
||||||
|
- 13:57 用户重跑数据仍全重复(这次确实是新代码)→ index 修复本身有 bug
|
||||||
|
- 写 e2e 测试 similar-list.test.ts(46 张卡模拟淘宝 item_id_ 动态 id + 无头 Chrome CDP)复现:getSimilarElements 返回引用 text 各不同但 index 全部缺失
|
||||||
|
- 根因:`root.querySelectorAll(...).indexOf(node)` —— NodeList 没有 indexOf 方法,TypeError 被 catch {} 静默吞掉,index 从未写入
|
||||||
|
- 修复:[...querySelectorAll(...)] 展开后再 indexOf(getSimilarElements + getChildren 两处)
|
||||||
|
- e2e 4/4 通过:46 引用 index 互不相同、标题/价格/parent 三列 unique=46
|
||||||
|
- 踩坑记录:测试页 DOM 结构要贴合淘宝分层(标题独占 div:nth-of-type(3)),否则选择器 0 命中或 184 条
|
||||||
|
- 流程已重启(L4lL0X),等用户重跑验证 Excel 写入
|
||||||
|
|
||||||
|
## 流程全程跑通 + 补 Excel 落盘(14:13-14:20)
|
||||||
|
- run_mw35g6d8 succeeded!47 条标题 46 唯一、价格 34 唯一、AI 分析 1844 字完整输出——数据重复问题正式解决
|
||||||
|
- 用户"什么都没得到":流程末尾只有内存表(Bivartable),没有 Excel 落盘节点
|
||||||
|
- 修复 1:末尾加 node_save_excel(Bivartable_DfWriteToFile,桌面/淘宝手机比价.xlsx),edge: node_write_ai → node_save_excel;流程运行时读盘,无需重启
|
||||||
|
- 修复 2:node_init_rows 表头 bug——value 是一维数组被 Bivartable 当 4 行,改为 [["标题","价格","店铺","销量"]]
|
||||||
|
- 立即交付:scripts/export-last-run.js 从 run.json 的 tableRows(python str 风格序列化,ast.literal_eval 解析)重建表格,exceljs 写桌面《淘宝手机比价.xlsx》(比价结果 47 行 + AI分析 sheet)
|
||||||
|
|
||||||
|
## 修复:Excel 落盘 EBUSY(14:31-14:36)
|
||||||
|
- 用户跑流程保存 Excel 报 EBUSY: resource busy —— 目标文件正被 WPS/Tencent 预览打开,Windows 锁文件
|
||||||
|
- 修复(data-table-common.ts Bivartable_DfWriteToFile/DoSave):捕获 EBUSY/EPERM/占用类错误,自动另存为 `原名_20261105_143500.xlsx` 时间戳文件并记 warn 日志,流程不失败
|
||||||
|
- 用户侧解决办法:跑流程前关掉正在预览的 Excel
|
||||||
|
- dev 重启任务 id: tn3Ik9
|
||||||
|
|
||||||
|
## 生成 Word 版比价报告(14:39-14:55)
|
||||||
|
- 用户嫌 Excel 表格不好看(列窄要手动拉开),要求 Word 版
|
||||||
|
- 路由:workbuddy-office-routing → 带材料生成 Word → workbuddy-local-office-edit(editor_sdk MCP,edsdk.py 封装)
|
||||||
|
- 流程:create_doc → doc_modify_section landscape(横向,表格宽)→ doc_insert_markdown(file:// 方式插大段 md)→ doc_set_table_properties(列宽 manual dxa、表头 D6E4F0 底色、隔行 F5F7FA)→ save_file 到桌面《淘宝手机比价报告.docx》→ present_files
|
||||||
|
- 踩坑 1:md 表格内 `|` 用 `\|` 转义不被引擎识别,会把行拆乱 → 用全角|
|
||||||
|
- 踩坑 2:抓取数据里混有 \r 和 \n(价格 "17739\n.04"),未清洗导致幻影行(50 行 vs 预期 48)→ 生成前所有列统一 replace \r\n + strip
|
||||||
|
- 踩坑 3:doc_modify_section 必须传 section_index(schema 未标必填但服务端要求)
|
||||||
|
- 跑流程自动出 Word 版:可在 flow 末尾加节点?暂未做——目前 Word 报告是手工生成的;如需每次运行自动生成 Word,后续可加能力节点
|
||||||
|
|
||||||
|
## 修复:关弹窗点击生效却等待+报错(14:51-14:58)
|
||||||
|
- run_mw367eym 整体成功但 loop_click_close failed:用户观察到"弹窗关掉了但等很久还报错"
|
||||||
|
- 根因链:点击关闭按钮 → 弹窗整体从 DOM 移除 → token 回读找不到元素 → 判失败 → 引擎 6s 超时内反复重试 → 弹窗已没了,每次 locate 报"未找到元素"
|
||||||
|
- 修复(session.ts click):readToken 捕获 not_found 时置 elementGone;回读失败 → 先 navigationEvidence(跳转)→ 再 elementGone(元素消失=弹窗被关)→ 返回 { dismissed: true } 成功
|
||||||
|
- browser.ts:日志区分「(页面已跳转,点击生效)」「(弹窗已关闭,点击生效)」
|
||||||
|
- 注意:用户重新拾取过关闭按钮,uiSelector 已变 img[src$="TB1QZN...png"](cssSelector 仍是 img[src*="close"],运行用 uiSelector)
|
||||||
|
- dev 任务 id: eboY4I
|
||||||
|
|
||||||
|
## 新流程:晨间助手全家桶(14:58-15:40)
|
||||||
|
- 用户想测不同功能,盘点 catalog 1016 组件中**已注册实现**的能力后,设计 30 节点长流程 flow_morning_assistant.json:
|
||||||
|
A 文件整理(scan 下载测试沙盒 → mkdir 4 类目录 → 遍历+IF 按扩展名移动)→ B 记事本键鼠(App_Run notepad → Keyboard_Send 摘要 → 截图 → App_Kill)→ C 数据加工(DoOpenExcel 读淘宝比价表 → Sort/Filter/Min/Max/Avg → DfWriteToFile 新表)→ D OCR+AI(截图 OCR → AI 总结 → 剪贴板 → 弹窗通知)
|
||||||
|
- 沙盒:桌面/RPA整理测试/ 7 个样例文件
|
||||||
|
- 生成脚本 scripts/gen-morning-flow.py;校验:组件存在+必填字段 0 错误;表达式(contains/lower/len/round/||)用真实引擎 vitest 4/4 通过
|
||||||
|
- 坑:测试文件相对路径深度(__exprcheck 在 4 层,需 ../../../../packages);@rpa/core 不导出 expression 模块,需从源码相对路径导入
|
||||||
|
- 待用户在应用里跑一遍验证(运行记录节点耗时/IF 分支走向/OCR 质量/AI)
|
||||||
|
|
||||||
|
## 修复:File_File_Move 目标为目录时 ENOTDIR(15:41-15:44)
|
||||||
|
- 晨间助手首跑失败:遍历移动文件报 "cp returned ENOTDIR (cannot overwrite directory ...\其他 with non-directory ...背景音乐.mp3)"
|
||||||
|
- 根因:files.ts 的 File_File_Move 把 pathDest/dest 直接当完整目标文件路径(rename 文件→目录失败,fallback cp 也报 ENOTDIR);我的流程里 pathDest 填的是分类目录
|
||||||
|
- 修复(files.ts):目标存在且 isDirectory 时自动 path.join(target, basename(source))("移入目录"语义),目标不存在仍按完整路径
|
||||||
|
- tsc 通过;编译产物 15:43:51 确认含新逻辑;dev 任务 id: 7DlMEh
|
||||||
|
- 待验证:重跑晨间助手(IF 分支走向、OCR、AI、记事本环节)
|
||||||
|
|
||||||
|
## 打包(15:42)
|
||||||
|
- `RPA_SKIP_TESTS=1 npm run package` 跳过单测打包成功(约 9 分钟)。
|
||||||
|
- 注意:脚本清理 release/dist 旧产物时 fs.rmSync 会被环境的批量删除保护拦截(>50 文件需确认),需先手动删掉 release/dist 下的旧产物再跑。
|
||||||
|
- 产物:release/dist/RPA-Studio-Setup-0.1.0-win-x64.exe(sha256 b213c91f…)、RPA-Studio-0.1.0-win-x64.zip(92ebbf3f…)。
|
||||||
|
|
||||||
|
## 晨间助手全家桶首跑成功(15:55)
|
||||||
|
- run_mw393tg6 succeeded,26 节点全成功
|
||||||
|
- 实证:7 个样例文件 100% 正确归类(zip→压缩包、png→图片、docx/pdf/txt→文档、mp3/exe→其他);桌面《低价手机清单.xlsx》生成(min 98 / max 15999 / avg 6360.75);OCR 识别出屏幕文字;AI 总结 100 字完整;截图存根目录
|
||||||
|
- 小瑕疵:截图时机在记事本关闭后(OCR 抓到的是桌面而非记事本内容);AI 从 OCR 文本联想出"遇小插曲"属幻觉
|
||||||
|
- 测试通过:文件操作+IF+遍历、App 控制+键鼠、Excel 读+排序筛选统计、OCR+AI+剪贴板+弹窗 全链路验证
|
||||||
|
|
||||||
|
## 新流程:Word 报告加工(16:05-16:12)
|
||||||
|
- 用户选了 F 方案。flow_word_report_process.json(11 节点):Word_Open 比价报告 → ReadAll 全文 → Replace「RPA 自动采集」→「(已加工校验)」→ Enter 空行 → Write 追加加工说明(含 ${len(docText)} 字数)→ InsertPicture 整理完成截图 → Save → File_Copy 复制「_加工版.docx」副本 → Close → 弹窗
|
||||||
|
- Word 能力是 PowerShell COM 实现(每节点独立开关文档,隐藏窗口);Word_Open 仅返回 {path} 句柄,objWord 传对象 handlePath 兼容
|
||||||
|
- 坑:Word_SaveAs 与 Save 实现相同(只返回路径,不真正另存)→ 用 File_Copy_V1 造副本(target_path=目录 + rename=1 + name=新名)
|
||||||
|
- 校验 0 错误;len() 支持字符串已 vitest 验证;start 组件 id 是 Trigger.Manual(非 Basic_DefaultStart),勿漏 seq.append(start)
|
||||||
|
- 生成脚本 scripts/gen-word-flow.py;待用户运行验证(提醒:别开着报告预览,文件锁会干扰 Word COM)
|
||||||
|
|
||||||
|
## 修复:算术验证码识别"不明白"(16:15-16:22)
|
||||||
|
- 用户反馈算术验证码识别有问题。排查:OCR_CalculationCode/CommonEnglishNumbersCode 等 32 个 OCR 组件全部共用 remaining-core 通用实现(@gutenye/ocr-node PaddleOCR 整图 detect 返回原文)
|
||||||
|
- 实测(PIL 生成 5 张算术验证码图):干净图上 OCR 识别全对("3+5=?")——根因不是识别不出来,而是**返回算式原文没有计算结果**,流程无法直接填答案;真实验证码有干扰时符号还会被认成 x/X/*、全角
|
||||||
|
- 修复(新文件 captcha-caps.ts,last-wins 覆盖):OCR 原文 → 归一化(全角/×/长破折号/杂字符)→ parseCalculation 解析算式 → 计算并返回结果字符串;解析失败返回清洗原文+warn 日志;CommonEnglishNumbersCode 加字符清洗
|
||||||
|
- index.ts 加载顺序 remaining-core → captcha-caps;测试 captcha-caps.test.ts 12/12 通过(解析层 7 单测 + 5 张真实 OCR 端到端 8/5/24/43/81)
|
||||||
|
- 坑:vitest ESM 无 require;测试文件 import 顺序决定 last-wins 覆盖(remaining-core 必须在 captcha-caps 之前);ctx.log 级别是 warn 不是 warning
|
||||||
|
- 测试图生成:PIL+干扰线 D:/office-test2/ocr-test/captcha_*.png;应用已重启验证产物含新实现
|
||||||
|
|
||||||
|
## 解答:锁屏组件为啥要填用户名密码分辨率(16:46-16:50)
|
||||||
|
- 用户问 ScreenControl_Screenlock 为啥要填 username/pwd/width/height/scale_factor/port
|
||||||
|
- 真相:实现只是一行锁屏命令(LockWorkStation),不读任何参数;这些必填字段是 catalog.json 照抄原厂 RPA「远程会话锁屏」定义带进来的(原厂锁屏需账号密码重建远程会话+设定分辨率缩放比),本机锁屏用不上,但校验器强制填 → 纯干扰
|
||||||
|
- 修复:catalog.json 里 Screenlock fields 清空 + 补 label「锁屏」;ScreenUnlock 补 label「解锁屏幕(仅唤醒,登录锁屏需手动输入密码)」
|
||||||
|
- 顺手:captcha-caps.ts 动态 import 改静态(消除 vite 打包警告),回归 12/12 通过
|
||||||
|
- catalog 启动时加载,已重启应用生效
|
||||||
|
|
||||||
|
## 全量参数审计:假必填清零(16:52-17:00)
|
||||||
|
- 用户追问"还有没有没用但要填的参数 / 有用但没有的参数" → 写 scripts/audit-catalog-params.py 全量对比 catalog 字段 vs 实现读取
|
||||||
|
- 审计方法:正则提取每个 register handler 函数体,收集 text(args,["k"])/args.k/pick 等直接读取 keys;args 整体传走标 indirect(keys 未知,不判);必填 key 再用全文兜底(带引号 + .key 词边界两种形式)
|
||||||
|
- 结论 A:**96 个假必填字段(72 组件)已全部改为选填**——最大族 web×31(WebBrowser_* 实现固定用当前会话)、designated_obj×23(元素句柄,校验器本来就有豁免)、accuracy2×3(图像查找只读 accuracy)、emails×3、其余散布(StrUtil 系列/Word_SaveAs fileAddr/Trigger.entryName 等)
|
||||||
|
- 结论 B:**"有用但目录没有的参数"为 0**——实现读取的 key 都被目录覆盖(保守审计),已知例外是实现缺失(如 Word_SaveAs 没真正另存)而非缺字段
|
||||||
|
- 坑:审计脚本初版误报 313 个——间接调用(selectorOf(args))读的 keys 提取不到;args.min_second 点号访问无引号,兜底正则必须两种形式都查
|
||||||
|
- 校验器 requiredInContext 已有条件豁免(designated_obj 仅 designated_info 开启时必填),目录改选填后 UI 红星同步消失
|
||||||
|
- 审计脚本留在 scripts/,后续新增组件可复跑防回归
|
||||||
|
|
||||||
|
## 项目清理(17:05-17:12)
|
||||||
|
- 用户要求清理无用文件/文档;扫描后分四组征求确认
|
||||||
|
- 直接删:根目录 3 个旧测试日志(.phase3-test2.log/.phase4-test.log/vitest-full.log)、.screenshots 空目录
|
||||||
|
- 用户确认删:temp/ 415MB(AI 拾取模型选型实验,代码零引用)、scripts/ 4 个无引用调试脚本(spawn-test/cdp-probe/desktop-e2e/export-last-run)、docs/ 8 份历史规划文档(01~06 编号系列+OCR 检测报告,git 历史可查)
|
||||||
|
- 用户选择保留:release/dist 1.8GB 打包产物
|
||||||
|
- README 有 5 处悬空引用(4 个文档链接 + cdp-probe),已同步改为三份活跃文档索引
|
||||||
|
- 提交 ec25db2,13 files、-3122 行;文档删前务必 grep 交叉引用
|
||||||
|
|
||||||
|
## git 追踪审查与模型移除(17:15-17:30)
|
||||||
|
- 审查 git 追踪的大文件:仅 resources/models 两个 onnx 共 176MB 占仓库 pack 148MB 大头;用户选择继续追踪、不改历史
|
||||||
|
- 顺带发现打包 bug:extraResources 漏拷 ui-icon-detect(5a3d4cb 补上)
|
||||||
|
- 用户随后确认 ui-icon-detect/model.onnx(80MB)无实际使用(仅老安装包兜底回退,webpage-element 永远先命中)→ 整目录删除 + 运行时去掉兜底候选(79bba42),打包拷贝行随之移除
|
||||||
|
- 坑:localUiModelPath 有第三顺位回退 app.getAppPath()/temp/models,指向刚删的 temp/ 目录——但 resources/models 优先级更高且模型齐全,dev 不受影响
|
||||||
|
|
||||||
|
## 仓库完整性核查(17:28-17:35)
|
||||||
|
- 用户问"仓库是否包含所有开发能用得上的" → 全面核查
|
||||||
|
- 结论:是。代码/文档/脚本/catalog/模型(webpage-element)/fixture/测试流程全在 git;node_modules、out、release 为可再生产物已忽略
|
||||||
|
- 发现并修复:captcha 端到端测试图原依赖仓库外 D:/office-test2(fresh clone 自动跳过跑不到)→ 5 张图(30KB)入库 __fixtures__/captcha,测试改 import.meta.dirname 解析,12/12 通过(dbe9894)
|
||||||
|
- 环境依赖(仓库外、有文档说明):RPA_ELECTRON_DIST(dev:local/start/demo-e2e 需要,因工作区 ACL 沙箱问题);真实桌面冒烟测试需 Windows 桌面(无 skip 守卫,属设计)
|
||||||
|
- ocr-smoke 的 3 张旧 fixture 图已不存在,该 describe 长期 skip,可考虑后续清理
|
||||||
|
|
||||||
|
## 迁入指南 V2 修订(次日 08:50-09:10,实为 11-06 会话)
|
||||||
|
- 用户要求按《RPA项目技术审查与Agent-Employee集成实施总方案》+真实源码全面修订迁入指南成 V2,只改文档不改源码
|
||||||
|
- 双侧并行探查核查:RPA 侧 14 项、AE 侧 10 项;关键人工复核(defaultEngine 捕获、driver 无 signal)
|
||||||
|
- 重大发现:AE 底座巨变——workflow-engine/capability-gateway/browser-plugin(Playwright!)/windows-runner(C# FlaUI observe-only) 都已实现;"空壳契约"叙事只剩 computer-control Runner 和 WindowsRunner.click/type
|
||||||
|
- RPA 侧确认:createEngine() 仍无注入参数、desktop.ts:9 模块加载时捕获 driver(非 win32 import 即抛)、5 处动态 electron import、check-deps 正则盲区实测、catalog 3 个 platformTag=1 脏值
|
||||||
|
- 交付:docs/rpa-core-migration-guide-v2.md(架构图/模块对应表/接入点 A1-A10 七要素/0-6 阶段/10 风险/10 验收/6 ADR+3 Q 待确认)+ v2-diff.md(V1 逐项判定 + 审查报告 12 条修正)
|
||||||
|
- 提交 a8f75c6 + 收录审查报告一笔
|
||||||
|
|
||||||
|
## ADR-R3/R6 拍板(11-06 09:40)
|
||||||
|
- 用户确认两项关键决策:①桌面点击逻辑改写进对面 C# 原生进程(PowerShell 通道仅限 demo);②按能力域拆分部署(纯 Node 域→backend plugin-worker,桌面/视觉/弹窗域→用户电脑 Electron 宿主)
|
||||||
|
- V2 文档已更新:§0 结论、§10 拆"已拍板/待评审"、A6 改哪侧、阶段 0 范围、风险 R1/R2 降级为已缓解
|
||||||
|
- 剩余 4 ADR(R1/R2/R4/R5)+ 3 Q 留待与对面对评审
|
||||||
|
|
||||||
|
## ADR-R2 拍板(11-06 10:00)
|
||||||
|
- 方案 A + 两补丁:RPA 流程=单能力 rpa.flow.execute;节点事件走 event-bus 旁路;人审/高危节点升级对面审批。B 触发条件留档(仅当产品需逐步控制)。不用 B 的三层理由(双账本/控制流节点无副作用/审批重试粒度)已写入 §10。
|
||||||
|
- 至此 6 项 ADR 全部拍板,仅剩 Q1/Q2 待对面确认。6-阶段计划可开工,阶段 1(RPA 侧收口)不依赖对面。
|
||||||
|
|
||||||
|
## 迁移阶段1开工(11-06 10:03-11:10)
|
||||||
|
- A3(9b76dc6):节点尝试级 AbortController(run取消+节点超时汇聚),超时真中止底层;取消映射 NodeState=cancelled(不适用 skipOnError/不进异常分支);CapabilityMeta.idempotent 门控重试;graph-validator.ts(重复ID/悬空边/自环/环/孤儿/不可达)runFlow 前置校验;11 新测试
|
||||||
|
- A1(1da0082):createEngine({driver})+setDriver;4 个能力文件模块级 driver 捕获改惰性 getDriver()(非 win32 import 不再炸)
|
||||||
|
- A9(1a20dd7):HostBridge.screen 服务(listDisplays/primaryDisplay/dipToScreenPoint/displayMatching/captureDisplay),vision/cv-vision 5 处动态 electron import 清零;demo 宿主实现 screen 服务;check-deps 升级覆盖动态 import(实测合成命中1/真实0);包依赖补 3 项
|
||||||
|
- T4(5e8fc96):修复昨日 catalog 重生成遗留回归——对面旧版源数据缺 select 选项,从旧快照收割 782 个 options 进 overrides;移除 InsertColumn/RowV3 假字段 direction;Time 系列 offset_unit_num 默认改 0(实现按 num 读);Mail attachment 恢复选项;app-system-tests 12 号流程期望更新(假必填清理后不再阻塞)
|
||||||
|
- rpa-cli 重写:显式 5 域注册 + engine.execute 语义真实入口
|
||||||
|
- 全量回归:390 过;唯一失败是 mouse-moveto-smoke 真桌面测试被用户鼠标干扰的环境抖动(两次全量失败用例不同=铁证),要求独占桌面
|
||||||
|
- 教训: harvest overrides 时只对齐了 required/label,漏了 options——重生成前必须全字段 diff 而不是抽查
|
||||||
88
.workbuddy/memory/2026-11-06.md
Normal file
88
.workbuddy/memory/2026-11-06.md
Normal file
@ -0,0 +1,88 @@
|
|||||||
|
# 2026-11-06 工作日志
|
||||||
|
|
||||||
|
## ADR-R2 补记:A++ 检查点协议(拍板)
|
||||||
|
- 用户确认在方案 A 基础上采用 A++:节点事件流从"透明度日志"升级为检查点协议(nodeId/status/checkpointId/idempotencyKey/resumable),崩溃后 rpa.flow.execute 带 resumeToken 重入断点续跑,权威账本仍为单 step。不动 A、不启用 B。
|
||||||
|
- 业界对照论据(可与对面沟通用):UiPath Orchestrator≈A(Job 黑盒+日志旁路);Camunda/Zeebe External Task 的单引擎假设不适用于双引擎集成(B 被否的根因);AWS Step Functions callback≈补丁 2;Temporal 理论最干净但需重写,不采纳。
|
||||||
|
- 两侧改动清单落档 §7-A5:RPA 侧=事件负载字段+resumeToken 重入+幂等键贯通(复用 A3 idempotent 门控);对面侧=step 允许带 token 重入不改租约语义+event-bus 识别检查点+workflow-recovery 回传 token。
|
||||||
|
- 落地节奏:阶段 2 只定字段设计(近零成本),阶段 3 垂直切片最小验证,阶段 4/5 强制。
|
||||||
|
|
||||||
|
## 文档已完成项标定
|
||||||
|
- §0 核心结论第 2 条重写:P0 整改已完成(原表述"大部分未解决"已过时);仅剩 bun 冒烟+包依赖 3 项核实。
|
||||||
|
- §7 A1/A3/A9 方案行标 ★ 已实现(2026-11-06 阶段 1)。
|
||||||
|
- §8 阶段 1 标 ✅ 已完成(提交 9b76dc6/1da0082/1a20dd7/5e8fc96);阶段 2 加"A++ 字段设计定稿";阶段 3 加"检查点续跑最小验证"。
|
||||||
|
- §9 R8 缓解列补 A++ 统一崩溃恢复语义。
|
||||||
|
- 提交:d99be8f。
|
||||||
|
|
||||||
|
## bun 冒烟(2026-11-06 下午补)
|
||||||
|
- bun 已装于 C:\Users\Administrator\.bun\bin\bun.exe(版本 1.4.4,PATH 未刷新需全路径调用;对面锁 1.4.3 为 patch 级差异)。
|
||||||
|
- 纯逻辑冒烟通过:bun test packages/rpa-core/src/engine/ + excel-common/excel-store/pdf-high = 37/37 全绿,exceljs/pdf 行为与 Node 无差异。
|
||||||
|
- 注意:engine 测试必须从仓库根目录跑(catalog.json 按 cwd 解析,包内跑必挂 2 个),与 npm 侧既有 cwd 问题一致。
|
||||||
|
- PowerShell 编码差异项留待阶段 2 迁包后验证(R6 已标"部分缓解")。提交:b18d5b1。
|
||||||
|
|
||||||
|
## Word_Save/Word_SaveAs 真实现(2026-11-06)
|
||||||
|
- word.ts:44 占位(只回显路径)替换为真实现:Word_Save=存在性校验+打开即存;Word_SaveAs=打开原文档→SaveAs2(目标)→返回新路径句柄。
|
||||||
|
- 参数与 catalog 对齐:objWord(必填)+fileAddr(目录,空则沿用原目录,相对时基于句柄当前目录解析)+filename(空则沿用原名,无扩展名补 .docx)。
|
||||||
|
- 新增 COM 用例(word-extended.test.ts):另存新目录+改名/空名沿用/相对目录/Save 拒绝不存在句柄,4/4 全绿。注意:相对目录基于句柄当前目录(与 Excel 一致),首版测试预期写错已修正。
|
||||||
|
- 提交:4b5c73b。
|
||||||
|
|
||||||
|
## catalog 3 个数字脏值清洗(2026-11-06)
|
||||||
|
- 脏值:Basic_BusinessOrderCount / Dialog_GetConfigV5 / Dialog_GetConfigV7 的 platformTag=数字 1,源头是上游 shizai 数据(源数据分布 null=994 / 1=3)。
|
||||||
|
- 修复:build-catalog.mjs 白名单归一化(非 win32/darwin/linux 字符串一律重派生,A8 方案落地);三条均为包内真实现,重派生为 win32。
|
||||||
|
- 验证:重生成后 win32=690/null=326/总数 1016 不变;全字段 diff 仅 3 条仅 platformTag 变化(overrides 无损);platform-groups+register-all 测试 6/6。
|
||||||
|
- 注意:Git Bash 的 /tmp 与 node 路径映射不一致(node 把 /tmp 解析为 <盘>:\tmp),快照 diff 用 cygpath 取真实路径。
|
||||||
|
- 提交:2bcda3a。
|
||||||
|
|
||||||
|
## 两个接入前补丁完成(2026-11-06,a01667b)
|
||||||
|
- 对面源码在 D:\code\Agent-Employee(backend/packages/...),StepStatus 8 态(workflow-engine/src/index.ts:37),StepRun 带 idempotencyKey(来自租约);对面明确"重启后中断步骤走恢复流程不得续跑"——与 A++ resumeToken 重入设计吻合。
|
||||||
|
- 补丁 1:NodeState 补 waiting_user/unknown 两态;导出 toStepStatus/NODE_STATUS_TO_STEP_STATUS 全量映射(唯一非同名 idle↔pending)。
|
||||||
|
- 补丁 2:RunEvent 增 checkpointId/resumable/idempotencyKey;runtime 在 node.finished 成功时填充(checkpointId=node.id,幂等键=runId:nodeId 跨重入稳定,meta.idempotent===false 不作为锚点)。契约测试 checkpoint-events.test.ts 4 用例。
|
||||||
|
- 附带:sm-crypto.d.ts 类型声明补齐(typecheck 归零,此前 basic-encryption 一直报 TS7016)。
|
||||||
|
- engine 全量 31→33 用例全绿;账面:本仓库代码活清零,剩余全为对面配合项。
|
||||||
|
|
||||||
|
## 遗留待办
|
||||||
|
- ocr-smoke 清理:用户拍板"算了"(2026-11-06),维持现状——素材缺失时整 describe 静默跳过;D:/office-test2 硬编码保留。后续若迁移对面仓库需重提。
|
||||||
|
- bun@1.4.3 安装+冒烟(阶段 2 前置,上次因用户在用电脑未装)
|
||||||
|
- Word_SaveAs 真实现、晨间助手截图时机(待对面确认)、ocr-smoke 清理、catalog 3 个数字脏值清洗
|
||||||
|
- 跑全量测试前先询问用户(占桌面)
|
||||||
|
|
||||||
|
## 对面更新核查(13:30)
|
||||||
|
- windows-runner 自 10-22 未变;workflow-engine 代码未变——既有可行性结论全部有效
|
||||||
|
- 对面新增事实一致性门禁(document-facts.mjs + 当前事实.json):代码/事实/文档区块须同提交,CI 拦截
|
||||||
|
- 对面 README Windows 行明确下一步"click/type、视觉/OCR 或随本轮交付"——R3 路线吻合
|
||||||
|
- V2 补记门禁合规要求(407aa38)
|
||||||
|
|
||||||
|
## C# actuate 原型补丁包(65343da)
|
||||||
|
- 产出 contrib/windows-runner-actuate/:PORTING-MAP.md(逐原语对照+3 语义差异+错误码映射)、ActuateProtocol.cs、NativeInput.cs(五招兜底逐行移植)、ActuateService.cs(安全门同构+前后证据)、PATCH-NOTES.md(Program.cs 最小 diff+门禁合规清单)
|
||||||
|
- 关键设计:NativeWindows 只读边界不动(输入 P/Invoke 独立);mouse_event→SendInput;模糊→精确匹配由 TS 宿主 observe 弥合
|
||||||
|
- 未编译:本机无 dotnet SDK;编译验证待装 .NET 10 SDK 或对面侧首次合入时
|
||||||
|
|
||||||
|
## .NET SDK 安装 + 编译验证(13:40)
|
||||||
|
- winget 装 .NET SDK 10.0.403(对面 net10.0-windows 匹配)
|
||||||
|
- 编译方式:对面 native 拷贝到 .workbuddy/tmp/actuate-build + 三件套 + PATCH-NOTES §2 最小 diff,不碰对面仓库
|
||||||
|
- dotnet build 0 警告 0 错误(TreatWarningsAsErrors 开启)
|
||||||
|
- 编译修正 2 处:NativeInput.Point private→internal(CS0050)、ActuateService 补 using System.Diagnostics(CS0103)→ 提交 f0defc2
|
||||||
|
- 零桌面交互协议冒烟 5 例:policy_invalid/desktop_session_mismatch/observe 无回归/requestId 回显 全过;越权窗口/非法动作先报 window_not_found(窗口选择先于动作校验,与 observe 顺序一致,预期行为)
|
||||||
|
- 真实窗口用例(click/type/activate)留对面 test/fixture
|
||||||
|
|
||||||
|
## 远端核查 + port 9222 保护(13:50)
|
||||||
|
- 远端无强推、无被删文件回填:33b85c0/1b952e3 是"yu"的合并提交,另一侧只有 93915df(catalog 远程连接端口 61619→9222)
|
||||||
|
- 发现陷阱:9222 只改了生成产物,未进 overrides,重生成会打回 61619(已发生过一次,故 merge 标题有 re-apply)
|
||||||
|
- 修复:WebBrowser_Create/V2/V3 的 port 字段(defaultValue+description)写入 catalog-overrides.json,重生成验证 1016 条零差异
|
||||||
|
- 本地领先远端多笔,待推送
|
||||||
|
|
||||||
|
## 拍板:晨间助手截图时机搁置(15:00)
|
||||||
|
- 用户拍板"先不管",提案 3(挂证据节奏方案)保留在 next-work-plan.md 供未来重启,不再作为工作项/对面对确认项
|
||||||
|
|
||||||
|
## 拍板:双端透明调度(15:25,81ce4bc)
|
||||||
|
- 用户拍板:同一能力前端(用户电脑宿主)与后端(plugin-worker)两驻地并存,调度对用户无感知,SaaS/本地部署通用
|
||||||
|
- 防乱设计三纪律写入 V2 §10 R6 补记:①一个脑决策(宿主 Resolver,registry handler 绑定本地实现或远端代理,流程/能力代码禁写驻地 if-else);②一份契约两处实现(同一套契约测试,代理自动生成防漂移);③数据不动原则(代码就数据;搬数据必须显式节点,路由器禁隐式搬运本地文件——隐私红线)
|
||||||
|
- 确定性路由表 6 条:桌面→仅本地;数据在哪就在哪算;双副本默认本地可 prefer backend;宿主离线可后端兜底;均不可用报 location_unavailable
|
||||||
|
- 代码落点:RunEvent 增 executor: "local"|"backend" 字段(用户不展示、审计必须可见),runtime 成功事件恒发 "local",契约测试覆盖;远端代理落地后由代理覆盖该字段
|
||||||
|
|
||||||
|
## RPA 线4 迁包完成(16:20,对面 feat/rpa-core-migration 已推远端)
|
||||||
|
- 对面 main 新增三笔治理提交:第五条线主控审查与合并流程、多线并行规则、PR/commit 模板(功能线含"RPA 插件");规则=不直接合 main、分支专人审核
|
||||||
|
- 迁包两笔提交:8085ff47(包+catalog/overrides/duplicates 资源+bun.lock+事实刷新)、4e345304(测试移 test/ 目录+compat-audit 报告改写 tmpdir)
|
||||||
|
- 关键发现与处置:①bun 1.4.4 打不动对面 patch(photon-node hunk#10),必须 bun 1.4.3(npm i -g bun@1.4.3)→ 649 包安装成功;②对面 workspace 规范禁测试文件在 src/ → 39 个测试移 test/,脚本重写导入(含副作用导入两轮修正);③platform-groups 测试预载顺序耦合修复(退化分支);④compat-audit 报告改写 os.tmpdir 防脏树
|
||||||
|
- 验证:对面环境 bun test 48/48 全绿(bun 1.4.3);check-workspace rpa-core 项清零;document-facts PASS(顺手把 main 既有漂移一并刷新)
|
||||||
|
- 遗留:opencode fork 快照漂移=对面 main 旧账(apps/desktop 大量 changed,需他们跑 check-opencode-fork --update);rpa-adapter 未动(依赖 Q4 每节点授权拍板已定、Q5 审批上游待确认)
|
||||||
|
- 本仓库同步提交 4f1fbc0(测试迁移+两处测试修复)
|
||||||
22
README.md
22
README.md
@ -1,16 +1,16 @@
|
|||||||
# RPA Studio
|
# RPA Studio
|
||||||
|
|
||||||
画布式 RPA 桌面端(Electron + React Flow)。把 1016 项能力目录拖到画布上,连线、配置、运行,生成可执行的 RPA 流程。
|
画布式 RPA 桌面端(Electron + React Flow)。把 658 项主能力拖到画布上,连线、配置、运行,生成可执行的 RPA 流程。
|
||||||
|
|
||||||
> 当前状态:任务优先的编排工作台。默认进入“编排”任务列表,编辑任务后进入上下流向画布;组件库按分类树展示,设置为独立入口。
|
> 当前状态:任务优先的编排工作台。默认进入“编排”任务列表,编辑任务后进入上下流向画布;组件库按分类树展示,设置为独立入口。
|
||||||
> 可执行能力持续接入中:**732 / 1016**(网页自动化、桌面自动化、数据、文件、办公、HTTP 等),其余灰标「待接入」,运行前检查会提前定位。
|
> 当前可编排能力:**658 / 894**(122 项不接入和 236 项重复旧版本已分离;网页、桌面、数据、文件、办公、HTTP 等主能力可直接编排)。
|
||||||
|
|
||||||
## 已实现
|
## 已实现
|
||||||
|
|
||||||
### 编排与运行
|
### 编排与运行
|
||||||
|
|
||||||
- Electron 31 + React 18 + TypeScript + React Flow 画布
|
- Electron 31 + React 18 + TypeScript + React Flow 画布
|
||||||
- 1016 项组件目录:搜索 / 11 组筛选 / 可执行与待接入状态;任务列表实时刷新
|
- 658 项主能力目录:搜索 / 11 组筛选;236 项重复旧版本自动隐藏,122 项专用能力已分离备份;任务列表实时刷新
|
||||||
- 拖拽到画布、节点卡片、连线、缩放/平移/MiniMap、框选、撤销/重做
|
- 拖拽到画布、节点卡片、连线、缩放/平移/MiniMap、框选、撤销/重做
|
||||||
- IF / 计次循环 / 遍历 / 条件循环 / 异常捕获 容器:可把组件拖入分支或循环体
|
- IF / 计次循环 / 遍历 / 条件循环 / 异常捕获 容器:可把组件拖入分支或循环体
|
||||||
- 属性面板:按字段类型渲染输入、必填、变量、失败策略(超时/重试/间隔/跳过/延时)
|
- 属性面板:按字段类型渲染输入、必填、变量、失败策略(超时/重试/间隔/跳过/延时)
|
||||||
@ -42,7 +42,8 @@
|
|||||||
|
|
||||||
### 质量与测试
|
### 质量与测试
|
||||||
|
|
||||||
- 全量测试 206 项(能力注册冒烟、流程运行集成、组件缺陷回归、测试流程包与安装保护、Word/Excel/PDF/桌面桥真实设备测试)
|
- 全量测试 231 项(能力注册冒烟、流程运行集成、组件缺陷回归、测试流程包与安装保护、Word/Excel/PDF/桌面桥真实设备测试)
|
||||||
|
- 应用与系统 50 项组件验收:13 组任务、75 个用例,含真实执行与人工确认项
|
||||||
- 本地演示站(`127.0.0.1:17863`):网页订单页 + 网页元素演练场(表单/下拉/勾选/密码/禁用按钮/悬停/长页面)
|
- 本地演示站(`127.0.0.1:17863`):网页订单页 + 网页元素演练场(表单/下拉/勾选/密码/禁用按钮/悬停/长页面)
|
||||||
- 内置 3 个网页自动化测试任务(浏览器导航、元素读写、滚动与 Cookie)+ 6 条固定示例任务
|
- 内置 3 个网页自动化测试任务(浏览器导航、元素读写、滚动与 Cookie)+ 6 条固定示例任务
|
||||||
- 打包冒烟测试:跳过单实例锁隔离启动,验证产物可用性
|
- 打包冒烟测试:跳过单实例锁隔离启动,验证产物可用性
|
||||||
@ -72,20 +73,21 @@ src/main/fixture 本地演示站
|
|||||||
src/preload 白名单桥
|
src/preload 白名单桥
|
||||||
src/renderer React UI:画布、组件库、属性面板、运行/设置页
|
src/renderer React UI:画布、组件库、属性面板、运行/设置页
|
||||||
src/shared 主/渲染共享类型
|
src/shared 主/渲染共享类型
|
||||||
scripts build-catalog / dev-safe / start / cdp-probe / 打包冒烟
|
scripts build-catalog / dev-safe / start / 打包冒烟
|
||||||
resources catalog.json(1016 项派生元数据)、夹具站(含元素演练场)、桌面桥
|
resources catalog.json(1016 项派生元数据)、夹具站(含元素演练场)、桌面桥
|
||||||
docs 现状分析、实施方案
|
docs 开发规范、迁移指南、重构任务清单
|
||||||
```
|
```
|
||||||
|
|
||||||
## 文档
|
## 文档
|
||||||
|
|
||||||
- [应用与系统 49 项组件测试任务](resources/test-flows/app-system/README.md):13 组可安装任务、使用步骤、覆盖清单及实际验收报告。
|
- [开发规范](docs/dev-conventions.md)
|
||||||
- [现状分析与可行性](docs/01-现状分析与可行性.md)
|
- [rpa-core 迁移指南](docs/rpa-core-migration-guide.md)
|
||||||
- [实施方案(三天)](docs/02-实施方案.md)
|
- [重构任务清单](docs/rpa-core-refactor-tasks.md)
|
||||||
|
- [应用与系统 50 项组件测试任务](resources/test-flows/app-system/README.md):13 组可安装任务、使用步骤、覆盖清单及实际验收报告。
|
||||||
|
|
||||||
## 边界
|
## 边界
|
||||||
|
|
||||||
- 1016 项目录中当前 732 项可执行;未接入的主要为:异常监控组件、桌面 Ground 剩余变体、SAP 专用、邮件收件(IMAP)、Word 选区 COM 等。图表数据读取(ECharts)已接入;下载对话框为明确不接入(下载能力已覆盖)。运行前检查会提前定位待接入组件、必填参数、多出口与断链节点。
|
- 原始目录 1016 项中,894 项未被排除;其中 658 项主能力进入组件库,236 项重复旧版本在 `resources/capability-duplicates.json` 中标记兼容隐藏,122 项专用/厂商硬件/门户通道在 `resources/excluded.json` 中备份。组件库不显示接入状态。
|
||||||
- 内置本地演示站默认监听 `127.0.0.1:17863`;新用户首次启动会获得“本地网页订单演示”任务,可离线验证网页输入、点击和文本读取。
|
- 内置本地演示站默认监听 `127.0.0.1:17863`;新用户首次启动会获得“本地网页订单演示”任务,可离线验证网页输入、点击和文本读取。
|
||||||
- 编排首页提供“安装示例任务”,可安装网页订单、文件文本、Excel读写、剪贴板、Word文档和异常分支 6 条固定示例;重复安装会更新同名示例,不会无限复制。
|
- 编排首页提供“安装示例任务”,可安装网页订单、文件文本、Excel读写、剪贴板、Word文档和异常分支 6 条固定示例;重复安装会更新同名示例,不会无限复制。
|
||||||
- Tauri/WebView2 类应用的 UIA 支持天然受限(Invoke 假成功、ValuePattern 读回掩码):桌面元素点击采用“前台化 + 真实坐标点击”策略,输入采用“ValuePattern 写入 + 剪贴板粘贴兜底”。
|
- Tauri/WebView2 类应用的 UIA 支持天然受限(Invoke 假成功、ValuePattern 读回掩码):桌面元素点击采用“前台化 + 真实坐标点击”策略,输入采用“ValuePattern 写入 + 剪贴板粘贴兜底”。
|
||||||
|
|||||||
81
contrib/windows-runner-actuate/PATCH-NOTES.md
Normal file
81
contrib/windows-runner-actuate/PATCH-NOTES.md
Normal file
@ -0,0 +1,81 @@
|
|||||||
|
# PATCH-NOTES:合入对面 windows-runner 的集成指引
|
||||||
|
|
||||||
|
本目录是 **本地原型补丁包**,不改动对面仓库。合入时按以下步骤执行。
|
||||||
|
|
||||||
|
## 1. 文件落点
|
||||||
|
|
||||||
|
| 本目录文件 | 落到对面仓库 | 说明 |
|
||||||
|
|---|---|---|
|
||||||
|
| `native/ActuateProtocol.cs` | `code/backend/services/windows-runner/native/` | 新增类型,不动现有文件 |
|
||||||
|
| `native/ActuateService.cs` | 同上 | 新增类型,不动现有文件 |
|
||||||
|
| `native/NativeInput.cs` | 同上 | 新增类型;**NativeWindows.cs 保持只读边界原样不动**(对面注释明言该层无输入接口,编译边界得以保全) |
|
||||||
|
|
||||||
|
csproj 无需改动:`UseWindowsForms` 已启用(SendKeys/Clipboard/Screen 可用),FlaUI 引用与本补丁无关。
|
||||||
|
|
||||||
|
## 2. Program.cs 分发改造(最小 diff)
|
||||||
|
|
||||||
|
现有 Main 只认 `observe`。改造为按 Operation 分发:
|
||||||
|
|
||||||
|
```csharp
|
||||||
|
// 原:if (request.ProtocolVersion != 1 || request.Operation != "observe" || ...)
|
||||||
|
// 改为先解公共头再分发(保持 64KiB/UTF-8/UUID 校验全部不变):
|
||||||
|
if (request.ProtocolVersion != 1 || !Guid.TryParse(requestId, out _))
|
||||||
|
throw new RunnerException("request_invalid");
|
||||||
|
switch (request.Operation)
|
||||||
|
{
|
||||||
|
case "observe": { /* 原 observe 路径逐字保留 */ }
|
||||||
|
case "actuate":
|
||||||
|
{
|
||||||
|
var actuate = JsonSerializer.Deserialize<ActuateRequest>(line, JsonOptions)
|
||||||
|
?? throw new RunnerException("request_invalid");
|
||||||
|
var result = ActuateService.Actuate(actuate, evidenceRoot);
|
||||||
|
Write(new RunnerResponse(1, requestId, true, null, null, result));
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
default: throw new RunnerException("request_invalid");
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
RunnerResponse 需加一个可选字段(observe 的线上形状不变,`WhenWritingNull` 已保证):
|
||||||
|
|
||||||
|
```csharp
|
||||||
|
internal sealed record RunnerResponse(int ProtocolVersion, string RequestId, bool Ok,
|
||||||
|
ComputerObservation? Data, string? ErrorCode, ActuateResult? ActuateData = null);
|
||||||
|
```
|
||||||
|
|
||||||
|
## 3. TS 宿主侧(native-observer.ts → actuate 适配器,属后续工作)
|
||||||
|
|
||||||
|
- 请求侧:actuate 前先用 observe 解析**精确**窗口标题与进程名(RPA 侧"标题包含匹配"的语义在宿主层弥合,见 PORTING-MAP 第二节);
|
||||||
|
- 响应侧:对 ActuateData.Evidence 逐项复核(realpath 归属证据根目录、SHA-256、PNG 魔数——复用 native-observer.ts 现成校验);
|
||||||
|
- 高影响审批:`rpa.windows` manifest 已是 `type=highImpact + idempotencyRequired`,capability-gateway 的精确 actionId 放行与租约语义直接适用,**无需改 gateway**;
|
||||||
|
- 超时:宿主杀进程超时须覆盖 mouse_move_smooth 最坏路径(≈4s)+ 两次截图(≈2s),建议 ≥15s。
|
||||||
|
|
||||||
|
## 4. 验证计划(用对面现成夹具)
|
||||||
|
|
||||||
|
- `test/fixture/`(AgentWorker.WindowsFixture):仿照 `native-observer.integration.test.ts` 增加 actuate 集成测试:
|
||||||
|
1. `mouse_move` → GetCursorPos 断言落点;
|
||||||
|
2. `mouse_click`(无桌面断言,仅断言 ok + 证据 SHA 与文件存在);
|
||||||
|
3. `keyboard_send`(chars 模式)→ UIA 读回文本;
|
||||||
|
4. `window_activate` → GetForegroundWindow 断言;
|
||||||
|
5. 反向用例:越权窗口 → `window_not_allowed`;Session 不匹配 → `desktop_session_mismatch`;非法 Kind → `action_invalid`。
|
||||||
|
- 兼容性矩阵(微信/Office/UWP)按对面既有测试纪律单独排期,不在本补丁范围。
|
||||||
|
|
||||||
|
## 5. 门禁合规清单(对面 2026-11-06 新规,硬性)
|
||||||
|
|
||||||
|
合入提交必须**同一提交内**包含:
|
||||||
|
|
||||||
|
1. `项目资料/开发方案/当前事实.json`:factRevision 递增,trackedInputs 如受影响同步更新;
|
||||||
|
2. 相关 README 的机器生成区块(`node code/tooling/document-facts.mjs --write` 重新生成);
|
||||||
|
3. `code/backend/README.md` Windows 行状态更新("click/type" 从"或随本轮交付"改为已实现口径);
|
||||||
|
4. 提交前本地跑:`node code/tooling/document-facts.mjs --check` && `node code/tooling/ci-gate.mjs --mode fast` && `git diff --check`。
|
||||||
|
|
||||||
|
## 6. 验证状态与已知限制
|
||||||
|
|
||||||
|
- **已编译通过 + 协议冒烟通过(2026-11-06,.NET SDK 10.0.403)**:
|
||||||
|
- `dotnet build`(对面 csproj 原样 + 本补丁三文件 + §2 最小 diff):**0 警告 0 错误**(TreatWarningsAsErrors 开启,直接达到对面合入标准);
|
||||||
|
- 编译修正两处:`NativeInput.Point` 由 private 改 internal(CS0050 可访问性)、`ActuateService.cs` 补 `using System.Diagnostics;`(CS0103);
|
||||||
|
- 零桌面交互协议冒烟 5 例:`policy_invalid` ✅ / `desktop_session_mismatch` ✅ / observe 老路径无回归 ✅ / requestId 逐字回显 ✅;另 2 例(越权窗口/非法动作)因**窗口选择先于动作校验**的既有顺序先报 `window_not_found`——顺序与对面 observe 一致,属预期行为,用例已按此修正理解;
|
||||||
|
- 需要真实窗口的用例(click/type/window_activate/action_invalid 路径)留待对面 `test/fixture` 集成测试(第 4 节),避免占用用户桌面。
|
||||||
|
- `Clipboard.SetDataObject` 依赖 STA:对面 Main 已是 `[STAThread]`,满足;
|
||||||
|
- mouse_event → SendInput 为现代等价替换(标志位映射见 PORTING-MAP 第四节);如对面评审倾向零差异,可回退为 mouse_event P/Invoke,一行替换;
|
||||||
|
- 平滑移动公式中的伪随机抖动与 JS `%` 保号语义一致(C# `%` 对 double 同为保号),无漂移。
|
||||||
49
contrib/windows-runner-actuate/PORTING-MAP.md
Normal file
49
contrib/windows-runner-actuate/PORTING-MAP.md
Normal file
@ -0,0 +1,49 @@
|
|||||||
|
# PowerShell → C# 移植对照表(RPA 桌面驱动 → windows-runner actuate)
|
||||||
|
|
||||||
|
来源:`packages/rpa-core/src/driver/windows/windows-driver.ts`(269 行,内嵌 PowerShell)
|
||||||
|
目标:对面 `code/backend/services/windows-runner/native/`(原型见本目录 `native/`)
|
||||||
|
|
||||||
|
## 一、原语级映射
|
||||||
|
|
||||||
|
| RPA 侧 PowerShell 原语 | 用途(驱动行号) | C# 等价物(原型落点) | 语义保真要点 |
|
||||||
|
|---|---|---|---|
|
||||||
|
| `SetCursorPos(x,y)` | moveTo / click / drag / moveSmooth(L142,150,158,173) | `NativeInput.SetCursorPos` | 物理像素;进程入口声明 Per-Monitor V2 DPI(对应 PS 每脚本的 `SetProcessDpiAwarenessContext(-4)`,一次声明进程级生效) |
|
||||||
|
| `mouse_event(flags,0,0,data,0)` down/up | click / drag(L150,158) | `SendInput`(MOUSEEVENTF_LEFTDOWN=0x0002 / LEFTUP=0x0004 / RIGHTDOWN=0x0008 / RIGHTUP=0x0010) | SendInput 是 mouse_event 的现代等价;标志位一一对应。双击=repeats 2,settleMs 逐次保留 |
|
||||||
|
| `mouse_event(0x0800,...,amount)` | scroll(L161) | `SendInput` + MOUSEEVENTF_WHEEL,mouseData=有符号 delta | 负值向下滚,`unchecked((uint)amount)` 保持位型一致 |
|
||||||
|
| `GetCursorPos` + JS `mousePath` 插值 | moveSmooth(L163-174) | `ActuateService.SmoothMovePath`(公式逐行移植) | 缓入缓出二次曲线 + 垂直抖动 `sin(i*12.9898)*43758.5453 % 1`;stepCount=clamp(distance/12, 6, 50);delay=clamp(1500/speed, 4, 80)ms。JS 与 C# 的 `%` 对负数同为保号语义,公式无漂移 |
|
||||||
|
| `Set-Clipboard` + `SendWait('^v')` | keyboard clipboard 模式(L183) | `Clipboard.SetDataObject(text, true)` + `SendKeys.SendWait("^v")` | 120ms 间隙保留;STA 已由对面 `[STAThread]` 保证 |
|
||||||
|
| `SendWait($c)` 逐字符循环 | keyboard manual 模式(L188) | 逐字符 `SendKeys.SendWait(Escape(ch))` + `Thread.Sleep(intervalMs)` | intervalMs 逐字符保留 |
|
||||||
|
| `SendWait(escaped)` 整段 | keyboard 默认模式(L192) | `SendKeys.SendWait(escaped)` | `sendKeysEscape` 字符集 `+^%~(){}[]` → `{+}` 等花括号转义,逐字符同规则移植(`ActuateService.EscapeSendKeys`) |
|
||||||
|
| `activateWindowScript` **抢前台五招兜底** | focus(L73-105,键盘与激活共用) | `NativeInput.FocusChain`(逐行移植) | 顺序与睡眠严格保留:①IsIconic→SW_RESTORE(9)+150ms ②SwitchToThisWindow+150ms ③ALT(0x12 down/up)+SetForegroundWindow+200ms ④AttachThreadInput 对线程+SetForegroundWindow+200ms ⑤SW_MINIMIZE(6)+150ms→SW_RESTORE(9)+300ms ⑥仍失败→稳定错误码 `focus_failed`(PS 版走 `RPA_ERROR::` stdout 通道) |
|
||||||
|
| `Get-Process | MainWindowTitle -like *title*` | 窗口查找(L60-68 等) | `Win32 EnumWindows` + 精确标题 + PID 回查进程名 + Session 校验(复用对面 observe 的 SelectWindow 模式) | **唯一语义变更**,见下节 1 |
|
||||||
|
| `GetWindowRect` + JSON 输出 | windowObject / foregroundPosition(L66,213) | `NativeWindows.GetWindowRect`(对面已有,直接复用) | RECT right/bottom → width/height 换算保持 |
|
||||||
|
| `ShowWindow(h, 3/6/9/cmd)` | activate/maximize / setState / 五招内部(L203,224) | `NativeInput.ShowWindow` | 命令值原样透传 |
|
||||||
|
| `SetWindowPos(h,0,x,y,w,h,4)` | setSize / setPos(L227,231) | `NativeInput.SetWindowPos` | flags=4(SWP_NOZORDER)原样保留;尺寸>0 校验保留(L230) |
|
||||||
|
| `Process.CloseMainWindow()` | close(L221) | `PostMessage(h, WM_CLOSE=0x0010)` | 等价语义(礼貌关闭,不杀进程) |
|
||||||
|
| `System.Windows.Forms.Screen.PrimaryScreen.Bounds` | screen.size(L246) | 不移植——observe op 的 CoordinateSpace 已携带窗口矩形+DPI | 避免重复采集 |
|
||||||
|
|
||||||
|
## 二、必须知道的三处语义差异(宿主适配层负责弥合)
|
||||||
|
|
||||||
|
1. **窗口匹配从模糊变精确**。PS 版 `Get-Process -like *title*` 是包含匹配;对面安全政策要求 WindowTitle 与 AllowedWindows **精确相等**(模糊匹配会把同名网页/提示框误当目标)。弥合方式:TS 宿主适配器先用 observe 解析出精确标题与进程名,再发起 actuate 请求——RPA 能力层(`window.findByTitle`)语义不变,只是解析下沉到宿主。
|
||||||
|
2. **进程模型从"每命令一进程"变"单请求单响应"**。PS 版每个原语起一个 powershell.exe;C# 版一次请求一次响应、宿主超时杀进程(对面现行架构)。RPA 流程里相邻原语本来也无状态依赖(除句柄传递,改由标题精确匹配承担),语义等价。注意:宿主超时必须覆盖 moveSmooth 最坏情况(约 4s)+ 两次证据截图。
|
||||||
|
3. **副作用证据从无到有**。PS 版动作无留痕;C# 版按对面证据体系在动作**前后各截一张**目标窗口 PNG(PrintWindow,SHA-256 落盘,默认 secret)——这是对面 capability-gateway 审计链要求的,属于增强而非等价项。
|
||||||
|
|
||||||
|
## 三、稳定错误码映射(PS 侧 → C# 侧)
|
||||||
|
|
||||||
|
| PS 侧失败 | C# actuate 稳定错误码 |
|
||||||
|
|---|---|
|
||||||
|
| `找不到窗口:<title>`(throw/RPA_ERROR) | `window_not_found`(无候选)/ `window_ambiguous`(多候选,对面已有语义)/ `window_not_allowed`(不在白名单) |
|
||||||
|
| `无法把窗口带到前台…`(五招全失败) | `focus_failed` |
|
||||||
|
| 输入注入 Win32 失败 | `input_failed` |
|
||||||
|
| 证据截图失败/空白 | `capture_unavailable`(沿用对面现有码) |
|
||||||
|
| 政策校验失败 | `policy_invalid` / `desktop_session_mismatch`(沿用对面现有码) |
|
||||||
|
|
||||||
|
## 四、开关映射:mouse_event → SendInput 标志位
|
||||||
|
|
||||||
|
| mouse_event flag | SendInput MOUSEEVENTF | 用途 |
|
||||||
|
|---|---|---|
|
||||||
|
| 0x0002 | LEFTDOWN | 左键按下 |
|
||||||
|
| 0x0004 | LEFTUP | 左键抬起 |
|
||||||
|
| 0x0008 | RIGHTDOWN | 右键按下 |
|
||||||
|
| 0x0010 | RIGHTUP | 右键抬起 |
|
||||||
|
| 0x0800 | WHEEL | 滚轮(data=delta,120 的倍数) |
|
||||||
31
contrib/windows-runner-actuate/README.md
Normal file
31
contrib/windows-runner-actuate/README.md
Normal file
@ -0,0 +1,31 @@
|
|||||||
|
# windows-runner-actuate 原型补丁包
|
||||||
|
|
||||||
|
**用途**:把 RPA 桌面驱动(`packages/rpa-core/src/driver/windows/windows-driver.ts`,PowerShell 实现)的鼠标/键盘/窗口执行能力移植为对面 Agent-Employee `windows-runner` 的 C# 原生实现——即 ADR-R3(C# 路线)的第一块实体。
|
||||||
|
|
||||||
|
**状态**:本地原型,未合入对面主干。对面 backend/README Windows 行已明确下一步为"click/type、视觉/OCR 或随本轮客户端交付",本补丁正是其路线图的第一步。
|
||||||
|
|
||||||
|
## 内容清单
|
||||||
|
|
||||||
|
| 文件 | 内容 |
|
||||||
|
|---|---|
|
||||||
|
| `PORTING-MAP.md` | PowerShell → C# 逐原语移植对照表:三处语义差异(模糊→精确匹配、进程模型、证据增强)、错误码映射、mouse_event→SendInput 标志位映射 |
|
||||||
|
| `native/ActuateProtocol.cs` | actuate 协议扩展类型(请求/动作/结果),复用对面 DesktopPolicy/EvidenceRef/ObservedWindow |
|
||||||
|
| `native/NativeInput.cs` | 输入类 P/Invoke + **抢前台五招兜底逐行移植**(RPA 实战参数,一行语义不改) |
|
||||||
|
| `native/ActuateService.cs` | 执行服务:安全门与 observe 同构(白名单/Session/精确匹配),10 种动作闭集分发,动作前后证据截图,平滑移动公式逐行保真 |
|
||||||
|
| `PATCH-NOTES.md` | 合入指引:Program.cs 最小 diff、TS 宿主适配要点、验证计划、**门禁合规硬性清单** |
|
||||||
|
|
||||||
|
## 设计红线(对面安全模型的尊重项)
|
||||||
|
|
||||||
|
1. `NativeWindows.cs` 只读编译边界不动——输入类 P/Invoke 独立成 `NativeInput.cs`;
|
||||||
|
2. 安全门一个不少:进程文件名白名单正则、窗口精确匹配、OS Session 三重核验后才允许输入;
|
||||||
|
3. 副作用证据:动作前后各一张窗口 PNG(PrintWindow + SHA-256,默认 secret),接入对面证据体系;
|
||||||
|
4. 稳定错误码进 stdout,系统路径/控件文字不回显;
|
||||||
|
5. 单请求单响应、超时杀进程,遵守对面进程模型。
|
||||||
|
|
||||||
|
## 交接方式
|
||||||
|
|
||||||
|
按 `PATCH-NOTES.md` 执行:3 个新文件 + Program.cs 最小 diff + TS 适配器后续工作。合入提交必须带事实文件与文档区块更新(对面 2026-11-06 门禁),清单见 PATCH-NOTES 第 5 节。
|
||||||
|
|
||||||
|
## 验证状态
|
||||||
|
|
||||||
|
**已编译通过 + 协议冒烟通过**(2026-11-06,.NET SDK 10.0.403):对面 csproj 原样 + 本补丁三文件 + Program.cs 最小 diff → `dotnet build` 0 警告 0 错误;五例零桌面交互协议冒烟全部符合预期(安全门错误码、双路分发、requestId 回显、observe 无回归)。真实窗口用例留待对面 `test/fixture` 集成测试。
|
||||||
57
contrib/windows-runner-actuate/native/ActuateProtocol.cs
Normal file
57
contrib/windows-runner-actuate/native/ActuateProtocol.cs
Normal file
@ -0,0 +1,57 @@
|
|||||||
|
using System.Text.Json.Serialization;
|
||||||
|
|
||||||
|
namespace AgentWorker.WindowsRunner;
|
||||||
|
|
||||||
|
// actuate 协议扩展提案:在不动 observe 契约的前提下增加执行类操作。
|
||||||
|
// 设计原则与 observe 完全一致:单请求单响应、稳定错误码、宿主固定证据根目录。
|
||||||
|
// 本文件只新增类型;Program.cs 的分发改造见 PATCH-NOTES.md(diff 指引)。
|
||||||
|
//
|
||||||
|
// 与 observe 的关键差异(由 RPA 桌面驱动语义决定):
|
||||||
|
// 1. Action 携带具体执行参数(坐标/文本/命令),全部字段服务端可审计;
|
||||||
|
// 2. 坐标一律物理屏幕像素,与 RPA 驱动现状及 CoordinateSpace 契约一致;
|
||||||
|
// 3. WindowTitle 必须是 AllowedWindows 的精确成员(模糊匹配在 TS 宿主 observe 阶段完成解析),
|
||||||
|
// RPA 侧"标题包含匹配"的旧语义不进入原生层。
|
||||||
|
|
||||||
|
// 操作闭集:mouse_move / mouse_click / mouse_drag / mouse_scroll / mouse_move_smooth /
|
||||||
|
// keyboard_send / window_activate / window_set_state / window_set_bounds / window_close。
|
||||||
|
// 与 observe 相同:模型不能直接指定 HWND;证据目录来自宿主启动参数。
|
||||||
|
internal sealed record ActuateRequest(
|
||||||
|
int ProtocolVersion,
|
||||||
|
string RequestId,
|
||||||
|
string Operation,
|
||||||
|
string WindowTitle,
|
||||||
|
DesktopPolicy Policy,
|
||||||
|
ActuateAction Action);
|
||||||
|
|
||||||
|
// 每个字段都可被逐项审计;不支持的 Kind 由原生层拒绝,不静默降级为观察。
|
||||||
|
internal sealed record ActuateAction(
|
||||||
|
string Kind,
|
||||||
|
int? X,
|
||||||
|
int? Y,
|
||||||
|
int? ToX,
|
||||||
|
int? ToY,
|
||||||
|
// 滚轮 delta(120 的倍数,负值向下)或窗口 ShowWindow 命令值。
|
||||||
|
int? Amount,
|
||||||
|
// keyboard_send 文本;上限由 64 KiB 请求预算约束。
|
||||||
|
string? Text,
|
||||||
|
// keyboard_send 三模式:"clipboard"(粘贴)/ "chars"(逐字符)/ "keys"(整段 SendKeys)。
|
||||||
|
string? Mode,
|
||||||
|
int? IntervalMs,
|
||||||
|
bool? Double,
|
||||||
|
// "left"(默认)或 "right"。
|
||||||
|
string? Button,
|
||||||
|
// mouse_move_smooth:false=匀速直线,true/缺省=缓入缓出+抖动。
|
||||||
|
bool? Ease,
|
||||||
|
// mouse_move_smooth 速度 1-100(默认 35,与 RPA 驱动一致)。
|
||||||
|
int? Speed,
|
||||||
|
// window_set_bounds 用(x/y 复用 X/Y,Amount 不用)。
|
||||||
|
int? Width,
|
||||||
|
int? Height,
|
||||||
|
int? HoldMs,
|
||||||
|
int? ReleaseDelayMs,
|
||||||
|
int? SettleMs);
|
||||||
|
|
||||||
|
// 动作结果:证据前后各一张,宿主 TS 适配器按 EvidenceRef 逐项复核。
|
||||||
|
// Data 与 ErrorCode 互斥,与 RunnerResponse 同规则。
|
||||||
|
internal sealed record ActuateResult(string ActionKind, string CapturedAt,
|
||||||
|
ObservedWindow ObservedWindow, IReadOnlyList<EvidenceRef> Evidence, int DurationMs);
|
||||||
279
contrib/windows-runner-actuate/native/ActuateService.cs
Normal file
279
contrib/windows-runner-actuate/native/ActuateService.cs
Normal file
@ -0,0 +1,279 @@
|
|||||||
|
using System.Diagnostics;
|
||||||
|
using System.Drawing;
|
||||||
|
using System.Drawing.Imaging;
|
||||||
|
using System.Globalization;
|
||||||
|
using System.Security.Cryptography;
|
||||||
|
using System.Text.RegularExpressions;
|
||||||
|
using System.ComponentModel;
|
||||||
|
|
||||||
|
namespace AgentWorker.WindowsRunner;
|
||||||
|
|
||||||
|
// actuate 执行服务:把 RPA 桌面驱动(PowerShell 实现)的鼠标/键盘/窗口操作
|
||||||
|
// 移植进对面 windows-runner 协议,安全门与 observe 完全同构。
|
||||||
|
// 移植来源:packages/rpa-core/src/driver/windows/windows-driver.ts(对照表见 PORTING-MAP.md)。
|
||||||
|
//
|
||||||
|
// 安全边界(与 WindowObservationService 同级):
|
||||||
|
// - 进程白名单(文件名正则)+ 窗口精确匹配 + OS Session 三重核验后才允许输入;
|
||||||
|
// - 只操作本用户交互 Session 的一个顶层窗口,不扫描其它桌面;
|
||||||
|
// - 后续动作前仍重新枚举窗口,不信任上一次请求的 HWND;
|
||||||
|
// - 副作用证据:动作前后各一张目标窗口 PNG(PrintWindow + SHA-256,默认 secret);
|
||||||
|
// - 非法 Kind 拒绝执行,不静默降级为观察;
|
||||||
|
// - 稳定错误码进 stdout,系统路径与控件文字不回显。
|
||||||
|
internal static partial class ActuateService
|
||||||
|
{
|
||||||
|
[GeneratedRegex("^[A-Za-z0-9._-]+\\.exe$", RegexOptions.IgnoreCase)]
|
||||||
|
private static partial Regex ProcessNamePattern();
|
||||||
|
|
||||||
|
private const long MaxWindowPixels = 16_000_000;
|
||||||
|
private const long MaxPngBytes = 8 * 1024 * 1024;
|
||||||
|
|
||||||
|
private sealed record WindowCandidate(nint Handle, string ProcessName, string Title);
|
||||||
|
|
||||||
|
// DPI 感知一次性声明:与 RPA 驱动每脚本 SetProcessDpiAwarenessContext(-4) 等价,
|
||||||
|
// 保证此后 SetCursorPos/GetCursorPos 全程物理像素语义。
|
||||||
|
internal static ActuateResult Actuate(ActuateRequest request, string evidenceRoot)
|
||||||
|
{
|
||||||
|
NativeInput.EnsureDpiAwareness();
|
||||||
|
ValidatePolicy(request);
|
||||||
|
var selected = SelectWindow(request.WindowTitle, request.Policy);
|
||||||
|
if (!NativeWindows.GetWindowRect(selected.Handle, out var nativeRect))
|
||||||
|
throw new RunnerException("window_unavailable");
|
||||||
|
var bounds = new WindowRect(nativeRect.Left, nativeRect.Top,
|
||||||
|
nativeRect.Right - nativeRect.Left, nativeRect.Bottom - nativeRect.Top);
|
||||||
|
if (bounds.Width < 1 || bounds.Height < 1 ||
|
||||||
|
(long)bounds.Width * bounds.Height > MaxWindowPixels)
|
||||||
|
throw new RunnerException("window_size_unsupported");
|
||||||
|
|
||||||
|
var startedAt = DateTimeOffset.UtcNow;
|
||||||
|
var before = CaptureWindowEvidence(selected.Handle, bounds, evidenceRoot, startedAt);
|
||||||
|
Execute(request.Action, selected);
|
||||||
|
var finishedAt = DateTimeOffset.UtcNow;
|
||||||
|
var after = CaptureWindowEvidence(selected.Handle, bounds, evidenceRoot, finishedAt);
|
||||||
|
NativeWindows.GetWindowThreadProcessId(selected.Handle, out var pid);
|
||||||
|
var observed = new ObservedWindow(selected.ProcessName, selected.Title,
|
||||||
|
$"0x{selected.Handle.ToInt64():X}",
|
||||||
|
selected.Handle == NativeWindows.GetForegroundWindow());
|
||||||
|
return new ActuateResult(request.Action.Kind, finishedAt.ToString("O", CultureInfo.InvariantCulture),
|
||||||
|
observed, [before, after],
|
||||||
|
(int)(finishedAt - startedAt).TotalMilliseconds);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 与 observe 的 ValidatePolicy 同构:两个白名单均非空、进程名仅接受文件名、
|
||||||
|
// WindowTitle 精确成员、OS Session 与本进程一致。actuate 不放宽任何一项。
|
||||||
|
private static void ValidatePolicy(ActuateRequest request)
|
||||||
|
{
|
||||||
|
var policy = request.Policy;
|
||||||
|
if (policy is null || policy.AllowedProcesses is null || policy.AllowedWindows is null ||
|
||||||
|
policy.AllowedProcesses.Length == 0 || policy.AllowedWindows.Length == 0 ||
|
||||||
|
policy.AllowedProcesses.Any(name => !ProcessNamePattern().IsMatch(name) || name.Contains("..")) ||
|
||||||
|
policy.AllowedWindows.Any(title => string.IsNullOrWhiteSpace(title) || title.Length > 256))
|
||||||
|
throw new RunnerException("policy_invalid");
|
||||||
|
if (string.IsNullOrWhiteSpace(request.WindowTitle) || request.WindowTitle.Length > 256 ||
|
||||||
|
!policy.AllowedWindows.Contains(request.WindowTitle, StringComparer.Ordinal))
|
||||||
|
throw new RunnerException("window_not_allowed");
|
||||||
|
var currentSession = Process.GetCurrentProcess().SessionId;
|
||||||
|
if (currentSession == 0 || policy.DesktopSessionId != currentSession.ToString(CultureInfo.InvariantCulture))
|
||||||
|
throw new RunnerException("desktop_session_mismatch");
|
||||||
|
if (request.Action is null || string.IsNullOrWhiteSpace(request.Action.Kind))
|
||||||
|
throw new RunnerException("action_invalid");
|
||||||
|
}
|
||||||
|
|
||||||
|
// 与 observe 的 SelectWindow 同构:枚举本 Session 顶层窗口,精确标题 + PID 回查进程名。
|
||||||
|
// 同名同进程窗口多于一个时报 ambiguous,不任选——副作用落错对象不可接受。
|
||||||
|
private static WindowCandidate SelectWindow(string title, DesktopPolicy policy)
|
||||||
|
{
|
||||||
|
var candidates = new List<WindowCandidate>();
|
||||||
|
var session = Process.GetCurrentProcess().SessionId;
|
||||||
|
_ = NativeWindows.EnumWindows((handle, _) =>
|
||||||
|
{
|
||||||
|
if (!NativeWindows.IsWindowVisible(handle)) return true;
|
||||||
|
if (NativeWindows.Title(handle) != title) return true;
|
||||||
|
NativeWindows.GetWindowThreadProcessId(handle, out var pid);
|
||||||
|
if (pid == 0) return true;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
using var process = Process.GetProcessById(checked((int)pid));
|
||||||
|
if (process.SessionId != session) return true;
|
||||||
|
var name = process.ProcessName + ".exe";
|
||||||
|
if (!policy.AllowedProcesses.Contains(name, StringComparer.OrdinalIgnoreCase))
|
||||||
|
return true;
|
||||||
|
candidates.Add(new WindowCandidate(handle, name, title));
|
||||||
|
}
|
||||||
|
catch (ArgumentException) { /* 进程已退出;旧 HWND 不能成为目标。 */ }
|
||||||
|
catch (InvalidOperationException) { /* 进程在枚举期间退出。 */ }
|
||||||
|
catch (Win32Exception) { /* 受保护进程不能读取。 */ }
|
||||||
|
return true;
|
||||||
|
}, nint.Zero);
|
||||||
|
if (candidates.Count == 0) throw new RunnerException("window_not_found");
|
||||||
|
if (candidates.Count != 1) throw new RunnerException("window_ambiguous");
|
||||||
|
return candidates[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
// 动作闭集分发;RPA 驱动 windows-driver.ts 各方法的逐段移植(见 PORTING-MAP.md)。
|
||||||
|
private static void Execute(ActuateAction action, WindowCandidate window)
|
||||||
|
{
|
||||||
|
switch (action.Kind)
|
||||||
|
{
|
||||||
|
case "mouse_move":
|
||||||
|
RequirePoint(action);
|
||||||
|
if (!NativeInput.SetCursorPos(action.X!.Value, action.Y!.Value))
|
||||||
|
throw new RunnerException("input_failed");
|
||||||
|
break;
|
||||||
|
case "mouse_click":
|
||||||
|
RequirePoint(action);
|
||||||
|
if (!NativeInput.SetCursorPos(action.X!.Value, action.Y!.Value))
|
||||||
|
throw new RunnerException("input_failed");
|
||||||
|
NativeInput.Click(action.Button == "right", action.Double == true ? 2 : 1,
|
||||||
|
Math.Max(0, action.SettleMs ?? 0));
|
||||||
|
break;
|
||||||
|
case "mouse_drag":
|
||||||
|
RequirePoint(action);
|
||||||
|
if (action.ToX is null || action.ToY is null) throw new RunnerException("action_invalid");
|
||||||
|
NativeInput.DragSequence(action.Button == "right", action.X!.Value, action.Y!.Value,
|
||||||
|
action.ToX.Value, action.ToY.Value,
|
||||||
|
Math.Max(0, action.HoldMs ?? 80), Math.Max(0, action.ReleaseDelayMs ?? 80));
|
||||||
|
break;
|
||||||
|
case "mouse_scroll":
|
||||||
|
if (action.Amount is null || action.Amount == 0) throw new RunnerException("action_invalid");
|
||||||
|
NativeInput.SendWheel(action.Amount.Value);
|
||||||
|
break;
|
||||||
|
case "mouse_move_smooth":
|
||||||
|
RequirePoint(action);
|
||||||
|
SmoothMove(action);
|
||||||
|
break;
|
||||||
|
case "keyboard_send":
|
||||||
|
if (string.IsNullOrEmpty(action.Text)) throw new RunnerException("action_invalid");
|
||||||
|
// 输入必须落在已核验窗口:先抢前台(五招兜底),对应 RPA 驱动
|
||||||
|
// activateTitle 选项 + 300ms 间隙;窗口已在前台时五招直接通过。
|
||||||
|
NativeInput.FocusChain(window.Handle);
|
||||||
|
Thread.Sleep(300);
|
||||||
|
SendKeyboard(action);
|
||||||
|
break;
|
||||||
|
case "window_activate":
|
||||||
|
NativeInput.FocusChain(window.Handle);
|
||||||
|
break;
|
||||||
|
case "window_set_state":
|
||||||
|
if (action.Amount is null) throw new RunnerException("action_invalid");
|
||||||
|
_ = NativeInput.ShowWindow(window.Handle, action.Amount.Value);
|
||||||
|
break;
|
||||||
|
case "window_set_bounds":
|
||||||
|
if (action.X is null || action.Y is null ||
|
||||||
|
action.Width is null || action.Height is null ||
|
||||||
|
action.Width < 1 || action.Height < 1)
|
||||||
|
throw new RunnerException("action_invalid");
|
||||||
|
NativeInput.MoveWindowBounds(window.Handle, action.X.Value, action.Y.Value,
|
||||||
|
action.Width.Value, action.Height.Value);
|
||||||
|
break;
|
||||||
|
case "window_close":
|
||||||
|
NativeInput.CloseWindow(window.Handle);
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
throw new RunnerException("action_invalid");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void RequirePoint(ActuateAction action)
|
||||||
|
{
|
||||||
|
if (action.X is null || action.Y is null) throw new RunnerException("action_invalid");
|
||||||
|
}
|
||||||
|
|
||||||
|
// keyboard 三模式(RPA 驱动 KeyboardDriver.send 的移植):
|
||||||
|
// clipboard:原文进剪贴板 + Ctrl+V(不做 SendKeys 转义,适合长文本);
|
||||||
|
// chars:逐字符 SendWait(每个字符按 SendKeys 规则转义,intervalMs 控速);
|
||||||
|
// keys:整段 SendWait 原样发送(保留 ^+% 等组合语义,供按键组合类能力使用)。
|
||||||
|
private static void SendKeyboard(ActuateAction action)
|
||||||
|
{
|
||||||
|
switch (action.Mode)
|
||||||
|
{
|
||||||
|
case "clipboard":
|
||||||
|
Clipboard.SetDataObject(action.Text!, true);
|
||||||
|
Thread.Sleep(120);
|
||||||
|
SendKeys.SendWait("^v");
|
||||||
|
break;
|
||||||
|
case "chars":
|
||||||
|
var interval = Math.Max(0, action.IntervalMs ?? 30);
|
||||||
|
foreach (var character in action.Text!)
|
||||||
|
{
|
||||||
|
SendKeys.SendWait(EscapeSendKeys(character));
|
||||||
|
if (interval > 0) Thread.Sleep(interval);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
default: // "keys"
|
||||||
|
SendKeys.SendWait(action.Text!);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// RPA 驱动 sendKeysEscape 的逐字符移植:SendKeys 特殊字符用花括号表示字面量。
|
||||||
|
private static string EscapeSendKeys(char character) =>
|
||||||
|
"+^%~(){}[]".Contains(character) ? $"{{{character}}}" : character.ToString();
|
||||||
|
|
||||||
|
// mouse.moveSmooth 的移植:从当前光标位置出发,按 RPA 驱动 mousePath 公式插值。
|
||||||
|
// 公式(缓入缓出 + 垂直抖动)逐行保真;JS 与 C# 的 % 对负数同为保号语义。
|
||||||
|
private static void SmoothMove(ActuateAction action)
|
||||||
|
{
|
||||||
|
var from = NativeInput.CursorPosition();
|
||||||
|
var toX = action.X!.Value;
|
||||||
|
var toY = action.Y!.Value;
|
||||||
|
var dx = toX - from.X;
|
||||||
|
var dy = toY - from.Y;
|
||||||
|
var distance = Math.Sqrt((double)dx * dx + (double)dy * dy);
|
||||||
|
if (distance < 2) return;
|
||||||
|
var stepCount = Math.Max(6, Math.Min(50, (int)Math.Round(distance / 12)));
|
||||||
|
var ease = action.Ease != false;
|
||||||
|
var speed = Math.Clamp(action.Speed ?? 35, 1, 100);
|
||||||
|
var delay = Math.Clamp((int)Math.Round(1500.0 / speed), 4, 80);
|
||||||
|
for (var i = 1; i <= stepCount; i++)
|
||||||
|
{
|
||||||
|
var t = (double)i / stepCount;
|
||||||
|
var easeFactor = ease ? (t < 0.5 ? 2 * t * t : 1 - Math.Pow(-2 * t + 2, 2) / 2) : t;
|
||||||
|
var jitter = ease ? Math.Sin(t * Math.PI) * (Math.Sin(i * 12.9898) * 43758.5453 % 1) * 3 : 0;
|
||||||
|
var x = (int)Math.Round(from.X + dx * easeFactor - (dy / distance) * jitter);
|
||||||
|
var y = (int)Math.Round(from.Y + dy * easeFactor + (dx / distance) * jitter);
|
||||||
|
if (!NativeInput.SetCursorPos(x, y)) throw new RunnerException("input_failed");
|
||||||
|
Thread.Sleep(delay);
|
||||||
|
}
|
||||||
|
if (!NativeInput.SetCursorPos(toX, toY)) throw new RunnerException("input_failed");
|
||||||
|
}
|
||||||
|
|
||||||
|
// 证据采集:与 observe 的 CaptureWindow 同构(PrintWindow → PNG → SHA-256)。
|
||||||
|
// actuate 始终留证据(副作用审计需要);PrintWindow 语义与像素有效性检查逐行保真。
|
||||||
|
private static EvidenceRef CaptureWindowEvidence(nint handle, WindowRect bounds,
|
||||||
|
string evidenceRoot, DateTimeOffset capturedAt)
|
||||||
|
{
|
||||||
|
var path = Path.Combine(evidenceRoot, $"actuate-{Guid.NewGuid():N}.png");
|
||||||
|
var stamp = capturedAt.ToString("O", CultureInfo.InvariantCulture);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
using var bitmap = new Bitmap(bounds.Width, bounds.Height);
|
||||||
|
using var graphics = Graphics.FromImage(bitmap);
|
||||||
|
var deviceContext = graphics.GetHdc();
|
||||||
|
bool captured;
|
||||||
|
try { captured = NativeWindows.PrintWindow(handle, deviceContext, 2); }
|
||||||
|
finally { graphics.ReleaseHdc(deviceContext); }
|
||||||
|
if (!captured || IsBlank(bitmap)) throw new RunnerException("capture_unavailable");
|
||||||
|
bitmap.Save(path, ImageFormat.Png);
|
||||||
|
var size = new FileInfo(path).Length;
|
||||||
|
if (size < 1 || size > MaxPngBytes) throw new RunnerException("capture_size_unsupported");
|
||||||
|
using var stream = File.OpenRead(path);
|
||||||
|
var hash = Convert.ToHexString(SHA256.HashData(stream)).ToLowerInvariant();
|
||||||
|
return new EvidenceRef($"ev_{hash[..24]}", "window_snapshot",
|
||||||
|
new Uri(path).AbsoluteUri, hash, "secret", stamp);
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
if (File.Exists(path)) File.Delete(path);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool IsBlank(Bitmap bitmap)
|
||||||
|
{
|
||||||
|
var first = bitmap.GetPixel(0, 0).ToArgb();
|
||||||
|
for (var row = 0; row < 10; row++)
|
||||||
|
for (var column = 0; column < 10; column++)
|
||||||
|
if (bitmap.GetPixel(column * (bitmap.Width - 1) / 9,
|
||||||
|
row * (bitmap.Height - 1) / 9).ToArgb() != first) return false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
212
contrib/windows-runner-actuate/native/NativeInput.cs
Normal file
212
contrib/windows-runner-actuate/native/NativeInput.cs
Normal file
@ -0,0 +1,212 @@
|
|||||||
|
using System.Runtime.InteropServices;
|
||||||
|
|
||||||
|
namespace AgentWorker.WindowsRunner;
|
||||||
|
|
||||||
|
// 输入类 P/Invoke 独立成类:NativeWindows 保持对面"只读编译边界"原样不动。
|
||||||
|
// 本层没有进程启动接口;所有输入只作用于经策略核验后的前台窗口。
|
||||||
|
// 坐标一律物理屏幕像素——进程入口(ActuateService.EnsureDpiAwareness)声明
|
||||||
|
// Per-Monitor V2 DPI 感知,与 RPA 侧 PowerShell 驱动每脚本声明的
|
||||||
|
// SetProcessDpiAwarenessContext(-4) 语义一致,保证 SetCursorPos 全程物理像素。
|
||||||
|
// 抢前台五招兜底从 RPA 驱动 activateWindowScript 逐行移植:
|
||||||
|
// Windows 前台锁定策略会拒绝普通 SetForegroundWindow(应用点击"运行"后自身是前台),
|
||||||
|
// 五招顺序与睡眠时间是踩坑调出来的实战参数,一行语义都不允许改动。
|
||||||
|
internal static class NativeInput
|
||||||
|
{
|
||||||
|
private const uint InputMouse = 0;
|
||||||
|
private const uint MouseEventfMove = 0x0001;
|
||||||
|
private const uint MouseEventfLeftDown = 0x0002;
|
||||||
|
private const uint MouseEventfLeftUp = 0x0004;
|
||||||
|
private const uint MouseEventfRightDown = 0x0008;
|
||||||
|
private const uint MouseEventfRightUp = 0x0010;
|
||||||
|
private const uint MouseEventfWheel = 0x0800;
|
||||||
|
internal const int SwRestore = 9;
|
||||||
|
internal const int SwMinimize = 6;
|
||||||
|
internal const int SwMaximize = 3;
|
||||||
|
private const uint WmClose = 0x0010;
|
||||||
|
private const byte VkMenu = 0x12; // ALT
|
||||||
|
private const uint KeyeventfKeyup = 0x0002;
|
||||||
|
|
||||||
|
[DllImport("user32.dll")]
|
||||||
|
[return: MarshalAs(UnmanagedType.Bool)]
|
||||||
|
internal static extern bool SetCursorPos(int x, int y);
|
||||||
|
|
||||||
|
[DllImport("user32.dll")]
|
||||||
|
[return: MarshalAs(UnmanagedType.Bool)]
|
||||||
|
private static extern bool GetCursorPos(out Point point);
|
||||||
|
|
||||||
|
[DllImport("user32.dll")]
|
||||||
|
[return: MarshalAs(UnmanagedType.Bool)]
|
||||||
|
internal static extern bool SetForegroundWindow(nint handle);
|
||||||
|
|
||||||
|
[DllImport("user32.dll")]
|
||||||
|
[return: MarshalAs(UnmanagedType.Bool)]
|
||||||
|
internal static extern bool ShowWindow(nint handle, int command);
|
||||||
|
|
||||||
|
[DllImport("user32.dll")]
|
||||||
|
[return: MarshalAs(UnmanagedType.Bool)]
|
||||||
|
internal static extern bool IsIconic(nint handle);
|
||||||
|
|
||||||
|
[DllImport("user32.dll")]
|
||||||
|
internal static extern void keybd_event(byte virtualKey, byte scanCode, uint flags, nint extraInfo);
|
||||||
|
|
||||||
|
[DllImport("user32.dll")]
|
||||||
|
[return: MarshalAs(UnmanagedType.Bool)]
|
||||||
|
private static extern bool AttachThreadInput(uint currentThread, uint targetThread, bool attach);
|
||||||
|
|
||||||
|
[DllImport("user32.dll")]
|
||||||
|
private static extern void SwitchToThisWindow(nint handle, bool altTab);
|
||||||
|
|
||||||
|
[DllImport("kernel32.dll")]
|
||||||
|
private static extern uint GetCurrentThreadId();
|
||||||
|
|
||||||
|
[DllImport("user32.dll")]
|
||||||
|
[return: MarshalAs(UnmanagedType.Bool)]
|
||||||
|
private static extern bool SetProcessDpiAwarenessContext(nint value);
|
||||||
|
|
||||||
|
[DllImport("user32.dll")]
|
||||||
|
[return: MarshalAs(UnmanagedType.Bool)]
|
||||||
|
private static extern bool SetWindowPos(nint handle, nint after, int x, int y,
|
||||||
|
int width, int height, uint flags);
|
||||||
|
|
||||||
|
[DllImport("user32.dll")]
|
||||||
|
[return: MarshalAs(UnmanagedType.Bool)]
|
||||||
|
private static extern bool PostMessage(nint handle, uint message, nint wParam, nint lParam);
|
||||||
|
|
||||||
|
[DllImport("user32.dll", SetLastError = true)]
|
||||||
|
private static extern uint SendInput(uint count, Input[] inputs, int size);
|
||||||
|
|
||||||
|
[StructLayout(LayoutKind.Sequential)]
|
||||||
|
internal struct Point { public int X; public int Y; }
|
||||||
|
|
||||||
|
[StructLayout(LayoutKind.Sequential)]
|
||||||
|
private struct Input
|
||||||
|
{
|
||||||
|
public uint Type;
|
||||||
|
public InputUnion Union;
|
||||||
|
}
|
||||||
|
|
||||||
|
[StructLayout(LayoutKind.Explicit)]
|
||||||
|
private struct InputUnion
|
||||||
|
{
|
||||||
|
[FieldOffset(0)] public MouseInput Mouse;
|
||||||
|
}
|
||||||
|
|
||||||
|
[StructLayout(LayoutKind.Sequential)]
|
||||||
|
private struct MouseInput
|
||||||
|
{
|
||||||
|
public int Dx;
|
||||||
|
public int Dy;
|
||||||
|
public uint MouseData;
|
||||||
|
public uint Flags;
|
||||||
|
public uint Time;
|
||||||
|
public nint ExtraInfo;
|
||||||
|
}
|
||||||
|
|
||||||
|
// DPI 感知声明是进程级一次性设置;失败(如已被宿主先行声明)不影响后续调用。
|
||||||
|
internal static void EnsureDpiAwareness()
|
||||||
|
{
|
||||||
|
try { _ = SetProcessDpiAwarenessContext(new nint(-4)); }
|
||||||
|
catch { /* 旧系统无此 API 时按物理像素继续;对面 CoordinateSpace 以 GetDpiForWindow 实测为准。 */ }
|
||||||
|
}
|
||||||
|
|
||||||
|
// SendInput 批量提交鼠标标志组合;单次失败按 input_failed 处理由上层转稳定错误码。
|
||||||
|
internal static void SendMouse(uint flags, uint data)
|
||||||
|
{
|
||||||
|
var input = new Input
|
||||||
|
{
|
||||||
|
Type = InputMouse,
|
||||||
|
Union = new InputUnion
|
||||||
|
{
|
||||||
|
Mouse = new MouseInput { Dx = 0, Dy = 0, MouseData = data, Flags = flags }
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if (SendInput(1, [input], Marshal.SizeOf<Input>()) != 1)
|
||||||
|
throw new RunnerException("input_failed");
|
||||||
|
}
|
||||||
|
|
||||||
|
internal static void SendWheel(int amount) =>
|
||||||
|
SendMouse(MouseEventfWheel, unchecked((uint)amount));
|
||||||
|
|
||||||
|
internal static void Click(bool right, int repeats, int settleMs)
|
||||||
|
{
|
||||||
|
var down = right ? MouseEventfRightDown : MouseEventfLeftDown;
|
||||||
|
var up = right ? MouseEventfRightUp : MouseEventfLeftUp;
|
||||||
|
for (var i = 0; i < repeats; i++)
|
||||||
|
{
|
||||||
|
SendMouse(down, 0);
|
||||||
|
SendMouse(up, 0);
|
||||||
|
if (settleMs > 0) Thread.Sleep(settleMs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
internal static Point CursorPosition()
|
||||||
|
{
|
||||||
|
if (!GetCursorPos(out var point)) throw new RunnerException("input_failed");
|
||||||
|
return point;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 窗口拖拽时序:按下→hold→移动→release→抬起,与 RPA 驱动逐段一致。
|
||||||
|
internal static void DragSequence(bool right, int fromX, int fromY, int toX, int toY,
|
||||||
|
int holdMs, int releaseDelayMs)
|
||||||
|
{
|
||||||
|
var down = right ? MouseEventfRightDown : MouseEventfLeftDown;
|
||||||
|
var up = right ? MouseEventfRightUp : MouseEventfLeftUp;
|
||||||
|
if (!SetCursorPos(fromX, fromY)) throw new RunnerException("input_failed");
|
||||||
|
SendMouse(down, 0);
|
||||||
|
if (holdMs > 0) Thread.Sleep(holdMs);
|
||||||
|
if (!SetCursorPos(toX, toY)) throw new RunnerException("input_failed");
|
||||||
|
if (releaseDelayMs > 0) Thread.Sleep(releaseDelayMs);
|
||||||
|
SendMouse(up, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 抢前台五招兜底(RPA 驱动 activateWindowScript 的逐行移植):
|
||||||
|
// ①最小化则恢复 +150ms ②SwitchToThisWindow +150ms ③ALT 借位 + SetForegroundWindow +200ms
|
||||||
|
// ④AttachThreadInput 挂到前台线程 + SetForegroundWindow +200ms ⑤最小化再恢复 +450ms
|
||||||
|
// 五招全失败抛 focus_failed——对应 PS 版 RPA_ERROR '无法把窗口带到前台'。
|
||||||
|
internal static void FocusChain(nint handle)
|
||||||
|
{
|
||||||
|
if (IsIconic(handle))
|
||||||
|
{
|
||||||
|
_ = ShowWindow(handle, SwRestore);
|
||||||
|
Thread.Sleep(150);
|
||||||
|
}
|
||||||
|
if (NativeWindows.GetForegroundWindow() != handle)
|
||||||
|
{
|
||||||
|
SwitchToThisWindow(handle, true);
|
||||||
|
Thread.Sleep(150);
|
||||||
|
}
|
||||||
|
if (NativeWindows.GetForegroundWindow() != handle)
|
||||||
|
{
|
||||||
|
keybd_event(VkMenu, 0, 0, nint.Zero);
|
||||||
|
_ = SetForegroundWindow(handle);
|
||||||
|
keybd_event(VkMenu, 0, KeyeventfKeyup, nint.Zero);
|
||||||
|
Thread.Sleep(200);
|
||||||
|
}
|
||||||
|
if (NativeWindows.GetForegroundWindow() != handle)
|
||||||
|
{
|
||||||
|
// 返回值即前台窗口的线程号;PID 出参不需要。
|
||||||
|
var foregroundThread = NativeWindows.GetWindowThreadProcessId(
|
||||||
|
NativeWindows.GetForegroundWindow(), out _);
|
||||||
|
var currentThread = GetCurrentThreadId();
|
||||||
|
var attached = AttachThreadInput(currentThread, foregroundThread, true);
|
||||||
|
try { _ = SetForegroundWindow(handle); }
|
||||||
|
finally { if (attached) _ = AttachThreadInput(currentThread, foregroundThread, false); }
|
||||||
|
Thread.Sleep(200);
|
||||||
|
}
|
||||||
|
if (NativeWindows.GetForegroundWindow() != handle)
|
||||||
|
{
|
||||||
|
_ = ShowWindow(handle, SwMinimize);
|
||||||
|
Thread.Sleep(150);
|
||||||
|
_ = ShowWindow(handle, SwRestore);
|
||||||
|
Thread.Sleep(300);
|
||||||
|
}
|
||||||
|
if (NativeWindows.GetForegroundWindow() != handle)
|
||||||
|
throw new RunnerException("focus_failed");
|
||||||
|
}
|
||||||
|
|
||||||
|
internal static void MoveWindowBounds(nint handle, int x, int y, int width, int height) =>
|
||||||
|
_ = SetWindowPos(handle, nint.Zero, x, y, width, height, 4); // flags=4 SWP_NOZORDER,与 RPA 驱动一致
|
||||||
|
|
||||||
|
internal static void CloseWindow(nint handle) =>
|
||||||
|
_ = PostMessage(handle, WmClose, nint.Zero, nint.Zero);
|
||||||
|
}
|
||||||
@ -1,334 +0,0 @@
|
|||||||
# 画布式 RPA 桌面端 · 现状分析与可行性(调研稿)
|
|
||||||
|
|
||||||
> 编写时间:2026-10-26
|
|
||||||
> 参考目录:`D:\code\Agent-Employee`(只读)、`C:\rpa-demo`(只读)
|
|
||||||
> 本文件是决策输入,不是最终实施方案;带「待确认」的条目需要产品方拍板。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 0. 一句话结论
|
|
||||||
|
|
||||||
`D:\code\Agent-Employee` 里的 **1016 项 RPA 目录全部是静态元数据**(997 个厂商组件定义 + 16 个原生控制节点 + 3 个项目自有触发器),**不含任何厂商运行时**;`C:\rpa-demo` 只有**浏览器**动作层(CDP 可信输入,26/26 自测通过),没有画布、没有桌面能力、没有执行引擎。
|
|
||||||
|
|
||||||
因此三天内**不可能**做出"1016 个能力都能跑"的产品。可行的目标是:
|
|
||||||
|
|
||||||
> **一个本地优先的画布式 RPA 桌面端纵向切片**:左侧 1016 项目录可搜索、可拖拽、可配置;画布支持连线/分支/循环/变量;点运行后由真实执行内核跑通一条端到端流程;未接入的组件在目录和节点上显式标记,运行时给出明确失败原因。
|
|
||||||
|
|
||||||
建议真实可执行组件目标 **100~140 项**(覆盖浏览器、文件、Excel、字符串/时间/列表、键鼠/窗口、HTTP/邮件),其余标记「目录可用、执行器待接入」。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 1. 参考资产盘点
|
|
||||||
|
|
||||||
### 1.1 `D:\code\Agent-Employee`:产品全量仓库,RPA 只是其中一块
|
|
||||||
|
|
||||||
| 资产 | 位置 | 结论 |
|
|
||||||
|---|---|---|
|
|
||||||
| 1016 项目录 | `code/frontend/apps/desktop/src/renderer/product-preview/pages/workflows/shizai-catalog-data.js`(1.8 MB) | **可直接复用**:997 组件完整字段、默认值、分类、函数签名 |
|
|
||||||
| 目录投影/分组 | 同目录 `component-catalog.js` | 可复用 11 组分组规则、字段类型映射、3 个触发器、别名表 |
|
|
||||||
| 现有流程页 | `product-preview/pages/workflows/page.js`(50 KB) | 已有:搜索筛选、增删节点、属性面板、缩放、纵向流式画布、localStorage 草稿。**不足:不是自由画布,不能拖拽定位,没有执行器** |
|
|
||||||
| 设计原型 | `项目资料/UI设计/20261021/流程编排页原型.html` | 纵向线性流程,非拖拽画布;可作视觉基调参考 |
|
|
||||||
| 浏览器插件契约 | `code/backend/packages/browser-plugin/src/index.ts` | 只声明 `navigate/click/type/snapshot` 四个动作,`unsigned-development` |
|
|
||||||
| Windows 插件契约 | `code/backend/packages/windows-plugin/src/index.ts` | 只声明契约,无实现 |
|
|
||||||
| Windows Runner | `code/backend/services/windows-runner/native/*.cs` | **只读 UIA/截图观察**,注释明确"没有 SendInput / SetForegroundWindow / 进程启动" |
|
|
||||||
| 工作流引擎 | `code/backend/packages/workflow-engine/src/*` | 服务端 PostgreSQL 工作流运行/审批/恢复,**不适合本机三天产品**(要 Docker/PG/多服务) |
|
|
||||||
| 预览客户端 | `交付物/客户端/Windows/x64/0.1.0-preview.1/` | 已有 Electron 安装包,但只跑前端演示,不含 RPA 执行器 |
|
|
||||||
|
|
||||||
**判断**:目录元数据是最大可复用资产;现有画布是"线性步骤列表",与用户要的"自由拖拽画布"不是一回事;后端工作流栈不适合三天工期。
|
|
||||||
|
|
||||||
### 1.2 1016 项目录的真实构成
|
|
||||||
|
|
||||||
来自 `shizai-catalog-data.js` 的 `counts`(已用脚本解析核对):
|
|
||||||
|
|
||||||
| 项 | 数量 |
|
|
||||||
|---|---:|
|
|
||||||
| 厂商组件定义 | 997 |
|
|
||||||
| 原生控制节点(IF/循环/异常/子流程…) | 16 |
|
|
||||||
| 项目自有触发器(人工/定时/文件到达) | 3 |
|
|
||||||
| **合计** | **1016** |
|
|
||||||
| 普通输入字段 | 4258(其中必填 1318,占 31%) |
|
|
||||||
| 输出字段 | 690 |
|
|
||||||
| 选项字段 | 6148 |
|
|
||||||
| 唯一函数名 | 997(组件与函数 1:1) |
|
|
||||||
| 字段数为 0 的组件 | 22 |
|
|
||||||
| 字段数 1–3 / 4–8 / 9–15 / 16+ | 473 / 414 / 75 / 13 |
|
|
||||||
|
|
||||||
按产品分组(沿用现有 `component-catalog.js` 规则):
|
|
||||||
|
|
||||||
| 分组 | 组件数 | 说明 |
|
|
||||||
|---|---:|---|
|
|
||||||
| 网页自动化(WebBrowser + Element) | 187 | 浏览器与元素操作,最接近 rpa-demo 能力 |
|
|
||||||
| Office 与文档(Excel/Word/PDF/转换) | 230 | Excel 单独 170 项,是最大单一包 |
|
|
||||||
| 数据处理(表格/字符串/时间/列表/字典/数学/正则…) | 167 | 纯函数为主,最容易批量实现 |
|
|
||||||
| 图像与 OCR | 100 | 依赖视觉模型/厂商服务,三天不做 |
|
|
||||||
| 网络与消息(HTTP/邮件/通知/NLP) | 82 | HTTP/邮件可做,NLP 不做 |
|
|
||||||
| 桌面与键鼠(键盘/鼠标/窗口/剪贴板/对话框) | 79 | 需 Windows 原生桥 |
|
|
||||||
| 应用与系统(App/System/Basic/屏幕控制) | 60 | 部分可做(启动进程/环境变量/延迟/变量) |
|
|
||||||
| 文件与传输(File/FTP) | 46 | 可做文件系统,FTP 可选 |
|
|
||||||
| 移动与外设 | 46 | 需 ADB/Appium/机械臂,三天不做 |
|
|
||||||
| 编排控制 | 16 | IF/计次/条件/无限循环/遍历/异常/子流程/退出 |
|
|
||||||
| 触发器 | 3 | 人工/定时/文件到达 |
|
|
||||||
|
|
||||||
**关键事实**:`functions[].function`(如 `Excel.Cs_ReadContentV2`)只是**名称映射**,原厂商实现依赖 Python 包、原生 SDK、浏览器扩展、Office COM、云端 OCR 等,目录里没有一行可执行代码。仓库 README 也明确写着"不代表这些动作已经可执行"。
|
|
||||||
|
|
||||||
### 1.3 `C:\rpa-demo`:可复用的浏览器执行层
|
|
||||||
|
|
||||||
| 能力 | 事实 |
|
|
||||||
|---|---|
|
|
||||||
| 形态 | Electron 31.7.7 + 多标签 `WebContentsView` + 右侧任务面板 |
|
|
||||||
| 动作层 | `actions.js`:CDP `Input` 域可信输入,5 级定位(testid→aria→text→css→xpath),统一结果/错误码/后置条件/证据 |
|
|
||||||
| 流程 | `flows/*.json` 声明式步骤 + 顺序回放 + 失败即停 + 每步证据 |
|
|
||||||
| 拾取器 | 注入点击捕获,生成候选 locator |
|
|
||||||
| 安全 | 域名 allowlist、高影响动作开关、`actionId` 幂等去重 |
|
|
||||||
| 自测 | `selftest-out.txt`:**26/26 通过**(点击/输入/下拉/等待/表格/错误分类/证据/越界/流程回放) |
|
|
||||||
| 不足 | 只有浏览器;无画布;无变量系统(只有 runId 注入);无桌面/Office;单窗口任务面板是死脚本 |
|
|
||||||
|
|
||||||
**判断**:`actions.js` + 拾取器 + 流程回放语义可以整体搬进新产品,改造成本低,是三天内"真的能跑"的最大保障。
|
|
||||||
|
|
||||||
### 1.4 环境事实(本机)
|
|
||||||
|
|
||||||
| 项 | 结果 |
|
|
||||||
|---|---|
|
|
||||||
| OS | Windows 11 专业版 26200 |
|
|
||||||
| Node / npm / Bun | 22.13.0 / 10.9.2 / 1.4.3 |
|
|
||||||
| Python | `py -3.10` 可用;已装 numpy/Pillow/opencv/pyautogui/pywin32;**无** pandas/openpyxl/python-docx/PyMuPDF |
|
|
||||||
| Office COM | Excel、Word 已安装(`Excel.Application` / `Word.Application` 注册存在) |
|
|
||||||
| 显示 | 1920×1080,主屏 100% 缩放(125%/150% 仍需冒烟) |
|
|
||||||
| 网络 | npm 官方源与 npmmirror 均可达 |
|
|
||||||
| 磁盘 | C: 仅剩 ~9.9 GB;D: 剩 ~159 GB → **项目与依赖放 D:** |
|
|
||||||
| Electron | `C:\rpa-demo\node_modules\electron` 已下载 31.7.7,可离线复用 |
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 2. 能做 / 不能做(三天口径)
|
|
||||||
|
|
||||||
### 2.1 建议纳入"真实可执行"
|
|
||||||
|
|
||||||
| 层 | 能力 | 预计可执行组件 | 实现方式 |
|
|
||||||
|---|---|---|---|
|
|
||||||
| L0 控制 | IF、计次循环、遍历列表、条件循环、异常捕获、跳出/继续、退出、子流程、设置变量、延迟、日志 | 16 个控制节点 + 基础组件 | 自研解释器 |
|
|
||||||
| L1 浏览器 | 打开/导航/点击/输入/取文本/取属性/等待/存在判断/下拉/悬停/滚动/截图/表格提取/执行 JS/标签页/上传下载 | 60~80 | 复用 `actions.js` + CDP + 拾取器 |
|
|
||||||
| L2 文件 | 读/写/复制/移动/删除/重命名/存在/列表/建目录/压缩/解压 | 25~35 | Node `fs` + `adm-zip` |
|
|
||||||
| L3 Excel/PDF/Word | 新建/打开/读区域/写单元格/写区域/追加/保存/工作表增删改/查找;PDF 页数/文本/合并;Word 写入 | 35~45 | `exceljs` + `pdf-lib` + `pdfjs-dist`;Word 走 COM 桥 |
|
|
||||||
| L4 数据 | 字符串/时间/数学/正则/列表/字典/JSON/CSV/数据表筛选排序去重 | 50~70 | 纯 JS 函数库 |
|
|
||||||
| L5 桌面 | 键鼠输入/热键/坐标点击/窗口查找激活/窗口状态/进程启动/剪贴板/消息框/文件对话框/环境变量/命令执行 | 35~45 | PowerShell 5.1 桥(UIA + SendInput) |
|
|
||||||
| L6 网络 | HTTP 请求、SMTP 发信、FTP 上传下载(可选)、PostgreSQL(可选) | 8~15 | `fetch` / `nodemailer` / `basic-ftp` / `pg` |
|
|
||||||
|
|
||||||
**合计目标:约 110~150 项可执行**(按函数名注册,允许同类组件共享实现内核)。
|
|
||||||
|
|
||||||
### 2.2 明确不做(三天)
|
|
||||||
|
|
||||||
- 移动端/外设/机械臂(Mobile、peripherals,46 项)
|
|
||||||
- OCR、验证码、图像模板匹配、视觉定位(100 项;最多做"截图 + 区域保存")
|
|
||||||
- NLP/AIGC/云服务类(NLP、DataService、Scenario 等)
|
|
||||||
- SAP、IE、Java 应用、Chrome 扩展等厂商专有通道
|
|
||||||
- 并行分支、分布式执行、审批流、多租户、云端市场
|
|
||||||
- 厂商 Python 运行时对接(本机未安装该产品,且无授权运行时)
|
|
||||||
|
|
||||||
### 2.3 "未接入"组件的产品处理
|
|
||||||
|
|
||||||
建议(待确认):**目录全量保留 + 状态徽标**。
|
|
||||||
|
|
||||||
- 可执行:绿色圆点 + 可拖拽
|
|
||||||
- 待接入:灰色徽标「待接入」,可拖拽但节点置灰,运行时在日志中明确报 `capability_not_implemented`,不静默跳过
|
|
||||||
- 目录顶部显示统计:`1016 项目录 / 当前可执行 N 项`
|
|
||||||
- 属性面板仍可编辑(元数据完整),便于后续接执行器
|
|
||||||
|
|
||||||
这样既满足"1016 个能力都能拖"的观感,又不谎称可执行。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 3. 推荐技术方案
|
|
||||||
|
|
||||||
### 3.1 形态
|
|
||||||
|
|
||||||
**新建独立 Electron 桌面应用(`D:\RPA`),不直接改 `Agent-Employee`。**
|
|
||||||
|
|
||||||
理由:
|
|
||||||
|
|
||||||
1. `Agent-Employee` 是 OpenCode v2 fork + 服务端栈,启动/构建/依赖都重;三天内集成会先耗在环境上。
|
|
||||||
2. 本产品需要主进程直接持有 `WebContentsView` 做 CDP、直接 `child_process` 调 PowerShell,独立应用最直接。
|
|
||||||
3. 目录元数据、画布视觉、动作语义都可复制/对齐,后续可平移回 `Agent-Employee`。
|
|
||||||
4. 交付物清晰:一个可双击的 exe + 源码 + 文档。
|
|
||||||
|
|
||||||
### 3.2 技术栈
|
|
||||||
|
|
||||||
| 层 | 选型 | 说明 |
|
|
||||||
|---|---|---|
|
|
||||||
| 壳 | Electron 31.7.7 | 复用本机已缓存版本,避免大版本差异 |
|
|
||||||
| 构建 | electron-vite + Vite + React 18 + TypeScript | 主进程/预加载/渲染进程一套工程 |
|
|
||||||
| 画布 | `@xyflow/react`(React Flow) | 拖拽、连线、缩放、MiniMap、分组容器,三天内唯一现实选择 |
|
|
||||||
| 状态 | Zustand + immer | 画布状态、运行状态分离 |
|
|
||||||
| 样式 | 原生 CSS 变量 + 设计令牌 | 对齐现有预览视觉(浅色、蓝灰、圆角) |
|
|
||||||
| 持久化 | 本地 JSON 文件(`userData/flows/*.json`)+ 运行记录 JSONL | 不引入数据库 |
|
|
||||||
| 执行 | 主进程 Node 解释器 + 能力注册表 | 与 UI 通过 IPC 事件流通信 |
|
|
||||||
| 桌面 | PowerShell 5.1 常驻子进程(JSON over stdio) | UIAutomation + user32 `SendInput`,免原生模块编译 |
|
|
||||||
| 浏览器 | 内嵌 `WebContentsView` + CDP | 搬 `actions.js` |
|
|
||||||
| 打包 | `@electron/packager` 产出 portable 目录/ZIP;有余量再加 NSIS | 避开 electron-builder 的签名工具下载风险 |
|
|
||||||
|
|
||||||
### 3.3 画布数据模型(草案)
|
|
||||||
|
|
||||||
```jsonc
|
|
||||||
{
|
|
||||||
"schemaVersion": 1,
|
|
||||||
"id": "flow_01",
|
|
||||||
"name": "网页订单录入",
|
|
||||||
"variables": [{ "name": "orderNo", "type": "string", "default": "" }],
|
|
||||||
"nodes": [
|
|
||||||
{ "id": "n1", "componentId": "Trigger.Manual", "position": { "x": 80, "y": 40 }, "values": {} },
|
|
||||||
{ "id": "n2", "componentId": "WebBrowser_CreateV3", "position": { "x": 80, "y": 180 },
|
|
||||||
"values": { "browserType": "内置浏览器" }, "outputVar": "web" },
|
|
||||||
{ "id": "n3", "componentId": "logic_control_if_else", "position": { "x": 80, "y": 320 },
|
|
||||||
"values": { "condition": "${amount} > 1000" } },
|
|
||||||
{ "id": "n4", "componentId": "Element_Click_V6", "position": { "x": 80, "y": 420 },
|
|
||||||
"parentId": "n3", "slot": "statement_if", "values": { "uiSelector": "…" } }
|
|
||||||
],
|
|
||||||
"edges": [{ "id": "e1", "source": "n1", "target": "n2", "sourceHandle": "out" }]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
- **节点**:`componentId` 指向 1016 目录;`values` 是字段键值;`outputVar` 接收返回值。
|
|
||||||
- **容器**:IF/循环/异常节点通过 `parentId` + `slot` 形成子画布;React Flow 原生父子节点即可承载。
|
|
||||||
- **连线**:默认顺序边;IF 节点两个出口 `true/false`;循环体走容器子节点,循环后走 `after` 出口。
|
|
||||||
- **变量**:`${var}`、`${var.field}`、`${list[0]}`;表达式使用自研小型解析器(不用 `eval`)。
|
|
||||||
|
|
||||||
### 3.4 执行模型
|
|
||||||
|
|
||||||
```
|
|
||||||
校验(组件存在/必填/连线/环检测)
|
|
||||||
→ 编译(容器树 → 执行计划)
|
|
||||||
→ 逐节点执行(上下文:变量、密钥引用、浏览器会话、证据目录)
|
|
||||||
→ 每个节点产出 { status, output, logs, evidence, errorCode, durationMs }
|
|
||||||
→ 失败按节点策略(停止 / 重试 N 次 / 跳过 / 人工确认)
|
|
||||||
```
|
|
||||||
|
|
||||||
错误码封闭集合:`not_found / timeout / blocked_url / capability_not_implemented / expression_error / io_error / office_error / desktop_error / cancelled / unknown`。
|
|
||||||
|
|
||||||
**证据**:每步可截图(浏览器/桌面)、保存输出摘要;运行目录 `userData/runs/<runId>/`,路径只回传相对路径。
|
|
||||||
|
|
||||||
### 3.5 目录 → 表单 → 执行器
|
|
||||||
|
|
||||||
1. 启动时把 1016 项加载为 `ComponentDefinition`(离线 JSON,随包)。
|
|
||||||
2. 字段 `displayType`(200+ 种)归并为 12 类控件:文本/表达式、数字、下拉、单选、复选、文件、目录、多行、代码、密钥、元素选择器、区域选择;未知类型降级为文本并标注。
|
|
||||||
3. 执行器注册表以 `functions[].function` 为键:
|
|
||||||
|
|
||||||
```ts
|
|
||||||
register("StrUtil.SplitStr", (ctx, args) => { … })
|
|
||||||
register("Excel.Cs_ReadContentV2", async (ctx, args) => { … })
|
|
||||||
```
|
|
||||||
|
|
||||||
4. 未注册的函数在编译期标记 `implemented=false`,运行时返回 `capability_not_implemented`,并给出"已注册 N/997"的统计。
|
|
||||||
|
|
||||||
### 3.6 桌面桥(关键风险点)
|
|
||||||
|
|
||||||
PowerShell 5.1 常驻进程,stdin/stdout 行分隔 JSON:
|
|
||||||
|
|
||||||
| 命令 | 实现 |
|
|
||||||
|---|---|
|
|
||||||
| `win.find/activate/state/rect/close` | user32 `FindWindow/EnumWindows/SetForegroundWindow/ShowWindow` |
|
|
||||||
| `uia.find/invoke/setValue/getText` | `UIAutomationClient` 程序集 |
|
|
||||||
| `input.click/move/scroll/type/hotkey` | user32 `SendInput`(绝对坐标用 `MOUSEEVENTF_ABSOLUTE`) |
|
|
||||||
| `screen.capture` | `System.Drawing` 截屏 |
|
|
||||||
| `clip.get/set` | `System.Windows.Forms.Clipboard` |
|
|
||||||
|
|
||||||
要点:
|
|
||||||
- 启动时声明 Per-Monitor DPI Awareness,坐标统一用**物理像素**;Electron 侧按 `screen.getPrimaryDisplay().scaleFactor` 换算。
|
|
||||||
- UIA 操作优先(元素可访问),坐标点击作为兜底并标注风险。
|
|
||||||
- 目标窗口不在前台时先 `SetForegroundWindow`,失败要显式报错,不盲点。
|
|
||||||
|
|
||||||
### 3.7 打包与交付
|
|
||||||
|
|
||||||
- `npm run build` → `dist/win-unpacked/`,双击 `RPA Studio.exe` 启动。
|
|
||||||
- `npm run package` → ZIP + 版本清单 + SHA256。
|
|
||||||
- 交付:源码、可执行包、`README`、示例流程 2 条、自测报告、演示脚本。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 4. 三天排期(含今天 2026-10-26)
|
|
||||||
|
|
||||||
> 假设:单人 + AI 辅助,今天可用约 12 小时,后两天各约 12 小时。每半天一个可验收产物。
|
|
||||||
|
|
||||||
### Day 1(10/26):地基 + 画布
|
|
||||||
| 时段 | 任务 | 验收 |
|
|
||||||
|---|---|---|
|
|
||||||
| 上午 | 工程脚手架(Electron+Vite+React+TS)、设计令牌、三栏布局、路由 | `npm run dev` 起窗口 |
|
|
||||||
| 下午 | 目录加载与 11 组分组、搜索/虚拟列表、拖拽入画布、节点卡片、连线/删除/框选、缩放/MiniMap | 能从 1016 项拖 5 个节点并连线 |
|
|
||||||
| 晚上 | 属性面板(12 类控件)、变量面板、流程保存/加载、撤销重做 | 刷新后流程还原 |
|
|
||||||
| 睡前 | 容器节点(IF/循环)父子拖放与几何 | 能把节点拖进 IF 分支 |
|
|
||||||
|
|
||||||
### Day 2(10/27):执行内核 + 浏览器 + 桌面
|
|
||||||
| 时段 | 任务 | 验收 |
|
|
||||||
|---|---|---|
|
|
||||||
| 上午 | 解释器(编译/执行/变量/表达式/错误策略/日志事件) | 直线流程可跑 |
|
|
||||||
| 下午 | 浏览器运行时(搬 `actions.js`)+ 元素拾取器 + 截图证据 | 打开网页、点击、输入、取文本 |
|
|
||||||
| 晚上 | 桌面桥(PowerShell)+ 键鼠/窗口/剪贴板/对话框 | 能操作记事本/计算器 |
|
|
||||||
| 睡前 | 数据/文件/字符串/时间 批量执行器(60+) | 单节点用例通过 |
|
|
||||||
|
|
||||||
### Day 3(10/28):Office/网络 + 联调 + 打包
|
|
||||||
| 时段 | 任务 | 验收 |
|
|
||||||
|---|---|---|
|
|
||||||
| 上午 | Excel/PDF/Word、HTTP/邮件执行器 | 读写 xlsx、发测试邮件 |
|
|
||||||
| 下午 | 运行面板(状态高亮/日志/变量/证据)、停止/重试、示例流程 | 端到端流程跑通 |
|
|
||||||
| 晚上 | 打包 portable、冒烟(100%/125%/150%)、README/演示脚本/自测报告 | 双击 exe 可演示 |
|
|
||||||
|
|
||||||
**若进度落后,砍功能顺序**:FTP/DB → Word/PDF → 邮件 → 桌面 UIA(保留坐标键鼠)→ 循环容器(保留 IF)→ 目录全量(退化为可执行子集)。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 5. 建议的验收脚本(Demo)
|
|
||||||
|
|
||||||
**场景 A(主推,跨 Web + Office + 消息)**
|
|
||||||
1. 打开内置浏览器 → 访问本地夹具页/演示站
|
|
||||||
2. 填写表单(姓名/金额)→ 点击提交
|
|
||||||
3. 抓取结果表格 → 写入 Excel(新建/指定模板)
|
|
||||||
4. IF 金额 > 1000 → 发邮件/桌面通知;否则记录日志
|
|
||||||
5. 运行面板显示每步耗时、截图、变量值
|
|
||||||
|
|
||||||
**场景 B(纯桌面 + 文件)**
|
|
||||||
1. 启动记事本 → 输入文本 → 保存到指定目录
|
|
||||||
2. 读取目录内 xlsx → 筛选 → 另存为新文件
|
|
||||||
3. 弹窗提示完成
|
|
||||||
|
|
||||||
**画布验收**:1016 项可搜、11 组筛选、拖拽/连线/容器/变量/保存重载/运行高亮/失败定位。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 6. 风险清单
|
|
||||||
|
|
||||||
| 风险 | 影响 | 缓解 |
|
|
||||||
|---|---|---|
|
|
||||||
| "1016 个组件"被理解成"1016 个都能跑" | 验收预期落差 | 目录徽标 + 启动页统计 + 运行时明确报错;文档写清边界 |
|
|
||||||
| 三天做不完 | 交付缩水 | 纵向切片优先;未完成项按"砍功能顺序"降级 |
|
|
||||||
| 桌面自动化 DPI/焦点/权限 | 演示失败 | PowerShell 桥显式 DPI、UIA 优先、坐标兜底;100/125/150 冒烟 |
|
|
||||||
| Excel COM 慢/弹窗/版本差异 | 卡住流程 | 文件级用 `exceljs`;COM 只用于"可见打开"演示 |
|
|
||||||
| 内嵌浏览器与用户 Chrome 差异 | 登录态/站点兼容 | 明确"内置浏览器",不做外部 Chrome 接管 |
|
|
||||||
| 未接入组件被拖入流程 | 运行中断 | 编译期阻断 + 画布红标 + 一键定位 |
|
|
||||||
| C 盘空间不足 | 构建失败 | 全部依赖与缓存放 D 盘;Electron 复用本地缓存 |
|
|
||||||
| 第三方包下载失败 | 阻塞 | 已确认 npm 源可达;关键包锁定版本并本地缓存 |
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 7. 待产品方确认(详见对话提问)
|
|
||||||
|
|
||||||
1. 基底:独立新项目 vs 并入 `Agent-Employee`
|
|
||||||
2. 三天验收重心:纵向切片 / 组件广度 / 画布完成度
|
|
||||||
3. 必须真实执行的类别
|
|
||||||
4. 未接入组件的处理方式
|
|
||||||
5. 画布控制结构深度(IF / 循环 / 并行)
|
|
||||||
6. 演示场景(网页+Excel+邮件 / 纯桌面 / 自定义系统)
|
|
||||||
7. 交付物形态(portable exe / 安装包 / 源码 / 视频)
|
|
||||||
8. 环境约束(可装 npm 包、可调 PowerShell、是否离线、缩放适配)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 8. 参考文件索引
|
|
||||||
|
|
||||||
- 目录数据:`D:\code\Agent-Employee\code\frontend\apps\desktop\src\renderer\product-preview\pages\workflows\shizai-catalog-data.js`
|
|
||||||
- 目录投影:同目录 `component-catalog.js`
|
|
||||||
- 现有流程页:同目录 `page.js`、`index.html`
|
|
||||||
- 设计原型:`D:\code\Agent-Employee\项目资料\UI设计\20261021\流程编排页原型.html`
|
|
||||||
- 浏览器动作层:`C:\rpa-demo\actions.js`、`main.js`、`flows.js`、`selftest.js`
|
|
||||||
- 动作契约:`C:\rpa-demo\AI-接手开发指南.md`
|
|
||||||
- 浏览器插件契约:`D:\code\Agent-Employee\code\backend\packages\browser-plugin\src\index.ts`
|
|
||||||
- Windows 只读 Runner:`D:\code\Agent-Employee\code\backend\services\windows-runner\native\NativeWindows.cs`
|
|
||||||
- 厂商实现分析:`D:\code\Agent-Employee\项目资料\开发方案\实在ai分析结果\01-总览与结论\全功能实现逻辑总览.md`
|
|
||||||
661
docs/02-实施方案.md
661
docs/02-实施方案.md
@ -1,661 +0,0 @@
|
|||||||
# 画布式 RPA 桌面端 · 实施方案(三天)
|
|
||||||
|
|
||||||
> 制定时间:2026-10-26 09:50
|
|
||||||
> 依据:`D:\code\Agent-Employee`(只读)、`C:\rpa-demo`(只读)、2026-10-26 需求确认
|
|
||||||
> 本方案是执行基线:范围、契约、排期、验收、降级顺序都在这里;实施中变更必须回写本文件。
|
|
||||||
|
|
||||||
> **2026-10-30 当前状态**:常用本地能力已进入注册表或运行时容器,覆盖控制/触发、除数据库外的数据处理、文件、网页元素层级/表格/下拉框/浏览器控制/Cookie/下载及 DOM 高级能力、HTTP、Excel 常用读写/格式/筛选/历史版本、系统路径与环境变量、进程管理、桌面键鼠/窗口对象/剪贴板图片/对话框、PDF、Word 文档编辑、SMTP 与群机器人 Webhook、文本差异、工作日判断、屏幕截图、离线图像查找与点击、FTP、人机交互表单对话框、Windows 凭据/远程桌面/屏保、ECharts 图表读取、AI 文本能力(用户自配服务)、离线 OCR;启动后目录显示 **732 项可执行 / 1016 项总目录**。当前 26 个测试文件共 167 条测试,浏览器 E2E 真实结果为“订单创建成功:RPA 客户,金额 128”。数据处理仅剩 8 个待评估的数据库组件。设置页可查看分类覆盖和处理器 ID 数量。此后以集中场景测试与稳定性修复为主,不再追求厂商云端、硬件和专用通道数量。(原 2026-10-27 快照:619 项可执行 / 23 个测试文件 138 条测试)
|
|
||||||
|
|
||||||
> 浏览器端到端门禁:`npm run test:e2e` 使用隔离用户目录启动应用和本地演示站,真实执行打开网页、两次可信输入、点击提交和读取结果;期望结果必须精确包含“订单创建成功:RPA 客户,金额 128”。浏览器输入、点击、定位歧义和遮挡检查机制已对照 `C:\rpa-demo/actions.js` 校准。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 0. 决策记录(2026-10-26 确认)
|
|
||||||
|
|
||||||
| # | 问题 | 结论 |
|
|
||||||
|---|---|---|
|
|
||||||
| 1 | 代码基底 | 在 `D:\RPA` 新建独立 Electron 应用 |
|
|
||||||
| 2 | 三天验收重心 | 一条真实流程端到端跑通 |
|
|
||||||
| 3 | 必须真实执行 | 浏览器网页、文件与 Excel、桌面键鼠/窗口/剪贴板/对话框、数据/字符串/时间/列表/正则、HTTP 与邮件、PDF/Word;图像/OCR「最好有」;移动端不做;数据库/FTP 靠后 |
|
|
||||||
| 4 | 未接入组件 | **全量 1016 可拖拽;未接入项灰标「待接入」,运行时明确报错**(2026-10-26 10:20 调整,覆盖早前"只导入可执行子集") |
|
|
||||||
| 5 | 画布深度 | 顺序 + IF + 循环容器 |
|
|
||||||
| 6 | 演示场景 | 待定;默认用本地夹具页,可替换为你的系统 |
|
|
||||||
| 7 | 元素拾取 | 浏览器必须;桌面手工填 |
|
|
||||||
| 8 | 交付物 | 可双击运行的 portable 包 + 自测报告(源码随仓库交付) |
|
|
||||||
| 9 | 约束 | 可装 npm 依赖;可用 PowerShell 做桌面自动化;**运行期完全离线**;适配 125%/150% 缩放 |
|
|
||||||
|
|
||||||
### 需要你知晓的三个默认取舍
|
|
||||||
|
|
||||||
1. **产品包含 1016 全量目录**:可执行项正常拖拽;未接入项灰标「待接入」,可拖入画布但运行时在该节点给出 `capability_not_implemented` 与替代建议。目录数据随包(约 1.8 MB)。
|
|
||||||
2. **OCR 不阻塞验收**:先做本地图像查找/点击(OpenCV 模板匹配,离线);OCR 用 Windows 自带引擎作为可选加分项,做不成就在自测报告中标注未交付。
|
|
||||||
3. **厂商资料只做行为参考**:`实在ai分析结果` 与 `Rea-AgentShizai-cankao` 用于校准流程格式、组件字段、执行语义、错误与选择器模型;不复制厂商代码/二进制/模型/前端资产(授权未确认,见复用审查)。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 1. 交付定义(Definition of Done)
|
|
||||||
|
|
||||||
三天结束时,必须能在一台干净的 Windows 11 x64 机器上:
|
|
||||||
|
|
||||||
1. 解压 `RPA-Studio-0.1.0-win-x64.zip`,双击 `RPA Studio.exe` 启动,**无需联网、无需 Node/Python**(Word/图像为可选能力,缺依赖时显式降级)。
|
|
||||||
2. 左侧组件库可见 **1016 项**,按 11 组与「全部/可执行/待接入」筛选;可执行项正常拖拽,待接入项灰标但仍可拖入画布。
|
|
||||||
3. 新建流程 → 从左侧组件库拖入 ≥6 个能力 → 连线 → 配置参数、变量、超时/重试/失败策略 → 保存。
|
|
||||||
4. 点「运行」后流程真实执行,画布节点按状态高亮,运行面板显示每步日志/耗时/变量/截图;待接入节点在此处给出明确失败信息。
|
|
||||||
5. 跑通一条端到端流程:**内置浏览器打开本地夹具页 → 填表提交 → 抓取结果表格 → 写入 Excel → 按金额分支 → 桌面通知/发信**。
|
|
||||||
6. 故意制造失败(选择器不存在/文件不存在/待接入组件),流程在该节点停止,日志给出可分类错误码、原因、建议与证据。
|
|
||||||
7. 自测报告列出全部用例与通过情况,明确"可执行/待接入"清单。
|
|
||||||
|
|
||||||
**非目标(三天)**:移动端、机械臂、验证码、云端 OCR、并行分支、分布式、审批流、多租户、市场、录制器、AI 生成流程。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 2. 范围基线
|
|
||||||
|
|
||||||
### 2.1 产品内可执行组件(计划 130,验收目标 120,保底 90;目录仍展示 1016 全量)
|
|
||||||
|
|
||||||
| 族 | 目标 | 代表组件 | 实现要点 |
|
|
||||||
|---|---:|---|---|
|
|
||||||
| 控制/触发 | 14 | `logic_control_if_else`、`logic_control_count_for`、`logic_control_array_for_v3`、`logic_control_while`、`logic_control_try_catch`、`logic_control_var`、`logic_control_break_for`、`logic_control_next_for`、`logic_control_stop`、`Trigger.Manual`、`Trigger.FileArrival` | 自研解释器;容器父子节点;IF 双出口;循环带最大次数保护 |
|
|
||||||
| 浏览器/元素 | 28 | `WebBrowser_CreateV3`、`WebBrowser_GoUrl`、`WebBrowser_Switch`、`Element_Click_V6`、`Element_SetValue_V6`、`Element_GetText_V5`、`Element_Exists`、`Element_SelectOperation_v1`、`WebBrowser_RunJs`、`WebBrowser_SaveAs` | 内嵌 `WebContentsView` + CDP;移植 `C:\rpa-demo\actions.js`;五级定位;拾取器 |
|
|
||||||
| 文件 | 14 | `File_Read`、`File_Write`、`File_Copy`、`File_File_Move`、`File_DeleteFile`、`File_Rename`、`File_FileExists`、`File_GetFileList`、`File_MakeDir`、`File_CompressFile`、`File_DecompressFile`、`File_GetFileInfo` | Node `fs` + `adm-zip` |
|
|
||||||
| Excel/CSV | 18 | `Excel_Cs_NewV5`、`Excel_Cs_OpenV4`、`Excel_Cs_ReadContentV2`、`Excel_Cs_WriteCell`、`Excel_Cs_WriteRange`、`Excel_Cs_ReadRangeV2`、`Excel_Save`、`Excel_Cs_SaveAs`、`Excel_GetSheetsName`、`Excel_Cs_Insert`、`Excel_Cs_Rename`、`CSV_Read`、`CSV_Write` | `exceljs` 文件级;COM 只用于"可见打开"演示 |
|
|
||||||
| 数据/文本 | 28 | `StrUtil_InStr/CutOutStrV1/SplitStr/Replace/CaseChange/Trim/Len/Left/Right/Mid`、`Time_GetDate/FormatNew/DateOffset`、`List_CreateList/InsertList/ListLength/ListSorted`、`Dict_Create/GetItem/SetItem/Keys`、`Regex_Test/FindAll`、`Math_Round/Abs`、`JSON_Parse/Stringify`、`Basic_SetVariable/ConvertNumber` | 纯 JS,表驱动批量接入 |
|
|
||||||
| 桌面 | 16 | `Keyboard_Send_V1`、`Keyboard_PressNew`、`Mouse_Click`、`Mouse_MoveTo`、`Mouse_DragTo`、`Mouse_RollTo`、`Window_SetActive`、`Window_GetActive`、`Window_WindowExists`、`Window_CloseWindow`、`App_Run`、`Clipboard_GetText`、`Clipboard_SetText`、`Dialog_Notice`、`Dialog_MsgBox`、`System_ExecuteCommand` | PowerShell 5.1 常驻桥:UIA + `SendInput` + user32 |
|
|
||||||
| 网络/邮件 | 4 | `Http_Get`、`Http_Post`、`Mail_Send`、`Mail_UnionSend` | `fetch` + `nodemailer` |
|
|
||||||
| PDF/Word | 5 | `PDF_GetPdfPageNum`、`PDF_GetPdfText`、`PDF_ConcatPdf`、`Word_New`/`Word_Write`(含保存) | `pdf-lib` + `pdfjs-dist`;Word 走 COM |
|
|
||||||
| 图像 | 3 | `Image_Screenshot`、`Image_Find`、`Image_Click` | 截屏 + 模板匹配(OpenCV/Python 3.10 已装) |
|
|
||||||
|
|
||||||
> 说明:
|
|
||||||
> 1. 注册数以"函数名 → 实现"计;同一实现可挂多个组件变体(如 `Element_Click_V6` 与其变体)。最终以自测报告为准,未通过的不计入;**保底 90 项**是验收底线。
|
|
||||||
> 2. 目录展示 **1016 全量**(997 + 16 + 3)。上表是"能真实执行"的子集;其余组件灰标「待接入」,属性面板仍可查看/编辑元数据,运行时给出 `capability_not_implemented`。
|
|
||||||
> 3. 组件字段与校验从 `componentsNew.json` 的 `definition.args0` 派生:`vType`(rpa_str/int/float/bool/list/dict/excel/hWeb/window/any)、`validatorRules`、`displayParam`、`tip`、`isAdvanced`。
|
|
||||||
|
|
||||||
### 2.2 明确不交付
|
|
||||||
|
|
||||||
- 移动端/外设/机械臂/U 盾/远程桌面(46+ 项)
|
|
||||||
- OCR 验证码、NLP、AIGC、云服务类组件
|
|
||||||
- SAP/IE/Java 应用/浏览器扩展通道
|
|
||||||
- 并行、子流程跨文件、审批、多租户、云同步
|
|
||||||
|
|
||||||
### 2.3 路线图(三天后)
|
|
||||||
|
|
||||||
| 阶段 | 内容 |
|
|
||||||
|---|---|
|
|
||||||
| P2 | 剩余组件接入;厂商 `.task` 导入/导出;完整选择器结构(commonProps/selector[])兼容 |
|
|
||||||
| P3 | 桌面 UIA 元素拾取、窗口选择器、图像 OCR、邮件 IMAP |
|
|
||||||
| P4 | 数据库/FTP/SAP、子流程与并行、审批与证据中心 |
|
|
||||||
| P5 | 与 `Agent-Employee` 的 Workflow/插件契约对齐、能力包分发 |
|
|
||||||
|
|
||||||
### 2.4 两份参考资产怎么用(2026-10-26 补充)
|
|
||||||
|
|
||||||
**结论:高价值行为参考,不做代码复制。** 它们解决四个关键问题:流程格式、组件定义、执行语义、错误与选择器模型。
|
|
||||||
|
|
||||||
| 资产 | 位置 | 用途 | 边界 |
|
|
||||||
|---|---|---|---|
|
|
||||||
| 复用审查(决策入口) | `实在ai分析结果/01-总览与结论/开发参考与复用审查.md` | 判断哪些机制可参考、哪些不能 | 厂商代码授权未确认,只参考机制 |
|
|
||||||
| 组件字典 | `Rea-AgentShizai-cankao/code/rpa-engine/componentDicts/componentsNew.json`(997 项) | 补全 `vType`、`validatorRules`、`displayParam`、`tip`、`isAdvanced`,驱动类型化表单与校验 | 仅构建期派生,不随包分发原始 JSON |
|
|
||||||
| 真实工程样例 | 同目录 `code/app/resources/examples/*.zip`(网页填表、新闻抓取、微信回复) | 对照 `.task` / `project.flow` / `globalVariable` / 生成的 `projects/flow_modules/*.py`,校准流程与执行语义 | 只读行为参考,不复制业务内容与截图 |
|
|
||||||
| 任务格式 | 样例包内 `*.task` | `componentModelEntities` + `MAIN_COMPONENT.statements` + `args0`/`options0`;据此定义节点 `options` 与容器 `statements` | 三天不做导入/导出 |
|
|
||||||
| 画布格式 | 样例包内 `project.flow` | 节点 x/y/width/height、shape、edges、anchors、`reTryCount`、`errorVar` | 厂商是"块级画布 + 块内步骤",我们是单层画布 |
|
|
||||||
| 执行与错误语义 | `Rea.../code/derived-engine-core`(203 份源)+ `rpa-engine/conf/err_code/err_codes.py` | 重试/超时/skip/log 规则、错误 reason+recommendation、元素变量插值 | 不复制实现;用公开库重写 |
|
|
||||||
| 选择器模型 | 样例 `globalVariable` 的 `SHIZAI_ELEMENT_DICT`;`common/util/elements_util.py` | `{commonProps, selector:[{name, props:[{name,value,pattern}]}]}`;`${}` 插值;pattern 枚举 | v1 存简化选择器,P2 兼容完整结构 |
|
|
||||||
| 组件包 | `rpa-engine/311_32/packages/*`(54 包,`.pyc` + `.pyi` + `config.json`) | 查函数签名、表单 schema、参数顺序 | 字节码不是源码,不复制 |
|
|
||||||
| 前端/插件 | `code/app/resources/*`、`code/support/*` | UI 交互、浏览器扩展安装、Office 插件行为参考 | 资产与二进制不复制 |
|
|
||||||
|
|
||||||
**采纳到契约里的具体设计:**
|
|
||||||
|
|
||||||
1. 节点 `options` 对齐 `options0`:`delayBeforeMs`、`delayAfterMs`、`timeoutMs`、`retryCount`(1..100)、`retryIntervalMs`(1..600s)、`skipOnError`、`logException`、`outputVar`。
|
|
||||||
2. 容器子节点保存为有序 `statements`(等价厂商 `input_statement`);画布仍用连线 + 父子关系,保存时编译为 statements。
|
|
||||||
3. 变量类型映射:`rpa_str/int/float/bool/list/dict/excel/hWeb/window/any` → 我们的 `string/number/boolean/list/dict/excel/browser/window/any`。
|
|
||||||
4. 错误对象扩展为 `{ code, reason, recommendation, location:{flowId,nodeId,componentId,arg} }`;失败日志展示"原因 + 建议 + 定位"。
|
|
||||||
5. 选择器 v1 存 `{kind:'css'|'aria'|'text', value, name?}`;拾取器同时产出厂商式候选,P2 再落完整 `commonProps/selector[]`。
|
|
||||||
6. 用真实样例做回归夹具:在本地夹具页复刻"读 Excel → 填网页 → 提交"的链路,验证语义,不复制内容。
|
|
||||||
7. 执行策略对齐:`skipOnError` 记错误日志并继续;`retryCount` 仅对可重试错误生效;参数校验失败不重试;`timeoutMs` 为整次调用总时限。
|
|
||||||
|
|
||||||
**明确不做:** 复制厂商 `.pyc/.dll/.exe/模型/前端资产`;调用厂商云服务;内置厂商运行时;三天内做 `.task` 导入/导出。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 3. 产品形态与信息架构
|
|
||||||
|
|
||||||
```
|
|
||||||
┌──────────────────────────────────────────────────────────────┐
|
|
||||||
│ 顶栏:Logo | 流程名 | 保存状态 | 撤销/重做 | 缩放 | ▶运行 ■停止 │
|
|
||||||
├─────────┬──────────────────────────────────────┬─────────────┤
|
|
||||||
│ 左栏 │ 画布(点阵/节点/连线/容器/MiniMap) │ 右栏 │
|
|
||||||
│ 流程列表 │ │ 属性/变量/运行 │
|
|
||||||
│ 组件库 │ │ │
|
|
||||||
│ 变量 │ │ │
|
|
||||||
│ 运行历史 │ │ │
|
|
||||||
├─────────┴──────────────────────────────────────┴─────────────┤
|
|
||||||
│ 底部运行抽屉:时间 | 节点 | 状态 | 耗时 | 输出/错误 | 截图 │
|
|
||||||
└──────────────────────────────────────────────────────────────┘
|
|
||||||
```
|
|
||||||
|
|
||||||
页面:
|
|
||||||
1. **首页/流程列表**:新建、导入、最近运行、`1016 项目录 / 可执行 N 项`。
|
|
||||||
2. **编辑器**(核心):左组件库 + 中画布 + 右属性/运行。
|
|
||||||
3. **运行历史**:run 列表、状态、耗时、日志与证据查看。
|
|
||||||
4. **设置**:浏览器域名白名单、证据开关、超时、外观。
|
|
||||||
|
|
||||||
组件库展示 **1016 全量**,分组沿用 11 组;顶部显示「1016 项目录 · 可执行 N 项」,并提供「全部 / 可执行 / 待接入」筛选。可执行项正常拖拽;待接入项灰标、可拖入但运行时明确失败。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 4. 技术架构
|
|
||||||
|
|
||||||
### 4.1 技术栈
|
|
||||||
|
|
||||||
| 层 | 选型 |
|
|
||||||
|---|---|
|
|
||||||
| 壳 | Electron 31.7.7(复用本机缓存) |
|
|
||||||
| 构建 | electron-vite + Vite + React 18 + TypeScript |
|
|
||||||
| 画布 | `@xyflow/react`(React Flow) |
|
|
||||||
| 状态 | Zustand(flowStore / runStore / uiStore) |
|
|
||||||
| 样式 | CSS 变量 + 设计令牌,无 UI 框架依赖 |
|
|
||||||
| 存储 | 本地 JSON(`userData`) |
|
|
||||||
| 测试 | Vitest(单元)+ `scripts/selftest.mjs`(Electron 主进程集成) |
|
|
||||||
|
|
||||||
### 4.2 目录结构
|
|
||||||
|
|
||||||
```
|
|
||||||
D:\RPA
|
|
||||||
├─ package.json
|
|
||||||
├─ electron.vite.config.ts
|
|
||||||
├─ resources/
|
|
||||||
│ ├─ catalog.json # 1016 全量 + 可执行状态 + 类型/校验元数据
|
|
||||||
│ ├─ fixture/ # 本地演示站点(离线)
|
|
||||||
│ └─ desktop-bridge.ps1 # 桌面自动化桥
|
|
||||||
├─ src/
|
|
||||||
│ ├─ shared/ # 类型、常量、错误码(主/渲染共用)
|
|
||||||
│ ├─ main/
|
|
||||||
│ │ ├─ index.ts # 生命周期、窗口、协议
|
|
||||||
│ │ ├─ ipc.ts # 类型化 IPC 路由
|
|
||||||
│ │ ├─ store/ # flows/runs/settings/secrets
|
|
||||||
│ │ ├─ engine/ # compiler/runtime/expression/context
|
|
||||||
│ │ ├─ capabilities/ # control/browser/files/excel/data/desktop/network/office/image
|
|
||||||
│ │ ├─ browser/ # view 管理、CDP、picker、evidence
|
|
||||||
│ │ ├─ bridge/ # PowerShell 客户端、Python 图像客户端
|
|
||||||
│ │ └─ fixture-server.ts
|
|
||||||
│ ├─ preload/index.ts # contextBridge 白名单 API
|
|
||||||
│ └─ renderer/
|
|
||||||
│ ├─ pages/ # Home/Editor/Runs/Settings
|
|
||||||
│ ├─ canvas/ # FlowCanvas/NodeCard/ContainerNode/Palette/edges
|
|
||||||
│ ├─ panels/ # Properties/Variables/RunPanel/Evidence
|
|
||||||
│ ├─ state/
|
|
||||||
│ └─ styles/
|
|
||||||
├─ examples/ # 示例流程 JSON
|
|
||||||
├─ scripts/ # build-catalog / selftest / package
|
|
||||||
├─ docs/
|
|
||||||
└─ release/ # portable 产物(不入 Git)
|
|
||||||
```
|
|
||||||
|
|
||||||
### 4.3 进程与边界
|
|
||||||
|
|
||||||
- **主进程**:唯一拥有文件系统、子进程、浏览器视图、执行引擎;渲染进程不能直接 `require`。
|
|
||||||
- **预加载**:`contextIsolation: true`,只暴露 `flow.* / run.* / catalog.* / browser.* / desktop.* / settings.*` 白名单方法。
|
|
||||||
- **渲染进程**:只做 UI 与状态;运行状态通过 `run:event` 订阅。
|
|
||||||
- **PowerShell 桥**:常驻子进程,行分隔 JSON;超时/崩溃自动重启一次并显式报错。
|
|
||||||
- **Python 图像桥**:按需启动,仅图像查找用;不可用时功能降级。
|
|
||||||
|
|
||||||
### 4.4 运行期离线
|
|
||||||
|
|
||||||
- 所有依赖打包进 asar/资源目录;不加载 CDN 字体/脚本。
|
|
||||||
- 内置浏览器只访问白名单(默认 `127.0.0.1`、`localhost`)。
|
|
||||||
- 夹具站由主进程在 `127.0.0.1` 随机端口启动。
|
|
||||||
- HTTP/邮件属于用户显式配置的外部系统,不是云依赖;演示默认用本地 mock。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 5. 关键契约
|
|
||||||
|
|
||||||
### 5.1 流程定义(`FlowDefinition` v1)
|
|
||||||
|
|
||||||
```jsonc
|
|
||||||
{
|
|
||||||
"schemaVersion": 1,
|
|
||||||
"id": "flow_...",
|
|
||||||
"name": "网页订单录入",
|
|
||||||
"description": "",
|
|
||||||
"variables": [
|
|
||||||
{ "name": "amount", "type": "number", "default": 0, "scope": "flow" }
|
|
||||||
],
|
|
||||||
"nodes": [
|
|
||||||
{
|
|
||||||
"id": "n1",
|
|
||||||
"componentId": "Trigger.Manual",
|
|
||||||
"functionName": "Trigger.Manual",
|
|
||||||
"position": { "x": 120, "y": 80 },
|
|
||||||
"values": {},
|
|
||||||
"options": { "outputVar": "", "timeoutMs": 30000, "skipOnError": false }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "n2",
|
|
||||||
"componentId": "logic_control_if_else",
|
|
||||||
"functionName": "logic_control_if_else",
|
|
||||||
"position": { "x": 120, "y": 260 },
|
|
||||||
"values": { "condition": "${amount} > 1000" },
|
|
||||||
"parentId": null
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "n3",
|
|
||||||
"componentId": "Element_Click_V6",
|
|
||||||
"position": { "x": 60, "y": 380 },
|
|
||||||
"parentId": "n2",
|
|
||||||
"slot": "statement_if",
|
|
||||||
"values": { "uiSelector": "css:#submit" },
|
|
||||||
"options": {
|
|
||||||
"outputVar": "",
|
|
||||||
"delayBeforeMs": 200,
|
|
||||||
"delayAfterMs": 200,
|
|
||||||
"timeoutMs": 15000,
|
|
||||||
"retryCount": 0,
|
|
||||||
"retryIntervalMs": 1000,
|
|
||||||
"skipOnError": false,
|
|
||||||
"logException": true
|
|
||||||
},
|
|
||||||
"enabled": true
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"edges": [
|
|
||||||
{ "id": "e1", "source": "n1", "target": "n2", "sourceHandle": "out", "targetHandle": "in" }
|
|
||||||
],
|
|
||||||
"meta": { "createdAt": "...", "updatedAt": "...", "appVersion": "0.1.0" }
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 5.2 容器语义
|
|
||||||
|
|
||||||
| 节点 | 出口 | 子槽 | 执行 |
|
|
||||||
|---|---|---|---|
|
|
||||||
| `logic_control_if_else` | `true` / `false` | `statement_if` / `statement_else` | 条件为真走真槽链,否则走假槽链,之后汇合到主链下一节点 |
|
|
||||||
| `logic_control_count_for` | `after` | `count_for_fn` | 从 start 到 end 按 step 循环,每轮执行子链 |
|
|
||||||
| `logic_control_array_for_v3` | `after` | `array_for_fn` | 遍历列表/字典,暴露 `item`、`index` |
|
|
||||||
| `logic_control_while` | `after` | `if_for_fn` | 条件为真循环,最大 1000 次 |
|
|
||||||
| `logic_control_try_catch` | `after` | `try_fn` / `except_fn` / `else_fn` | 捕获异常,暴露 `${error}` |
|
|
||||||
|
|
||||||
子槽内节点按"连线优先、无连线按 y 排序"形成链;禁止跨槽连线;容器可嵌套。
|
|
||||||
|
|
||||||
### 5.3 表达式
|
|
||||||
|
|
||||||
- 文本字段支持 `${expr}` 插值;IF/循环条件为纯表达式。
|
|
||||||
- 语法:字面量、变量、成员/下标、算术/比较/逻辑/三元、括号。
|
|
||||||
- 函数白名单:`len, upper, lower, trim, replace, split, join, contains, startsWith, endsWith, matches, number, string, round, abs, min, max, random, now, today, formatDate, json, parseJson`。
|
|
||||||
- **不使用 `eval` / `new Function`**;自研 tokenizer + Pratt parser,错误定位到字符位置。
|
|
||||||
- 变量缺失:默认报 `expression_error`;可在设置中改为空串并告警。
|
|
||||||
|
|
||||||
### 5.4 运行事件
|
|
||||||
|
|
||||||
```ts
|
|
||||||
type RunEvent =
|
|
||||||
| { type: "run.started"; runId: string; flowId: string; startedAt: string }
|
|
||||||
| { type: "node.started"; runId: string; nodeId: string; attempt: number }
|
|
||||||
| { type: "node.log"; runId: string; nodeId: string; level: "info"|"warn"|"error"; message: string }
|
|
||||||
| { type: "node.finished"; runId: string; nodeId: string; status: NodeStatus; output?: unknown; errorCode?: string; durationMs: number; evidence: string[] }
|
|
||||||
| { type: "run.finished"; runId: string; status: "succeeded"|"failed"|"cancelled"; durationMs: number; error?: { nodeId: string; code: string; message: string } }
|
|
||||||
```
|
|
||||||
|
|
||||||
`NodeStatus = idle | running | succeeded | failed | skipped | cancelled`
|
|
||||||
|
|
||||||
### 5.5 错误模型
|
|
||||||
|
|
||||||
封闭 `code` 集合:
|
|
||||||
|
|
||||||
`not_found`、`timeout`、`ambiguous`、`blocked_url`、`capability_not_implemented`、`expression_error`、`validation_error`、`io_error`、`office_error`、`desktop_error`、`network_error`、`cancelled`、`unknown`
|
|
||||||
|
|
||||||
运行结果中的错误对象(对齐厂商 `err_codes.py` 的"原因 + 建议"风格):
|
|
||||||
|
|
||||||
```ts
|
|
||||||
type RunError = {
|
|
||||||
code: string;
|
|
||||||
reason: string; // 中文原因,如"未找到元素:css:#submit"
|
|
||||||
recommendation?: string; // 中文建议,如"检查选择器或等待条件"
|
|
||||||
location?: { flowId?: string; nodeId?: string; componentId?: string; arg?: string };
|
|
||||||
detail?: string; // 原始错误/堆栈,仅日志
|
|
||||||
};
|
|
||||||
```
|
|
||||||
|
|
||||||
### 5.6 能力注册表
|
|
||||||
|
|
||||||
```ts
|
|
||||||
type Capability = {
|
|
||||||
functionName: string; // 目录 functions[0].function
|
|
||||||
componentIds: string[]; // 允许挂载的组件
|
|
||||||
run(ctx: RunContext, args: Record<string, unknown>, node: FlowNode): Promise<CapabilityResult>;
|
|
||||||
// 可选:字段归一、选择器校验、证据策略
|
|
||||||
};
|
|
||||||
```
|
|
||||||
|
|
||||||
`capabilities/manifest.ts` 汇总所有已注册函数;`scripts/build-catalog.mjs` 生成 `resources/catalog.json`:**1016 全量** + `implemented: true|false` + 归一后的字段/类型/校验元数据。目录不再裁剪,未实现项由运行时守卫拦截。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 6. 画布交互规格
|
|
||||||
|
|
||||||
### 6.1 组件库
|
|
||||||
|
|
||||||
- 搜索(中文名/英文 ID/标签/描述),11 组筛选,状态筛选(全部/可执行/待接入),虚拟滚动。
|
|
||||||
- 每项:图标、名称、ID、字段数、分类色条、状态点(绿=可执行,灰=待接入)。
|
|
||||||
- 拖拽:HTML5 `dragstart` 写入 `componentId`;画布 `drop` 用 `screenToFlowPosition` 定位。待接入项同样可拖拽。
|
|
||||||
- 双击:追加到当前选中节点之后。
|
|
||||||
- 目录为空/无匹配:空态提示。
|
|
||||||
|
|
||||||
### 6.2 节点
|
|
||||||
|
|
||||||
- 卡片:图标 + 名称 + 副标题(关键参数摘要)+ 输出变量胶囊 + 状态点 + 错误徽标。
|
|
||||||
- 待接入节点:整体降饱和 + 右上角「待接入」徽标 + tooltip「当前版本无执行器,运行到此会停止」。
|
|
||||||
- 选中/多选/框选;拖动;吸附网格;复制/粘贴/删除;右键菜单(复制、禁用、删除、查看定义)。
|
|
||||||
- 连接点:左侧输入,右侧输出;IF 两个输出(真/假);循环一个 `after` 输出 + 容器体。
|
|
||||||
- 连线:贝塞尔/折线;非法连接(自连、重复、跨容器、成环)即时拒绝并提示。
|
|
||||||
|
|
||||||
### 6.3 容器
|
|
||||||
|
|
||||||
- IF/循环以"分组节点"呈现:标题 + 折叠 + 子区域(虚线框)。
|
|
||||||
- 拖入子区域自动设置 `parentId` 与 `slot`;拖出清除。
|
|
||||||
- 容器高度随子节点自适应,可手动调整。
|
|
||||||
- 子槽为空时显示拖拽提示。
|
|
||||||
|
|
||||||
### 6.4 画布操作
|
|
||||||
|
|
||||||
滚轮缩放(0.3–2)、空格/中键平移、`Ctrl+0` 适配、MiniMap、背景点阵、框选、对齐辅助线(可选)。
|
|
||||||
快捷键:`Ctrl+S` 保存、`Ctrl+Z/Y` 撤销重做、`Delete` 删除、`Ctrl+D` 复制、`Ctrl+Enter` 运行、`Esc` 取消选择。
|
|
||||||
|
|
||||||
### 6.5 属性面板
|
|
||||||
|
|
||||||
Tab:属性 / 变量 / 运行。
|
|
||||||
|
|
||||||
属性分区:
|
|
||||||
1. **基本**:节点名称、组件 ID、启用、备注;待接入项显示「当前版本无执行器」。
|
|
||||||
2. **输入**:按目录字段渲染(12 类控件);必填标记;默认值;占位提示;高级字段折叠;字段类型来自 `vType`。
|
|
||||||
3. **输出**:输出变量名(自动建议)、变量类型(从 `vType` 推断)。
|
|
||||||
4. **失败策略**:超时(默认 30s,上限 7 天)、重试次数(0–100)、重试间隔(1–600s)、失败后跳过、记录异常、前置/后置延时。
|
|
||||||
5. **证据**:是否截图、是否保存输出。
|
|
||||||
|
|
||||||
字段控件映射(把 200+ `displayType` 归并):
|
|
||||||
|
|
||||||
| displayType 关键词 | 控件 |
|
|
||||||
|---|---|
|
|
||||||
| text-expression / text / coloredText / label | 文本 + 变量插入 |
|
|
||||||
| textArea / multipleTextArea / quillInput | 多行文本 |
|
|
||||||
| number / integerInputNumber / inputNumber / sliderBar / rangeInput | 数字 |
|
|
||||||
| dropDownSelect / defeasibleDropDownSelect / dynamicSelect | 下拉 |
|
|
||||||
| radioGroup / radio / radioLeft / buttonGroup / coloredRadio | 单选 |
|
|
||||||
| checkbox / switchControl / multifileSwitch | 布尔/多选 |
|
|
||||||
| selectFile / multiSelectFileOrDir / saveAsDialog / selectControllerFile | 文件/目录选择 |
|
|
||||||
| comboSelector / selectUi / uiwindowselector / phoneSelector | 元素选择器(浏览器拾取) |
|
|
||||||
| code / jsonEditor / codeInput | 代码/JSON 编辑器 |
|
|
||||||
| encryption / designPassword | 密钥(Electron safeStorage) |
|
|
||||||
| 其它 | 文本兜底 + "原始类型"提示 |
|
|
||||||
|
|
||||||
校验:必填、数字格式、变量名合法、选择器非空、文件路径存在(可关)。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 7. 执行内核
|
|
||||||
|
|
||||||
### 7.1 编译期校验
|
|
||||||
|
|
||||||
1. 组件在目录中;未接入项标记 `pending`,允许保存;运行前提示「N 个待接入节点」,执行到该节点报 `capability_not_implemented` 并停止。
|
|
||||||
2. 必填字段非空(表达式允许 `${}`);类型与 `vType` 粗校验。
|
|
||||||
3. 失败策略范围:`timeoutMs` 0–604800000、`retryCount` 0–100、`retryIntervalMs` 1000–600000、延时 ≥0。
|
|
||||||
4. 连线合法:无环、无悬空、容器槽匹配。
|
|
||||||
5. 触发器唯一且在起点。
|
|
||||||
6. 变量引用可解析(未定义变量警告/错误可配)。
|
|
||||||
|
|
||||||
失败返回结构化 `validation_error[]`(含 `reason/recommendation/location`),画布定位到节点。
|
|
||||||
|
|
||||||
### 7.2 运行时
|
|
||||||
|
|
||||||
```
|
|
||||||
run(flow):
|
|
||||||
ctx = { vars: defaults, secrets, runId, signal, evidenceDir, logger }
|
|
||||||
plan = compile(flow)
|
|
||||||
emit run.started
|
|
||||||
try:
|
|
||||||
executeChain(plan.entry, ctx)
|
|
||||||
emit run.finished(succeeded)
|
|
||||||
catch AbortError: emit run.finished(cancelled)
|
|
||||||
catch e: emit run.finished(failed, node, code)
|
|
||||||
```
|
|
||||||
|
|
||||||
`executeChain`:顺序执行;遇容器递归;每节点:
|
|
||||||
1. 解析 `values`(表达式插值/类型转换)。
|
|
||||||
2. 应用 `delayBeforeMs`。
|
|
||||||
3. 执行 `capability.run`,总时限 `timeoutMs`;可重试错误按 `retryCount/retryIntervalMs` 重试(校验错误、`capability_not_implemented`、`blocked_url` 不重试)。
|
|
||||||
4. 写入 `outputVar`。
|
|
||||||
5. 应用 `delayAfterMs`;记录日志/证据/耗时。
|
|
||||||
6. 失败时:`skipOnError=true` → 记错误日志、节点状态 `skipped`、继续;否则按错误停止整个运行。
|
|
||||||
7. `logException=true` 时把堆栈写入运行日志(不写入 UI 摘要)。
|
|
||||||
|
|
||||||
### 7.3 取消
|
|
||||||
|
|
||||||
`AbortController` 贯穿:浏览器命令、PowerShell 请求、HTTP、文件循环都检查 `signal`;停止后不再启动新节点。
|
|
||||||
|
|
||||||
### 7.4 证据
|
|
||||||
|
|
||||||
- 浏览器/桌面节点可截图(PNG)。
|
|
||||||
- 每节点输出 JSON 摘要(截断 8KB)。
|
|
||||||
- 运行目录:`userData/runs/<runId>/run.json` + `nodes/<nodeId>.json` + `evidence/*.png` + `run.log`。
|
|
||||||
- UI 只显示相对路径;证据查看器内置。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 8. 运行时实现
|
|
||||||
|
|
||||||
### 8.1 浏览器
|
|
||||||
|
|
||||||
- 主窗口右侧/独立窗口嵌入 `WebContentsView`,地址栏 + 前进后退 + 刷新 + 关闭。
|
|
||||||
- 移植 `actions.js`:`navigate/click/type/select/hover/scroll/wait/extract/screenshot/tabs/runJs`;保留统一结果、错误码、后置条件、`actionId` 去重。
|
|
||||||
- 定位:`testid → aria → text → css → xpath`;导出时 xpath 仅可等价转 css。
|
|
||||||
- 选择器值 v1 存 `{ kind, value, name? }`;拾取器同时生成厂商式候选项(`commonProps + selector[]` 的简化版),P2 再落完整结构。
|
|
||||||
- 拾取器:注入 overlay,`mousemove` 高亮、点击捕获并 `preventDefault`,生成候选定位,回填属性面板;元素库(多元素复用)留到 P2,v1 直接写入节点。
|
|
||||||
- 域名白名单在任何页面动作前校验;默认仅 `127.0.0.1`/`localhost`。
|
|
||||||
- 高影响动作(click/type/下载/上传)运行前弹一次确认(可设置关闭)。
|
|
||||||
|
|
||||||
### 8.2 桌面(PowerShell 桥)
|
|
||||||
|
|
||||||
`resources/desktop-bridge.ps1`,stdin/stdout 行分隔 JSON:
|
|
||||||
|
|
||||||
| 命令 | 说明 |
|
|
||||||
|---|---|
|
|
||||||
| `win.list` / `win.find` / `win.activate` / `win.state` / `win.rect` / `win.close` | EnumWindows / SetForegroundWindow / ShowWindow |
|
|
||||||
| `uia.find` / `uia.invoke` / `uia.setValue` / `uia.getText` | UIAutomationClient(元素优先) |
|
|
||||||
| `input.click` / `input.move` / `input.drag` / `input.scroll` | user32 `SendInput` 绝对坐标 |
|
|
||||||
| `input.type` / `input.key` / `input.hotkey` | `SendInput` Unicode / 虚拟键 |
|
|
||||||
| `clip.get` / `clip.set` | WinForms Clipboard |
|
|
||||||
| `screen.capture` | System.Drawing |
|
|
||||||
| `app.run` | `Start-Process` |
|
|
||||||
| `sys.env` / `sys.cmd` | 环境变量 / 命令执行(超时+输出截断) |
|
|
||||||
|
|
||||||
DPI:
|
|
||||||
- 桥启动时 `SetProcessDpiAwarenessContext(PER_MONITOR_AWARE_V2)`。
|
|
||||||
- 所有坐标使用物理像素;Node 侧用 `screen.getPrimaryDisplay().scaleFactor` 换算。
|
|
||||||
- 125%/150% 必须跑冒烟用例。
|
|
||||||
|
|
||||||
### 8.3 Excel / PDF / Word
|
|
||||||
|
|
||||||
- Excel:`exceljs` 读写、sheet 增删改、区域读写、保存/另存;COM 只做"可见打开/关闭"(可选开关)。
|
|
||||||
- CSV:自研解析(带引号/换行)。
|
|
||||||
- PDF:`pdfjs-dist` 取文本/页数;`pdf-lib` 合并/新建。
|
|
||||||
- Word:PowerShell COM(新建/写入/保存/关闭/读取全文);无 Word 时显式报 `office_error`。
|
|
||||||
|
|
||||||
### 8.4 图像
|
|
||||||
|
|
||||||
- 截屏:Electron `desktopCapturer` 或 PowerShell。
|
|
||||||
- 模板匹配:优先 Python 3.10 + OpenCV `matchTemplate`(本机已装);未检测到 Python 时用内置 JS 降级(`pngjs` 降采样 + 归一化互相关);两者都不可用时,该组件在目录中禁用并提示 `capability_unavailable`,不影响其他功能。
|
|
||||||
- OCR:可选,Windows.Media.Ocr;不达标就标注未交付。
|
|
||||||
|
|
||||||
> Word 组件依赖本机安装 Word(COM);无 Word 时返回 `office_error`,不影响其他组件。Excel 文件级读写用 `exceljs`,不依赖 Office。
|
|
||||||
|
|
||||||
### 8.5 HTTP / 邮件
|
|
||||||
|
|
||||||
- HTTP:`fetch`(Node 22 内置),支持方法/头/查询/Body/超时/保存响应。
|
|
||||||
- 邮件:`nodemailer` SMTP;收件用 `imapflow`(如时间允许);演示用本地 mock SMTP。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 9. 数据存储与安全
|
|
||||||
|
|
||||||
| 项 | 方案 |
|
|
||||||
|---|---|
|
|
||||||
| 流程 | `%APPDATA%/rpa-studio/flows/<id>.json`,原子写(临时文件+rename) |
|
|
||||||
| 目录 | `resources/catalog.json`(1016 全量 + 状态 + 字段元数据),启动加载,只读 |
|
|
||||||
| 运行 | `runs/<runId>/`,保留最近 50 次,自动清理 |
|
|
||||||
| 设置 | `settings.json`(白名单、超时、证据、缩放) |
|
|
||||||
| 密钥 | `safeStorage` 加密后存 `secrets.json`,UI 只显示引用名 |
|
|
||||||
| 日志 | 主进程滚动日志 + 运行日志;单文件 5MB 截断 |
|
|
||||||
| 安全 | 渲染进程无 Node;IPC 白名单;表达式无 eval;浏览器协议仅 http/https;文件操作不做沙箱(本地工具定位) |
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 10. 三天排期(逐半天 + 门禁)
|
|
||||||
|
|
||||||
> 今天 10/26 从 10:00 起算;每天按 12~14 小时安排;每半天一个可演示产物。
|
|
||||||
|
|
||||||
### Day 1(10/26):工程 + 目录 + 画布 + 属性
|
|
||||||
|
|
||||||
| 时间 | 任务 | 产出/门禁 |
|
|
||||||
|---|---|---|
|
|
||||||
| 10:00–11:00 | 脚手架(electron-vite/React/TS)、窗口、三栏布局、令牌、路由 | `npm run dev` 出窗口 |
|
|
||||||
| 11:00–12:30 | 目录管线:解析 1016 → 合并 `componentsNew.json` 的 `vType/validatorRules/tip` → 分组/状态/字段归一 → `catalog.json`;组件库搜索/分组/状态筛选/虚拟滚动 | 左栏 1016 项可搜可拖,灰标正确 |
|
|
||||||
| 13:30–17:00 | 画布:拖入、节点卡、连线、选择/删除/复制、缩放/平移/MiniMap、撤销重做 | 6 节点可连线 |
|
|
||||||
| 17:00–19:30 | 容器:IF/计次/遍历 分组节点、子节点拖入、槽位、尺寸自适应 | 节点可拖进 IF 分支 |
|
|
||||||
| 20:00–22:00 | 属性面板:控件映射、校验、输出变量/类型、失败策略(超时/重试/跳过/延时)、变量面板、表达式插入 | 配置可保存到节点 |
|
|
||||||
| 22:00–23:30 | 保存/加载、示例流程、空态/错误态、Day1 演示 | 重启后流程还原 |
|
|
||||||
| **门禁 G1** | 拖 6 节点 + IF + 循环 → 配参数 → 保存 → 重启还原 → 截图留档 | 不通过不进入 Day2 |
|
|
||||||
|
|
||||||
### Day 2(10/27):执行内核 + 浏览器 + 桌面
|
|
||||||
|
|
||||||
| 时间 | 任务 | 产出/门禁 |
|
|
||||||
|---|---|---|
|
|
||||||
| 09:00–12:00 | 表达式解析器、编译器(校验/容器/链/待接入守卫)、运行时(上下文/事件/取消/失败策略:超时/重试/跳过/延时)、Vitest | 直线/IF/循环/重试用例通过 |
|
|
||||||
| 13:00–16:30 | 浏览器:WebContentsView、CDP 动作移植、拾取器、证据、本地夹具站 | 打开夹具页→点击→输入→取文本 |
|
|
||||||
| 16:30–19:30 | 桌面桥:PowerShell 常驻、键鼠/窗口/剪贴板/对话框/命令、DPI 换算 | 记事本输入 + 窗口激活 |
|
|
||||||
| 20:00–23:30 | 能力批量:控制(14)+ 文件(14);运行面板 v1 | 单节点用例批量通过 |
|
|
||||||
| **门禁 G2** | 一条"浏览器 3 步 + 桌面 2 步"流程可运行,事件流与日志正确 | 不通过则 Day3 砍 PDF/Word |
|
|
||||||
|
|
||||||
### Day 3(10/28):Office/网络 + 联调 + 打包
|
|
||||||
|
|
||||||
| 时间 | 任务 | 产出/门禁 |
|
|
||||||
|---|---|---|
|
|
||||||
| 09:00–11:00 | Excel/CSV(18) | 读写 xlsx、多 sheet、另存 |
|
|
||||||
| 11:00–12:00 | 数据/文本包(28,表驱动) | 字符串/时间/列表/字典/正则/数学/JSON 用例通过 |
|
|
||||||
| 13:00–14:30 | PDF/Word(5)、HTTP/SMTP(4) | PDF 合并/取文本、Word 写入、本地 mock 通过 |
|
|
||||||
| 14:30–15:30 | 图像查找(3,OpenCV) | 模板匹配坐标/置信度达标 |
|
|
||||||
| 15:30–17:30 | 运行面板完善(变量/证据/重试/停止/待接入定位)、E2E 夹具流程、错误态 | 场景 A 跑通 |
|
|
||||||
| 17:30–19:00 | 125%/150% 缩放冒烟、边界修复、性能(200 节点流程) | DPI 用例通过 |
|
|
||||||
| 19:00–21:00 | 打包 portable、干净目录冒烟、自测报告、README/演示脚本 | 双击 exe 可演示 |
|
|
||||||
| 21:00–22:00 | 缓冲/修复/交付整理 | 交付清单齐 |
|
|
||||||
| **门禁 G3** | 干净机器解压→运行→场景 A→失败用例→报告 | 最终验收 |
|
|
||||||
|
|
||||||
### 砍功能顺序(进度落后时)
|
|
||||||
|
|
||||||
1. FTP/数据库(本就不做)
|
|
||||||
2. Word(保留 PDF)
|
|
||||||
3. 邮件收件(保留发信)
|
|
||||||
4. 桌面 UIA(保留坐标键鼠)
|
|
||||||
5. 循环容器(保留 IF)
|
|
||||||
6. 图像/OCR
|
|
||||||
7. 目录组件数(可执行 120 → 80,保浏览器/文件/Excel/数据)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 11. 自测与验收
|
|
||||||
|
|
||||||
`scripts/selftest.mjs` 以 Electron 主进程模式启动,逐项执行并写 `docs/自测报告.md`。
|
|
||||||
|
|
||||||
| # | 用例 | 期望 |
|
|
||||||
|---|---|---|
|
|
||||||
| T01 | 目录加载/分组/搜索/状态 | 1016 项齐全;可执行/待接入计数与注册表一致 |
|
|
||||||
| T02 | 画布增删/连线/容器 | 数据结构与几何正确 |
|
|
||||||
| T03 | 保存/加载/撤销 | 重启一致 |
|
|
||||||
| T04 | 表达式(算术/字符串/时间/成员) | 结果正确,错误定位 |
|
|
||||||
| T05 | 顺序执行 | 节点按序、变量传递 |
|
|
||||||
| T06 | IF 真/假分支 | 只执行对应分支 |
|
|
||||||
| T07 | 计次循环 | 次数/步长正确,可中断 |
|
|
||||||
| T08 | 遍历列表 | item/index 正确 |
|
|
||||||
| T09 | 失败策略:停止/重试/跳过/超时/延时 | 行为与配置一致;校验错误不重试 |
|
|
||||||
| T10 | 取消运行 | 在途动作中止,状态 cancelled |
|
|
||||||
| T11 | 浏览器导航/点击/输入/取文本 | 真实 DOM 变化,可信输入 |
|
|
||||||
| T12 | 元素拾取器 | 生成候选定位并可回填 |
|
|
||||||
| T13 | 浏览器截图证据 | 文件存在、相对路径 |
|
|
||||||
| T14 | 文件读/写/复制/删除/列表 | 临时目录结果正确 |
|
|
||||||
| T15 | Excel 写/读/另存 | 单元格值正确 |
|
|
||||||
| T16 | CSV 读/写 | 引号/换行正确 |
|
|
||||||
| T17 | PDF 页数/文本/合并 | 结果与样本一致 |
|
|
||||||
| T18 | Word 写入/保存 | 文档存在且含文本 |
|
|
||||||
| T19 | 键盘输入记事本 | 文本一致 |
|
|
||||||
| T20 | 鼠标点击/窗口激活 | 目标窗口前台 |
|
|
||||||
| T21 | 剪贴板读写 | 往返一致 |
|
|
||||||
| T22 | 对话框/消息框 | 返回用户选择 |
|
|
||||||
| T23 | HTTP GET/POST | 本地服务响应正确 |
|
|
||||||
| T24 | 邮件发送(mock SMTP) | 服务器收到报文 |
|
|
||||||
| T25 | 图像查找 | 坐标/置信度达标 |
|
|
||||||
| T26 | 125%/150% 缩放 | 画布不裁切、点击不偏 |
|
|
||||||
| T27 | 断网启动 | 正常启动与运行离线流程 |
|
|
||||||
| T28 | 打包产物启动 | 双击可用,无缺失依赖 |
|
|
||||||
| T29 | 待接入组件拖入并运行 | 节点灰标;运行到该节点报 `capability_not_implemented`,含原因/建议/定位 |
|
|
||||||
| T30 | 目录元数据类型/校验 | `vType` 推断输出类型;必填校验生效 |
|
|
||||||
| T31 | 状态筛选/搜索 | 「全部/可执行/待接入」与关键词组合正确 |
|
|
||||||
|
|
||||||
报告格式:用例、能力、前置、步骤、期望、实际、结论、证据路径。**未通过项必须列出,不得隐藏。**
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 12. 打包与交付
|
|
||||||
|
|
||||||
```
|
|
||||||
release/RPA-Studio-0.1.0-win-x64/
|
|
||||||
RPA Studio.exe
|
|
||||||
resources/app.asar
|
|
||||||
resources/catalog.json
|
|
||||||
resources/fixture/...
|
|
||||||
resources/desktop-bridge.ps1
|
|
||||||
examples/*.json
|
|
||||||
README.md
|
|
||||||
version.json
|
|
||||||
RPA-Studio-0.1.0-win-x64.zip
|
|
||||||
SHA256SUMS.txt
|
|
||||||
docs/自测报告.md
|
|
||||||
```
|
|
||||||
|
|
||||||
- 使用 `@electron/packager`(portable 目录),避免 electron-builder 的签名/工具下载风险。
|
|
||||||
- 打包脚本校验:无 devDependencies、无 node_modules 泄漏、无 `.env`/密钥、资源齐全。
|
|
||||||
- 干净目录冒烟:复制到 `D:\tmp\rpa-smoke`,断网启动,跑示例流程。
|
|
||||||
- 版本清单记录源码提交号、构建时间、Electron 版本、可执行组件数。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 13. 风险与应对
|
|
||||||
|
|
||||||
| 风险 | 概率 | 影响 | 应对 |
|
|
||||||
|---|---|---|---|
|
|
||||||
| 三天做不完 | 高 | 交付缩水 | 按砍功能顺序;G1/G2 门禁强制 |
|
|
||||||
| 桌面 DPI/焦点 | 中 | 演示失败 | 桥声明 DPI、UIA 优先、坐标兜底、双缩放冒烟 |
|
|
||||||
| Excel COM 弹窗/慢 | 中 | 卡流程 | 文件级 exceljs;COM 仅可选 |
|
|
||||||
| 浏览器站点兼容 | 中 | 登录/框架页失败 | 只承诺内置浏览器 + 本地夹具;iframe/影子 DOM 显式报错 |
|
|
||||||
| 未注册组件误入 | 高 | 运行中断 | 全量目录 + 灰标 + 预运行「N 个待接入」提示 + 运行期 `capability_not_implemented`(含替代建议) |
|
|
||||||
| npm/Electron 下载 | 低 | 阻塞 | 源已验证;锁版本;Electron 用本地缓存 |
|
|
||||||
| C 盘空间 | 中 | 构建失败 | 全部在 D 盘;缓存指向 D |
|
|
||||||
| 需求蔓延 | 高 | 延期 | 本文件为基线,新需求进 P2+ 清单 |
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 14. 交付清单
|
|
||||||
|
|
||||||
- [ ] `D:\RPA` 源码仓库(git,提交按 G1/G2/G3 打点)
|
|
||||||
- [ ] `release/RPA-Studio-0.1.0-win-x64.zip`(可双击)
|
|
||||||
- [ ] `docs/自测报告.md`(31 用例 + 可执行组件清单)
|
|
||||||
- [ ] `docs/操作手册.md`(10 分钟上手)
|
|
||||||
- [ ] `examples/` 两条示例流程
|
|
||||||
- [ ] 本实施方案 + 现状分析
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 15. 待确认的默认假设(不回复即按此执行)
|
|
||||||
|
|
||||||
1. 产品暂定名 **RPA Studio**,可一处配置改名。
|
|
||||||
2. 组件库展示 **1016 全量**;未接入项灰标、可拖拽、运行时明确失败;可执行目标 120(保底 90)。
|
|
||||||
3. 演示场景默认"本地夹具页:填表→抓表格→写 Excel→按金额分支→通知/发信",后续可换成你的系统。
|
|
||||||
4. OCR 为加分项,不阻塞验收;图像查找优先。
|
|
||||||
5. 运行期离线;构建期允许联网装依赖。
|
|
||||||
6. 两份厂商参考资料只做行为参考;不复制代码/二进制/模型/前端资产;三天不做 `.task` 导入/导出。
|
|
||||||
@ -1,420 +0,0 @@
|
|||||||
# 能力接入规划与排期
|
|
||||||
|
|
||||||
> 生成时间:2026-10-28 · 校准时间:2026-10-30 · 目录 1016 项 · 已接入 732 项(运行时注册 744 个 id;以 src/main/catalog.ts 判定为准)
|
|
||||||
|
|
||||||
## 一、明确不接入清单(永久排除)
|
|
||||||
|
|
||||||
组件库显示「不接入」;运行前检查直接提示"该能力已明确不接入"。
|
|
||||||
|
|
||||||
### 门户专用(3 个)——绑定特定门户系统账号
|
|
||||||
|
|
||||||
- Basic_GetSystemAccount_v2 | 获取平台账号(通用)
|
|
||||||
- Basic_GetSystemAccount | 获取系统账号(门户专用)
|
|
||||||
- Basic_GetSystemAccount_v1 | 获取平台账号
|
|
||||||
|
|
||||||
### 厂商专属云服务(17 个)——依赖厂商专属数据/场景服务
|
|
||||||
|
|
||||||
- DataService_DeleteControllerFiles_v1 | 删除文件
|
|
||||||
- DataService_ReportException | 上报异常信息
|
|
||||||
- DataService_ProductionQueue | 将数据存入队列
|
|
||||||
- DataService_UploadControllerFiles_v1 | 上传文件
|
|
||||||
- DataService_ServiceNew | 数据服务(松塔)
|
|
||||||
- DataService_DeleteControllerFiles | 删除控制器文件
|
|
||||||
- DataService_Service | 数据服务
|
|
||||||
- DataService_DownloadControllerFiles | 下载控制器文件
|
|
||||||
- DataService_DownloadControllerFiles_v1 | 下载文件
|
|
||||||
- DataService_ConsumptionQueue | 从队列获取数据
|
|
||||||
- DataService_UploadControllerFiles | 上传文件至控制器
|
|
||||||
- DataService_AiScreenText | 提取屏幕数据(AI)
|
|
||||||
- DataService_AiScreenDate | 选择日期时间(AI)
|
|
||||||
- DataService_AiScreenOperate | 操作屏幕(AI)
|
|
||||||
- Scenario_CreditVerify | 征信报告解析
|
|
||||||
- Scenario_VatInvoice | 增值税发票验真
|
|
||||||
- Scenario_PhysicalExamination | 体检报告解析
|
|
||||||
|
|
||||||
### SAP 专用(1 个)——依赖SAP客户端
|
|
||||||
|
|
||||||
- Element_DoubleClickSap | 双击SAP表格单元格
|
|
||||||
|
|
||||||
### 硬件外设(41 个)——依赖手机设备连接
|
|
||||||
|
|
||||||
- Mobile_ConnectPhone | 连接手机
|
|
||||||
- Mobile_MobileElementAarray | 获取手机相似元素数组
|
|
||||||
- Mobile_Call | 拨打电话
|
|
||||||
- Mobile_MobileSetText_V1 | 手机键盘输入
|
|
||||||
- Mobile_ClickButton | 系统导航键
|
|
||||||
- Mobile_SetInputV1 | 手机输入文本
|
|
||||||
- Mobile_ConnectPhoneV3 | 连接手机
|
|
||||||
- Mobile_GetAttribute | 手机元素属性拾取
|
|
||||||
- Mobile_Exists | 手机元素判断
|
|
||||||
- Mobile_MobileElementList | 获取手机相似元素列表
|
|
||||||
- Mobile_FileTransfer | 手机文件传输
|
|
||||||
- Mobile_GetPhoneInfomation | 获取手机连接详情
|
|
||||||
- Mobile_ClickLocation | 手机屏幕坐标点击
|
|
||||||
- Mobile_SaveFilePhone | 保存文件到手机
|
|
||||||
- Mobile_SetInput | 设置文本|填写输入框
|
|
||||||
- Mobile_MobileCvClick | 手机智能元素点击
|
|
||||||
- Mobile_TouchSlideV1 | 触摸滑动
|
|
||||||
- Mobile_GetPhoneFiles | 从手机中获取文件
|
|
||||||
- Mobile_OpenMobileApp | 打开手机APP
|
|
||||||
- Mobile_TouchSlide | 触摸滑动
|
|
||||||
- Mobile_GetMobileUiTree | 获取手机UI树
|
|
||||||
- Mobile_GetScreenOrientation | 获取手机屏幕方向
|
|
||||||
- Mobile_GetElement | 获取手机元素对象
|
|
||||||
- Mobile_MobileCvSetText | 智能设置文本|填写输入框
|
|
||||||
- Mobile_ClosePhone | 关闭连接手机
|
|
||||||
- Mobile_SetInputV2 | 手机输入文本
|
|
||||||
- Mobile_SetScreenOrientation | 设置手机屏幕方向
|
|
||||||
- Mobile_ImageClick | 点击手机图像
|
|
||||||
- Mobile_CloseMobileApp | 关闭手机APP
|
|
||||||
- Mobile_ElementShot | 手机元素截图
|
|
||||||
- Mobile_ScreenShot | 手机截屏
|
|
||||||
- Mobile_GetClipboardText | 获取手机剪切板文本
|
|
||||||
- Mobile_ConnectPhoneV2 | 连接手机
|
|
||||||
- Mobile_GetText | 手机元素文本拾取
|
|
||||||
- Mobile_MobileSetText | 手机键盘输入
|
|
||||||
- Mobile_MobileElementList_V1 | 获取手机相似元素列表
|
|
||||||
- Mobile_ElementClick | 手机元素点击
|
|
||||||
- Mobile_GetChildren | 获取手机子元素
|
|
||||||
- Mobile_SetClipboardText | 设置手机剪切板文本
|
|
||||||
- Mobile_ConnectPhoneV1 | 连接手机
|
|
||||||
- System_GetMobileMessage | 获取手机短信
|
|
||||||
|
|
||||||
### 厂商专属云服务(30 个)——依赖厂商训练模型/打码服务
|
|
||||||
|
|
||||||
- OCR_RotateVerificationCodeV1 | 旋转图片验证码
|
|
||||||
- OCR_PuzzleVerificationCode_v1 | 推理拼图验证码
|
|
||||||
- OCR_ClickVerificationCodeV2 | 点选验证码
|
|
||||||
- OCR_SourceCommonEnglishNumbersCodeV2 | 普通验证码
|
|
||||||
- OCR_SimpleSliderVerificationCode | 滑块验证码(滑动条式)
|
|
||||||
- OCR_SourceMergeCardOCR | 识别卡证文字
|
|
||||||
- OCR_SliderVerificationCodeV5 | 滑动拼图验证码
|
|
||||||
- OCR_SliderVerificationCodeV4 | 滑动拼图验证码
|
|
||||||
- OCR_SourceMergeOtherOCR | 识别其他场景文字
|
|
||||||
- OCR_SliderCode | 滑块验证码
|
|
||||||
- OCR_SourceMergeGeneralOCR | 通用文字识别
|
|
||||||
- OCR_RotateVerificationCodeV2 | 旋转图片验证码
|
|
||||||
- OCR_RotateVerificationCode | 滑块验证码(旋转式)
|
|
||||||
- OCR_ClickVerificationCodeV1 | 点选验证码
|
|
||||||
- OCR_PuzzleVerificationCode | 推理拼图验证码
|
|
||||||
- OCR_SimpleSliderVerificationCodeV2 | 滑动条验证码
|
|
||||||
- OCR_SourceCommonEnglishNumbersCode | 普通验证码
|
|
||||||
- OCR_SourceMergeCarOCR | 识别汽车场景文字
|
|
||||||
- OCR_SliderVerificationCodeV2 | 滑块验证码(拼图样式)
|
|
||||||
- OCR_SliderVerificationCodeV1 | 滑块验证码(拼图样式)
|
|
||||||
- OCR_SliderVerificationCode | 滑块验证码(拼图样式)
|
|
||||||
- OCR_SourceMergeEducationOCR | 识别教育场景文字
|
|
||||||
- OCR_SliderVerificationCodeV3 | 滑动拼图验证码
|
|
||||||
- OCR_ClickVerificationCode | 点选验证码
|
|
||||||
- OCR_SourceCommonEnglishNumbersCodeV1 | 普通验证码
|
|
||||||
- OCR_PuzzleVerificationCode_v2 | 推理拼图验证码
|
|
||||||
- OCR_SourceMergePaperOCR | 识别票据文字
|
|
||||||
- OCR_SimpleSliderVerificationCodeV1 | 滑动条验证码
|
|
||||||
- OCR_SourceMergeGeneralOCR_V1 | 识别通用文字
|
|
||||||
- OCR_SliderVerificationCodePosition | 获取拼图滑块位置
|
|
||||||
|
|
||||||
### 厂商专属云服务(4 个)——依赖厂商离线OCR模型文件
|
|
||||||
|
|
||||||
- OcrOffline_ClickOcrText_V1 | 鼠标点击OCR文本
|
|
||||||
- OcrOffline_MoveToOcrText | 鼠标移动到OCR文本上
|
|
||||||
- OcrOffline_ClickOcrText | 鼠标点击OCR文本
|
|
||||||
- OcrOffline_MoveToOcrText_V1 | 鼠标移动到OCR文本上
|
|
||||||
|
|
||||||
### 厂商专属云服务(6 个)——依赖百度云OCR账号与密钥
|
|
||||||
|
|
||||||
- OCR_BaiDu_CarOCR | 汽车场景类OCR
|
|
||||||
- OCR_BaiDu_OtherOCR | 其他场景类OCR
|
|
||||||
- OCR_BaiDu_EducationOCR | 教育场景类OCR
|
|
||||||
- OCR_BaiDu_GeneralOCR | 通用OCR
|
|
||||||
- OCR_BaiDu_CardOCR | 卡片类OCR
|
|
||||||
- OCR_BaiDu_PaperOCR | 票据类OCR
|
|
||||||
|
|
||||||
### 硬件外设(6 个)——依赖USB通道外设
|
|
||||||
|
|
||||||
- peripherals_UsbCom | 开/关USB通道
|
|
||||||
- peripherals_CloseNetworkUsbCom | 关闭NetworkUSB通道
|
|
||||||
- peripherals_CloseUsbCom | 关闭USB通道
|
|
||||||
- peripherals_NetworkUsbCom | 开/关NetworkUSB通道
|
|
||||||
- peripherals_OpenNetworkUsbCom | 开启NetworkUSB通道
|
|
||||||
- peripherals_OpenUsbCom | 开启USB通道
|
|
||||||
|
|
||||||
### 硬件外设(1 个)——依赖银行安全控件键盘
|
|
||||||
|
|
||||||
- SafeControlInput_KeyInput | 安全控件文本输入
|
|
||||||
|
|
||||||
### 硬件外设(4 个)——依赖机械臂设备
|
|
||||||
|
|
||||||
- System_ResetMechanicalArm | 复位机械臂
|
|
||||||
- System_MechanicalArmClick | 机械臂点击
|
|
||||||
- System_ConnectingMechanicalArm | 连接机械臂
|
|
||||||
- System_CloseMechanicalArm | 断开机械臂
|
|
||||||
|
|
||||||
### 硬件外设(2 个)——依赖U盾设备
|
|
||||||
|
|
||||||
- System_ConnectingUshield | 连接U盾
|
|
||||||
- System_CloseUshield | 断开U盾
|
|
||||||
|
|
||||||
### 语义已被覆盖(1 个)——下载能力已覆盖下载场景
|
|
||||||
|
|
||||||
- WebBrowser_DownloadDialog | 处理下载对话框
|
|
||||||
|
|
||||||
**小计:116 个 id**(2026-10-30 校准:原 128 项中 12 项已接入,移至下方清单)
|
|
||||||
|
|
||||||
### 已接入(原排除项,2026-10-30 校准移出)
|
|
||||||
|
|
||||||
通过设置页配置的 OpenAI 兼容 AI 服务接入(`src/main/capabilities/ai-caps.ts`):
|
|
||||||
|
|
||||||
- DataService_AIGenTxt | AI生成文本
|
|
||||||
- DataService_LargeModel | 大模型
|
|
||||||
- DataService_ZBrain | 自定义AI模型
|
|
||||||
- DataService_AgentCall | 调用智能体
|
|
||||||
- DataService_ChatBot | chatbot对话机器人
|
|
||||||
- DataService_KeyWord | 关键词抽取
|
|
||||||
- DataService_Correct | 文本纠错
|
|
||||||
- DataService_Review | 文本审核
|
|
||||||
- Scenario_TextComparison | 文档比对
|
|
||||||
|
|
||||||
通过本地离线 OCR 模型接入(`src/main/capabilities/ocr-caps.ts`):
|
|
||||||
|
|
||||||
- OcrOffline_GetOcrTextNew | 图像OCR识别
|
|
||||||
- OcrOffline_GetOcrTable | 获取表格数据(OCR)
|
|
||||||
- OcrOffline_GetOcrTextPosition | 图像OCR识别(含坐标)
|
|
||||||
|
|
||||||
## 二、暂缓池
|
|
||||||
|
|
||||||
### 已完成(超出原排期,2026-10-30 校准)
|
|
||||||
|
|
||||||
- [P1] 人机交互对话框 17/17:输入/文件/消息/自定义表单对话框(`src/main/capabilities/human-dialog.ts`)
|
|
||||||
- [P2] Windows 系统 12/12:凭据/远程桌面/屏保/输入法(`src/main/capabilities/windows-system.ts`)
|
|
||||||
- [P2] 图表数据读取 2/2:Element_GetLineBarChartData、Element_GetPieChartData(`src/main/capabilities/web-batch2.ts`)
|
|
||||||
- [P2] Excel 高阶 14/38:单元格格式、区域选择、行列显隐、尾部空白单元格、列标识、表格样式、Load/Close(`src/main/capabilities/excel-common.ts`)
|
|
||||||
- [P3] 待评估池移出 20 项:FTP 10 项(`src/main/capabilities/ftp.ts`)、NLP 9 项(AI 服务 `src/main/capabilities/ai-caps.ts`)、File_OpenFile
|
|
||||||
- 原排除清单移出 12 项:见第一节「已接入」
|
|
||||||
|
|
||||||
以下为剩余暂缓项:
|
|
||||||
|
|
||||||
### [P3] 待评估(84 个)
|
|
||||||
|
|
||||||
- child_modules_v1 | 调用子流程
|
|
||||||
- concurrent_child_process_v1 | 并发调用子流程
|
|
||||||
- Basic_Dodecryption | 设置密文变量
|
|
||||||
- Database_BatchSQLExecute | 批量执行SQL语句
|
|
||||||
- Database_DBConnect | 连接数据库
|
|
||||||
- Database_SingleSQLQuery | 执行单SQL查询
|
|
||||||
- Database_BatchInsertExecute | SQL批量插入数据
|
|
||||||
- Database_AllSQLQuery_V1 | 执行全SQL查询
|
|
||||||
- Database_CloseDBConnect | 关闭连接
|
|
||||||
- Database_SQLExecute | 执行SQL语句
|
|
||||||
- Database_AllSQLQuery | 执行全SQL查询
|
|
||||||
- Element_DesktopChildren | 获取子元素
|
|
||||||
- File_UploadBusinessResults_v1 | 上报办理结果
|
|
||||||
- File_StartBusiness | 上报办理业务
|
|
||||||
- File_UploadBusinessResults | 上报办理结果(门户专用)
|
|
||||||
- File_Collectupload | 上报采集数据
|
|
||||||
- File_UploadFolder | 结果打包上传
|
|
||||||
- FileTypeConversion_Office2pdf | OFFICE文档转PDF
|
|
||||||
- FileTypeConversion_Pdf2image2 | PDF转图片
|
|
||||||
- GoogleTable_GetAllData | 获取列信息(Google Sheets)
|
|
||||||
- GoogleTable_GetAllSheetsName | 获取Sheet名(Google Sheets)
|
|
||||||
- GoogleTable_WriteToSheetRegionColumns | 写入列(Google Sheets)
|
|
||||||
- GoogleTable_WriteToSheetRegionRows | 写入行(Google Sheets)
|
|
||||||
- GoogleTable_WriteToSheetCell | 写入单元格(Google Sheets)
|
|
||||||
- GoogleTable_WriteToSheetCellFormula | 写入单元格公式(Google Sheets)
|
|
||||||
- GoogleTable_GetSheetRowCount | 获取行数(Google Sheets)
|
|
||||||
- GoogleTable_DownloadSheets | 导出Sheet(Google Sheets)
|
|
||||||
- GoogleTable_CopyPasteSheet | 复制粘贴(Google Sheets)
|
|
||||||
- GoogleTable_GetColumnsByTableTitle | 按表头读取列(Google Sheets)
|
|
||||||
- GoogleTable_ChangeSheet | 激活Sheet(Google Sheets)
|
|
||||||
- GoogleTable_RenameSheet | 重命名Sheet(Google Sheets)
|
|
||||||
- GoogleTable_WriteToSheetRegionregion | 写入区域(Google Sheets)
|
|
||||||
- Image_Move_V2 | 鼠标移动到图像上
|
|
||||||
- Image_Base64Decode | 图片base64编码转二进制
|
|
||||||
- Image_Base64Encode | 图片转base64编码
|
|
||||||
- Image_Wait_V1 | 等待图像
|
|
||||||
- Image_Exists | 判断图像
|
|
||||||
- Image_Location | 图像位置
|
|
||||||
- Image_Location_V1 | 获取图像位置
|
|
||||||
- Image_Exists_V1 | 判断图像
|
|
||||||
- Image_CropImg | 截取图像
|
|
||||||
- Image_RelativePosition | 对比图像相对位置
|
|
||||||
- Image_Wait | 等待图像
|
|
||||||
- Image_Move | 移动到图像
|
|
||||||
- Image_Move_V1 | 移动到图像
|
|
||||||
- Keyboard_EnableManualOperation | 开启模拟人工操作
|
|
||||||
- Keyboard_DisableManualOperation | 关闭模拟人工操作
|
|
||||||
- Mark_Mark | 图片打标
|
|
||||||
- NLP_XunFei_MachineTranslation | 机器翻译
|
|
||||||
- NLP_XunFei_FeelingAnalysis | 第三方情感分析模型
|
|
||||||
- NLP_XunFei_KeywordExtraction | 关键字提取
|
|
||||||
- OCR_MergeCardOCR | 卡证文字识别
|
|
||||||
- OCR_CalculationCode | 算术验证码
|
|
||||||
- OCR_IdCardV2 | OCR身份证
|
|
||||||
- OCR_BusinessCardV2 | OCR名片
|
|
||||||
- OCR_VehicleInvoiceV2 | OCR机动车发票
|
|
||||||
- OCR_MergeOtherOCR | 其他场景文字识别
|
|
||||||
- OCR_BusinessLicenseV3 | OCR营业执照(自研)
|
|
||||||
- OCR_PassportV2 | OCR护照
|
|
||||||
- OCR_MergeGeneralOCR | 通用文字识别
|
|
||||||
- OCR_KeyValueIdentification | 识别键值对结构
|
|
||||||
- OCR_AmountIdentification | 识别价格金额
|
|
||||||
- OCR_GeneralTextV2 | OCR高精度文本
|
|
||||||
- OCR_VatInvoiceV2 | OCR增值税发票
|
|
||||||
- OCR_DrivingLicenseV2 | OCR行驶证
|
|
||||||
- OCR_AccountPageV2 | OCR户口页
|
|
||||||
- OCR_PlateLicenseV2 | OCR车牌
|
|
||||||
- OCR_BankCardV2 | OCR银行卡
|
|
||||||
- OCR_BankCardV3 | OCR银行卡(自研)
|
|
||||||
- OCR_IdCardV3 | OCR身份证(自研)
|
|
||||||
- OCR_VatInvoiceV3 | OCR增值税发票(自研)
|
|
||||||
- OCR_MergePaperOCR | 票据文字识别
|
|
||||||
- OCR_DriverLicenseV2 | OCR驾驶证
|
|
||||||
- OCR_RealEstateCertV2 | OCR不动产权证
|
|
||||||
- OCR_TrainTicketV2 | OCR火车票
|
|
||||||
- OCR_TaxiInvoiceV2 | OCR出租车发票
|
|
||||||
- OCR_OfficialSealV2 | OCR公章
|
|
||||||
- OCR_MergeCarOCR | 汽车场景文字识别
|
|
||||||
- OCR_MergeEducationOCR | 教育场景文字识别
|
|
||||||
- OCR_HouseCertV2 | OCR房产证
|
|
||||||
- OCR_CommonEnglishNumbersCode | 普通验证码
|
|
||||||
- OCR_BusinessLicenseV2 | OCR营业执照
|
|
||||||
- OCR_VinV2 | OCR车辆vin码
|
|
||||||
- Element_exception_monitor_v2 | 异常监控
|
|
||||||
|
|
||||||
### [P2] 视觉定位与 OCR 读屏(5 个)
|
|
||||||
|
|
||||||
- logic_control_if_ai_vision | 判断屏幕图像(AI)
|
|
||||||
- CV_GetText | 获取文本
|
|
||||||
- CV_SetText | 文本输入
|
|
||||||
- CV_MouseHover | 移动到元素上
|
|
||||||
- CV_MouseClick | 元素点击
|
|
||||||
|
|
||||||
### [P3] 桌面 Ground 剩余变体(12 个)
|
|
||||||
|
|
||||||
- Element_GetCheckGround | 获取勾选框状态值
|
|
||||||
- Element_SetCheckGround | 设置勾选框状态
|
|
||||||
- Element_RegionGround | 获取元素坐标
|
|
||||||
- Element_GetSelectGround | 获取下拉框选项值
|
|
||||||
- Element_MouseToGround | 鼠标移动到元素上
|
|
||||||
- Element_SetAttributeGround | 设置网页元素属性
|
|
||||||
- Element_ChildrenGround | 获取子元素
|
|
||||||
- Element_GetAttributeGround | 获取网页元素属性
|
|
||||||
- Element_ScrollIntoViewGround | 滚动页面
|
|
||||||
- Element_SetSelectGround | 设置下拉框选择项
|
|
||||||
- Element_ShotGround | 元素截图
|
|
||||||
- Element_GetButtonEnableGround | 元素是否可点击
|
|
||||||
|
|
||||||
### [P2] Excel 高阶:格式/图片/加密/透视表/筛选等(24 个)
|
|
||||||
|
|
||||||
- Excel_Cs_Group | 组合行列
|
|
||||||
- Excel_Cs_DeletePicture | 删除图片
|
|
||||||
- Excel_Cs_RefreshPivotTable | 刷新Excel数据透视表
|
|
||||||
- Excel_Cs_DeletePictureV2 | 删除图片
|
|
||||||
- Excel_Cs_Encrypt | 加密Excel
|
|
||||||
- Excel_Cs_Move | 移动Sheet
|
|
||||||
- Excel_Cs_InsertPicture | 插入图片
|
|
||||||
- Excel_Cs_Merge | 合并Sheet
|
|
||||||
- Excel_InsertImage | 插入图片
|
|
||||||
- Excel_Cs_InsertPivotView | 新建透视图
|
|
||||||
- Excel_Cs_Interop | Excel动态调用
|
|
||||||
- Excel_Cs_InsertPivotTable | 新建Excel数据透视表
|
|
||||||
- Excel_Cs_RunMacro | 执行宏
|
|
||||||
- Excel_Loop | excel内容循环
|
|
||||||
- Excel_Cs_Export | 导出Excel为PDF
|
|
||||||
- Excel_Cs_ExcelAddin | 数据处理
|
|
||||||
- Excel_Cs_InsertPictureV2 | 插入图片
|
|
||||||
- Excel_Cs_FilterPivotTable | 筛选Excel数据透视表
|
|
||||||
- Excel_Cs_RunMacroV2 | 执行宏
|
|
||||||
- Excel_Cs_SetSlicer | Excel切片器
|
|
||||||
- Excel_ExecuteMacro | 执行宏
|
|
||||||
- Excel_Cs_RefreshAll | 刷新Excel数据
|
|
||||||
- Excel_DeleteImage | 删除图片
|
|
||||||
- Excel_ExportRangeAsPicture | 导出区域为图片
|
|
||||||
|
|
||||||
### [P2] 邮件收件:需 IMAP 库(28 个)
|
|
||||||
|
|
||||||
- Mail_UnmarkSeen | 标为未读
|
|
||||||
- Mail_DoReplyMail_v1 | 回复邮件(Outlook)
|
|
||||||
- Mail_RecvConnect | 连接邮箱
|
|
||||||
- Mail_RecvMailGet | 取第几封邮件
|
|
||||||
- Mail_ExOperateMail | 操作邮件
|
|
||||||
- Mail_RecvGetList | 邮件筛选
|
|
||||||
- Mail_ReplyMail_v1 | 回复邮件
|
|
||||||
- Mail_FindMail_v1 | 查找邮件
|
|
||||||
- Mail_DoMoveMail | 移动邮件
|
|
||||||
- Mail_UnionConnect | 连接邮箱
|
|
||||||
- Mail_ExConnect | 连接Exchange邮箱
|
|
||||||
- Mail_RecvClose | 关闭连接
|
|
||||||
- Mail_DoSendMail | 发送邮件
|
|
||||||
- Mail_RecvCount | 取收件箱邮件数
|
|
||||||
- Mail_DoGetMailMessage_v2 | 获取邮件(Outlook)
|
|
||||||
- Mail_DoGetMailMessage_v1 | 查找邮件(Outlook)
|
|
||||||
- Mail_ReplyMail | 回复邮件
|
|
||||||
- Mail_FindMail_v2 | 获取邮件
|
|
||||||
- Mail_ExSearchMail | 查找邮件
|
|
||||||
- Mail_DoGetMailMessage | 查看邮件
|
|
||||||
- Mail_ExSendMail | 发送邮件
|
|
||||||
- Mail_OperateMail | 操作邮件
|
|
||||||
- Mail_MarkSeen | 标为已读
|
|
||||||
- Mail_DoMoveMail_v1 | 操作邮件(Outlook)
|
|
||||||
- Mail_ReplyMail_v2 | 回复邮件
|
|
||||||
- Mail_DoReplyMail | 回复邮件
|
|
||||||
- Mail_DoSendMail_v1 | 发送邮件(Outlook)
|
|
||||||
- Mail_FindMail | 查找邮件
|
|
||||||
|
|
||||||
### [P2] PDF 高阶:需引入渲染能力(4 个)
|
|
||||||
|
|
||||||
- PDF_PdfEncryption | 加密 PDF 文档
|
|
||||||
- PDF_GetPdfImg | 提取 PDF 图片
|
|
||||||
- PDF_GetPdfTable | 读取 PDF 表格
|
|
||||||
- PDF_PdfToImg | PDF 转图片
|
|
||||||
|
|
||||||
### [P2] Word 选区与光标:COM 操作(11 个)
|
|
||||||
|
|
||||||
- Word_ReadSelect | 读取选中文本
|
|
||||||
- Word_CutSelect | 剪切
|
|
||||||
- Word_ReadTable | 读取Word表格
|
|
||||||
- Word_SelectRow | 选择行
|
|
||||||
- Word_DeleteSelect | 删除
|
|
||||||
- Word_SetPosition | 设置光标位置
|
|
||||||
- Word_PositionToSelect | 查找文本后设置光标位置
|
|
||||||
- Word_MovePosition | 移动光标位置
|
|
||||||
- Word_CopySelect | 复制
|
|
||||||
- Word_PasteSelect | 粘贴
|
|
||||||
- Word_SelectAll | 全选
|
|
||||||
|
|
||||||
## 三、接入排期总览
|
|
||||||
|
|
||||||
| 阶段 | 内容 | 状态 |
|
|
||||||
|---|---|---|
|
|
||||||
| P0 | 网页核心 170、桌面 Ground 核心 5、网页批二 15 id、网页批三 16 id | 已完成 2026-10-28 |
|
|
||||||
| P1 | 人机交互对话框 17、Time 家族 24 | 已完成 2026-10-30 校准 |
|
|
||||||
| P1 | 文件上传验证 | 下一迭代 |
|
|
||||||
| P2 | Excel 高阶 24、PDF 高阶 4、Word 选区 11、邮件收件 28、视觉 5 | 后续迭代(图表 2、Windows 系统 12 已完成) |
|
|
||||||
| P3 | 桌面 Ground 剩余 12、待评估 84 | 按需 |
|
|
||||||
| 永久排除 | 明确不接入清单 116 项(第一节) | 不排期 |
|
|
||||||
|
|
||||||
## 四、落地机制
|
|
||||||
|
|
||||||
- 本文档为能力接入的唯一规划来源;新增能力须先在此登记阶段
|
|
||||||
- 2026-10-30 校准:数字以运行时注册表为准(744 个 id,目录可执行 732 / 1016);原排除/暂缓中已实现的 78 项已移入「已接入 / 已完成」
|
|
||||||
- 下一步:将第一节清单固化为 resources/excluded.json,组件库显示「不接入」灰标
|
|
||||||
|
|
||||||
## 六、桌面拾取与定位——原厂实现解读与我们的增强项(2026-10-30)
|
|
||||||
|
|
||||||
### 原厂实在智能的实现(源码解读)
|
|
||||||
|
|
||||||
- 拾取(ui_selector_window.py):全屏半透明覆盖窗 + 高亮窗贴**窗格级**元素 + 左键确认;输出 uiSelector = { window: [{class,type,name} 链], shot: 自动截图 }——**拾取时自动截取目标窗口图片存档**
|
|
||||||
- 执行定位(Element/py/scope/native.py):路径链逐层走 UIA 树——每层匹配 type/class/name **通配符** + 同名兄弟 **index 序号**;窗口锚用标题通配(EnumWindows)
|
|
||||||
- 桌面点击执行:**窗口定位 + 图像/坐标模式**(原厂 Image/CV 能力家族庞大的原因)
|
|
||||||
|
|
||||||
### 我们当前方案对比
|
|
||||||
|
|
||||||
- 定位:UIA 属性选择器(name/id 精确匹配,单层)——精度高于原厂通配符,但**缺路径链/通配/序号 fallback**
|
|
||||||
- 拾取:v2 两阶段(悬停/微调)+ PageUp/PageDown 层级切换 + 最小化应用——已实现
|
|
||||||
|
|
||||||
### 登记的增强项(后续迭代)
|
|
||||||
|
|
||||||
1. 链式查找:拾取器已输出元素链(layers),执行端支持逐层匹配(原厂模式)——解决 name 重复场景
|
|
||||||
2. name 通配符匹配与同名序号 index
|
|
||||||
3. 拾取确认时自动截取目标窗口图片存档(调试回看)
|
|
||||||
4. WebView2 深层 DOM 控件的 UIA 暴露深度调优
|
|
||||||
@ -1,186 +0,0 @@
|
|||||||
# 04-外接浏览器支持改动方案(Edge / Chrome)
|
|
||||||
|
|
||||||
- **编写时间**:2026-10-30
|
|
||||||
- **编写人**:刘爱家
|
|
||||||
- **关联文档**:01-现状分析与可行性 / 02-实施方案 / 03-能力接入规划与排期
|
|
||||||
- **目标**:让「打开网页」及全部网页自动化组件支持在客户本机的 Edge / Chrome 浏览器中执行,不再强制使用内置运行浏览器;内置浏览器保留为默认兜底。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 一、现状分析(代码级)
|
|
||||||
|
|
||||||
### 1.1 现有架构
|
|
||||||
|
|
||||||
```
|
|
||||||
网页组件(web.ts / web-batch2.ts / web-crawl.ts,169 项目录项)
|
|
||||||
│ BrowserLike 接口(engine/types.ts,33 个方法)
|
|
||||||
▼
|
|
||||||
BrowserSession(browser/session.ts,1056 行,单例)
|
|
||||||
│ this.cdp(method, params)
|
|
||||||
▼
|
|
||||||
win.webContents.debugger(Electron 内置窗口「RPA 运行浏览器」)
|
|
||||||
```
|
|
||||||
|
|
||||||
- **会话模型**:`BrowserSession.create()` 单例复用一个 `BrowserWindow`,全局变量 `__rpaBrowserWindow` 持有;单窗口单 Tab 模型(`tab_unavailable` 错误即来自此)。
|
|
||||||
- **CDP 使用现状**:全部经 `webContents.debugger`(协议版本 1.3),使用的域:
|
|
||||||
- `DOM.getDocument / DOM.querySelector / DOM.setFileInputFiles`
|
|
||||||
- `Input.dispatchMouseEvent / Input.dispatchKeyEvent / Input.insertText`
|
|
||||||
- `Runtime.evaluate`(仅 runJs 一处)
|
|
||||||
- `Page.enable / Page.captureScreenshot / Page.handleJavaScriptDialog`
|
|
||||||
- `Network.enable / Network.getResponseBody`(网页监听)
|
|
||||||
- **全部是标准 CDP 域,Edge / Chrome(同为 Chromium)原生支持,命令层零改动。**
|
|
||||||
- **目录字段已预留**:`WebBrowser_Create / V2 / V3` 均有 `type`(浏览器类型,默认 chrome)、`isRemote`(是否远程)、`ipAddress`、`port`(默认 61619)、`arguments`(携带参数)、`openNewTab` 字段——**但实现不读取**,无论填什么一律打开内置窗口。
|
|
||||||
|
|
||||||
### 1.2 关键约束
|
|
||||||
|
|
||||||
| 约束 | 说明 |
|
|
||||||
|---|---|
|
|
||||||
| Electron 31 主进程 = Node 20.x | 无稳定原生 WebSocket 客户端,连接外部调试端口需引入 `ws` |
|
|
||||||
| 用户已开着的浏览器 | 未带调试端口启动的实例无法连接(Chromium 安全限制),必须带参重启 |
|
|
||||||
| 内置窗口独有 API | `webContents.debugger` 仅 Electron 可用;`Page.captureScreenshot` 外部浏览器同样可用(证据截图无障碍) |
|
|
||||||
| 元素拾取器 | 现拾取链路绑定内置窗口,外部浏览器需改走 CDP DOM 快照 |
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 二、总体设计
|
|
||||||
|
|
||||||
**核心思路:传输层抽象,一次替换,上层不动。**
|
|
||||||
|
|
||||||
```
|
|
||||||
BrowserLike(接口不变)
|
|
||||||
▼
|
|
||||||
BrowserSession(cdp() 委托给 transport,其余逻辑不变)
|
|
||||||
▼
|
|
||||||
CdpTransport 接口(新增)
|
|
||||||
├── EmbeddedTransport —— 现有 webContents.debugger 封装(内置浏览器,默认)
|
|
||||||
└── ExternalTransport —— WebSocket 直连外部浏览器调试端口(新增,依赖 ws)
|
|
||||||
```
|
|
||||||
|
|
||||||
- `BrowserLike` 接口 33 个方法、上层 169 个网页组件**全部零改动**。
|
|
||||||
- 内置浏览器保留:`type` 未填或填内置类型时走原路径,行为与现在完全一致(回归风险隔离)。
|
|
||||||
|
|
||||||
### 浏览器类型枚举约定
|
|
||||||
|
|
||||||
| type 值 | 行为 |
|
|
||||||
|---|---|
|
|
||||||
| `chrome` | 探测本机 Chrome 路径 → 带调试端口启动 → CDP 接管 |
|
|
||||||
| `edge` | 探测本机 Edge 路径 → 同上 |
|
|
||||||
| `chrome_remote` / `edge_remote`(或 isRemote=1) | 不启动进程,直接连 `ipAddress:port` 上已监听调试端口的实例 |
|
|
||||||
| `zexplorer` / 缺省 | 内置运行浏览器(现状行为,兜底) |
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 三、分阶段实施
|
|
||||||
|
|
||||||
### 阶段一:传输层抽象 + 外部浏览器连接(核心,约 2~3 天)
|
|
||||||
|
|
||||||
**新增 `src/main/browser/transport.ts`**
|
|
||||||
|
|
||||||
```ts
|
|
||||||
export interface CdpTransport {
|
|
||||||
send<T>(method: string, params?: Record<string, unknown>): Promise<T>;
|
|
||||||
on(event: string, handler: (params: unknown) => void): void; // Page.loadEventFired 等
|
|
||||||
close(): Promise<void>;
|
|
||||||
kind: "embedded" | "external";
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
- `EmbeddedTransport`:包装现有 `webContents.debugger` 的 `send/on`(`session.ts` 现有 `cdp()` 逻辑原样迁入)。
|
|
||||||
- `ExternalTransport`:基于 `ws` 包,连接 `http://ip:port` 的调试端点:
|
|
||||||
1. `GET /json/list` 枚举页面级 target,取 `webSocketDebuggerUrl`(或 `PUT /json/new?url=` 开新 Tab)
|
|
||||||
2. 建立WebSocket,消息按 CDP 格式封包(`{id, method, params}` / 响应 `{id, result|error}` / 事件 `{method, params}`),promise map 按 id 配对
|
|
||||||
3. 断线检测(ws close 事件)→ 抛 `browser_disconnected`,带「请带调试端口重启浏览器」提示
|
|
||||||
|
|
||||||
**新增 `src/main/browser/external.ts`(外部浏览器进程管理)**
|
|
||||||
|
|
||||||
- 路径探测(按优先级):
|
|
||||||
- Chrome:注册表 `HKLM\SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\chrome.exe` → `C:\Program Files\Google\Chrome\Application\chrome.exe` → `C:\Program Files (x86)\...`
|
|
||||||
- Edge:`HKLM\...\App Paths\msedge.exe` → `C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe`(Win10/11 必装,探测成功率 100%)
|
|
||||||
- 启动:`child_process.spawn(browserPath, [--remote-debugging-port=<port>, --user-data-dir=<独立目录>, url, ...args])`
|
|
||||||
- **必须带独立 `--user-data-dir`**:否则端口参数会被用户已开的实例吞掉(连不上且无报错的头号坑)
|
|
||||||
- 端口管理:默认取 `port` 字段(61619),被占用时自增重试;启动后轮询 `GET /json/version`(超时 15s)确认就绪
|
|
||||||
- 进程生命周期:记录 child process 引用;流程 `WebBrowser_Close` 时可选关闭(目录可加「执行后是否关闭浏览器」字段)
|
|
||||||
|
|
||||||
**改造 `src/main/browser/session.ts`**
|
|
||||||
|
|
||||||
- `BrowserSession` 构造函数改为接收 `CdpTransport`;`cdp()` 方法改为一行委托 `this.transport.send()`
|
|
||||||
- 单例键从 `__rpaBrowserWindow` 扩展为「按 (type, user-data/port) 维度」的会话缓存,支持同流程混合使用内置 + 外部
|
|
||||||
- `close()`:embedded 关窗口(现状逻辑);external 断开 ws,按配置决定是否杀浏览器进程
|
|
||||||
|
|
||||||
### 阶段二:「打开网页」组件参数生效(约 1 天)
|
|
||||||
|
|
||||||
**改造 `src/main/capabilities/browser.ts` 的 `WebBrowser_Create / V2 / V3 / CreateBackground`:**
|
|
||||||
|
|
||||||
```
|
|
||||||
type + isRemote + ipAddress + port + arguments ──→ BrowserSessionFactory
|
|
||||||
├─ type ∈ {chrome, edge} 且 isRemote=0 → external.ts 启动并连接
|
|
||||||
├─ isRemote=1 → 直连 ipAddress:port
|
|
||||||
└─ 其余 → 内置浏览器(现状)
|
|
||||||
```
|
|
||||||
|
|
||||||
- `waitingload`(等待页面加载完成):external 模式下监听 `Page.loadEventFired` 实现,语义与内置一致
|
|
||||||
- `arguments`(携带参数):透传给浏览器启动命令行
|
|
||||||
- 日志明确输出实际使用的浏览器(如「已使用 Edge (127.0.0.1:9222) 打开 https://...」),便于客户排查
|
|
||||||
|
|
||||||
### 阶段三:多标签页管理(约 2~3 天)
|
|
||||||
|
|
||||||
现状单 Tab 模型在外部浏览器不成立,需引入 `Target` 域:
|
|
||||||
|
|
||||||
- `Target.getTargets` / `Target.attachToTarget`(flatten 模式)/ `Target.closeTarget`
|
|
||||||
- 落地组件:`WebBrowser_GetTAB_V1`(按标题/URL 匹配 Tab,type 字段终于生效)、`WebBrowser_Switch`(切换当前操作的 Tab)、`CreateV3` 的 `openNewTab`
|
|
||||||
- 实现方式:ExternalTransport 按 targetId 维护多个 ws 会话,BrowserSession 增加 `switchTab(targetId)`;内置浏览器对应到多个 `webContents`
|
|
||||||
|
|
||||||
### 阶段四:拾取器与高级组件适配(约 2~3 天)
|
|
||||||
|
|
||||||
- 元素拾取:外部浏览器场景拾取请求走 CDP `DOMSnapshot.captureSnapshot` / `DOM.getDocument` 生成候选元素树(内置窗口继续走现有拾取链路)
|
|
||||||
- 逐组件验证(阶段一~三完成后自然可用,重点回归):`WebBrowser_GetCookieNew`、`WebBrowser_StartWeblistening / GetWeblistening`(Network 域,外部同样可用)、`WebBrowser_BatchDataCrawlNew`(browserType 字段接阶段二同一工厂)、`WebBrowser_SaveAs`(`DOM.setFileInputFiles` + `Page.handleJavaScriptDialog`)
|
|
||||||
- `Element_*` 系列 100+ 组件无需改动,靠接口层自动生效
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 四、依赖与风险
|
|
||||||
|
|
||||||
### 新增依赖
|
|
||||||
|
|
||||||
| 包 | 大小 | 用途 | 风险 |
|
|
||||||
|---|---|---|---|
|
|
||||||
| `ws` | ~50KB 纯 JS | 连接外部浏览器调试端口(Node 20 无原生 WebSocket) | 无原生编译,供应链风险低 |
|
|
||||||
|
|
||||||
**客户机器零安装**:Edge/Chrome 原生支持 CDP,不装任何扩展插件。
|
|
||||||
|
|
||||||
### 风险与对策
|
|
||||||
|
|
||||||
| 风险 | 对策 |
|
|
||||||
|---|---|
|
|
||||||
| 用户已开浏览器实例吞掉端口参数(连不上且无提示) | 必带独立 `--user-data-dir` 启动;文档注明该限制 |
|
|
||||||
| 调试端口与客户其他软件冲突 | 端口自增探测 + `/json/version` 就绪轮询 |
|
|
||||||
| 外部浏览器被客户手动关闭,流程中途断连 | ws close 事件 → 明确报错「浏览器已关闭」,错误路由可接重开逻辑 |
|
|
||||||
| 页面弹窗阻塞 CDP | `Page.handleJavaScriptDialog` 与内置行为一致,现有组件已覆盖 |
|
|
||||||
| 混用内置/外部时上下文串流 | 会话缓存按 (type, port) 隔离;`RunContext.browser` 生命周期不变 |
|
|
||||||
| URL 安全 | 沿用现有 `assertUrl`(仅 http/https)白名单逻辑,外部连接的 ip:port 另做私网地址校验 |
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 五、工作量与排期估算
|
|
||||||
|
|
||||||
| 阶段 | 内容 | 预估 |
|
|
||||||
|---|---|---|
|
|
||||||
| 一 | 传输层抽象 + 外部连接 + 进程管理 | 2~3 天 |
|
|
||||||
| 二 | 打开网页组件参数生效 | 1 天 |
|
|
||||||
| 三 | 多标签 Target 域 | 2~3 天 |
|
|
||||||
| 四 | 拾取器适配 + 高级组件回归 | 2~3 天 |
|
|
||||||
| 测试 | 与同事网页自动化测试联动,双浏览器(Edge/Chrome)× 关键链路 | 1~2 天 |
|
|
||||||
| **合计** | | **约 1.5~2 周** |
|
|
||||||
|
|
||||||
阶段一+二完成后即可向客户交付「用 Edge/Chrome 打开并执行基础网页自动化」;阶段三、四为完整体验补齐,可穿插进行。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 六、测试计划(验收标准)
|
|
||||||
|
|
||||||
1. **连接**:Edge / Chrome 各验证——全新启动、已装非默认路径(注册表探测)、isRemote 远程连接、端口占用自增
|
|
||||||
2. **基础链路**:打开网页 → 点击 → 输入 → 取文本 → 截图 → 关闭,双浏览器全通过
|
|
||||||
3. **多标签**:开 3 个 Tab → GetTAB 按标题匹配 → 切换 → 各 Tab 独立操作 → closeTarget
|
|
||||||
4. **兜底**:type 缺省/填 zexplorer 时行为与现版本完全一致(内置浏览器回归)
|
|
||||||
5. **异常**:浏览器中途被手动关、URL 非法、调试端口不通、页面弹窗
|
|
||||||
6. **数据采集/监听**:BatchDataCrawlNew、StartWeblistening 在外部浏览器下与内置结果一致
|
|
||||||
788
docs/RPA-Review-Full.md
Normal file
788
docs/RPA-Review-Full.md
Normal file
@ -0,0 +1,788 @@
|
|||||||
|
# RPA 项目技术审查与 Agent-Employee 集成实施总方案
|
||||||
|
|
||||||
|
> 文档性质:统一总方案 + 最新源码复审记录。
|
||||||
|
> 审查对象:资料库中的 `PRA-only-demo-main(2).zip`(本轮解压源码快照)。
|
||||||
|
> 审查重点:RPA Core 边界、运行时安全、能力注册、驱动注入、平台适配、元素拾取、Agent-Employee 接入。
|
||||||
|
> 结论状态:静态源码审查;未将未执行的测试写成通过。
|
||||||
|
|
||||||
|
## 0. 执行摘要
|
||||||
|
|
||||||
|
这份文档统一此前几轮讨论、项目原有技术文档和两轮源码审查结论。它不是“项目已经成熟”的背书,而是当前代码能否安全演示、能否抽成可复用 RPA Core、能否接入 Agent-Employee 的事实清单与实施路线。
|
||||||
|
|
||||||
|
**总体判断:**项目已经从单纯 Electron Demo 走到有流程引擎、能力注册、驱动接口、元素拾取和能力目录的阶段;但当前 `@rpa/core` 还不能被认定为真正独立、可注入、可跨平台的执行内核。最主要的阻塞不在于“再加多少能力”,而在于执行取消/重试安全、Engine/Registry/Driver 生命周期、Electron 依赖残留、包依赖闭包和宿主治理契约。
|
||||||
|
|
||||||
|
当前建议:
|
||||||
|
1. **Windows 10/11 演示优先**,先收敛一条稳定端到端流程,不要为了演示临时扩展 Linux/macOS。
|
||||||
|
2. **接入 Agent-Employee 前先修 P0/P1**,尤其是取消与重试、Driver 注入、动态 Electron 依赖和包依赖闭包。
|
||||||
|
3. **先做垂直切片**:Gateway 授权 → RPA Runtime → 一个文件能力 → 一个浏览器能力 → 一个桌面能力 → 事件/证据回传,再逐批搬能力。
|
||||||
|
4. **跨平台按能力域分批实现**。当前代码是 Windows-first,不是 Windows/Linux/macOS 三端可用;类型里出现平台占位不代表已有实现。
|
||||||
|
|
||||||
|
## 1. 项目目标与已确认约束
|
||||||
|
|
||||||
|
以下是此前项目讨论中已经明确的目标,不是本轮从代码推断出来的事实:
|
||||||
|
|
||||||
|
- 项目定位:面向办公自动化的 RPA 平台,后续接入已有 Agent-Employee / opencode 底座。
|
||||||
|
- 用户负责 RPA 部分;商业项目已有独立的服务端 AI 组件,RPA Core 不应重复承担服务端模型部署职责。
|
||||||
|
- 总体方向:**AI 模式 hybrid;部署方式 local;跨平台目标 all**。
|
||||||
|
- 元素拾取策略:**本地优先、云端兜底**;hover 实时识别高亮,click 确认。
|
||||||
|
- 目标软件范围:Office、浏览器、Electron、Qt 以及自绘界面。
|
||||||
|
- 识别策略:UIA/Accessibility 等结构化接口优先,OCR/视觉模型补足;不把 YOLO 训练作为前置条件,也不把 SAM 作为当前主链路。
|
||||||
|
- 当前实际交付重点:Windows 10/11;Linux/macOS 是后续架构目标,不应在未完成驱动与平台能力矩阵前宣传已支持。
|
||||||
|
- 技术现状:Electron + Node.js + React + TypeScript;RPA Core 位于 `packages/rpa-core`,宿主 UI/拾取窗口仍在 `src/main` / renderer。
|
||||||
|
|
||||||
|
## 2. 当前架构画像
|
||||||
|
|
||||||
|
```text
|
||||||
|
Electron Desktop Host
|
||||||
|
├─ Renderer / Flow Canvas / Properties UI
|
||||||
|
├─ Main Process
|
||||||
|
│ ├─ flow persistence / settings / logging
|
||||||
|
│ ├─ desktop picker / overlay / vision lab
|
||||||
|
│ └─ host bridge initialization
|
||||||
|
└─ @rpa/core (workspace package)
|
||||||
|
├─ Flow Runtime / expression / error model
|
||||||
|
├─ module-global Capability Registry
|
||||||
|
├─ capability handlers (files, HTTP, Office, desktop, OCR, vision...)
|
||||||
|
├─ WindowsDriver / PowerShell helpers
|
||||||
|
├─ HostBridge / SubflowSource module singletons
|
||||||
|
└─ Browser CDP transport / external browser launcher
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2.1 已经具备的基础
|
||||||
|
|
||||||
|
- 有独立的 `packages/rpa-core` 目录和包名 `@rpa/core`。
|
||||||
|
- 有 Flow Runtime、RunContext、NodeState、RuntimeError、能力注册表与能力元数据。
|
||||||
|
- 有 Windows PowerShell driver 封装及鼠标、键盘、窗口、屏幕、UIA 五域接口定义。
|
||||||
|
- 有 `registerAll()` 能力分组加载入口。
|
||||||
|
- 有 HostBridge / SubflowSource 抽象,减少部分宿主 API 的直接依赖。
|
||||||
|
- 有元素拾取相关代码、OCR/视觉能力、能力目录和较多单元/冒烟测试。
|
||||||
|
|
||||||
|
### 2.2 不能据此宣称已经完成的事项
|
||||||
|
|
||||||
|
- Core 仍有动态 `import("electron")`,不是 Electron-free。
|
||||||
|
- `createEngine()` 当前没有真正接收 Driver;`registerAll(engine)` 也没有把注册表绑定到该 Engine。
|
||||||
|
- 多个能力仍直接依赖 PowerShell、Win32、COM、Electron 屏幕截图 API。
|
||||||
|
- 当前没有 MacDriver / LinuxDriver。
|
||||||
|
- `@rpa/core` 的 package manifest 没有声明源码实际使用的全部外部依赖。
|
||||||
|
- 尚未完成 Agent-Employee 真实契约的编译级对接。
|
||||||
|
- 本轮未运行 Vitest、typecheck、Electron build、Bun test、Windows 实机 E2E。
|
||||||
|
|
||||||
|
## 3. 本轮继续审查新增发现
|
||||||
|
|
||||||
|
### P1-NEW-01:RPA Core 内仍有动态 Electron 依赖,依赖护栏漏检
|
||||||
|
|
||||||
|
**证据:**
|
||||||
|
- `packages/rpa-core/src/capabilities/vision.ts:18,66,432`:动态 `import("electron")`,读取 `desktopCapturer` / `screen`。
|
||||||
|
- `packages/rpa-core/src/capabilities/cv-vision.ts:51,213`:动态 `import("electron")`,获取截图与 DIP 坐标转换。
|
||||||
|
- `packages/rpa-core/scripts/check-deps.mjs:20-20,38-43`:当前扫描表达式只匹配静态 `import/export ... "specifier"`,不匹配 `import("electron")`。
|
||||||
|
- `packages/rpa-core/src/engine/host-bridge.ts:6-8` 与 `docs/rpa-core-migration-guide.md:28` 却声称 Core 禁止 Electron / 完全不依赖 Electron。
|
||||||
|
|
||||||
|
**影响:**
|
||||||
|
- Node/Bun 独立运行时,调用视觉能力时会尝试加载 Electron;CLI 只跑未触发这些分支的能力,不能证明整个 Core 脱离 Electron。
|
||||||
|
- Agent-Employee backend 若不是 Electron 主进程,视觉能力会在运行时失败或无法复用。
|
||||||
|
- 依赖护栏“通过”目前只能说明它没有识别到静态违规,不能作为边界验收证据。
|
||||||
|
|
||||||
|
**整改:**
|
||||||
|
1. 将屏幕截图、显示器信息、DPI 换算抽成 `ScreenCaptureHost` / `DisplayCoordinateService` 接口,由 Electron Host 实现注入。
|
||||||
|
2. `vision.ts`、`cv-vision.ts` 只依赖接口,不再动态加载 Electron。
|
||||||
|
3. 依赖护栏补动态 import、`require()`、`createRequire()` 与 `import()` 字符串扫描;最好使用 TypeScript AST,而不是继续堆正则。
|
||||||
|
4. CI 增加一个纯 Node/Bun 进程加载 Core 并实际调用视觉 Host mock 的测试。
|
||||||
|
|
||||||
|
**验收:**在没有 Electron 包可解析的 Node/Bun 环境下,Core 能加载;视觉能力在 Host 未注入时返回明确的 `host_service_unavailable`,注入 Fake Host 后可完成截图/坐标测试。
|
||||||
|
|
||||||
|
### P1-NEW-02:`@rpa/core` 包依赖声明不完整
|
||||||
|
|
||||||
|
**证据:**
|
||||||
|
- `packages/rpa-core/src/capabilities/vision.ts` 使用 `@napi-rs/canvas`。
|
||||||
|
- `packages/rpa-core/src/capabilities/pdf.ts` 使用 `pdfjs-dist`、`@napi-rs/canvas`、`@cantoo/pdf-lib`。
|
||||||
|
- `packages/rpa-core/package.json` 未声明上述三个依赖;它们目前只出现在根项目依赖(其中 `@cantoo/pdf-lib`、`@napi-rs/canvas`、`pdfjs-dist`)。
|
||||||
|
- `package-lock.json` 的 `packages/rpa-core` workspace 条目同样未列出这些依赖。
|
||||||
|
|
||||||
|
**影响:**当前 monorepo 根目录安装时可能通过依赖提升“碰巧可用”,但把 `packages/rpa-core` 单独迁入 Agent-Employee 或单独安装时,相关能力可能出现 `Cannot find package`。这属于包边界和发布闭包问题。
|
||||||
|
|
||||||
|
**整改:**逐个核对 Core 的运行时外部依赖;凡 Core 源码运行时 import 的依赖,要么声明在 Core dependencies/optionalDependencies,要么从 Core 移除并由 Host 注入。原生模块需额外验证目标 OS 与 Bun/Node 兼容性。
|
||||||
|
|
||||||
|
**验收:**在隔离目录只安装 `@rpa/core` 声明的依赖,运行包加载、PDF/OCR/图像相关代表能力;不得依赖根项目的幽灵依赖。
|
||||||
|
|
||||||
|
### P1-NEW-03:`registerAll(engine)` 参数和 Engine 实例没有实际隔离效果
|
||||||
|
|
||||||
|
**证据:**
|
||||||
|
- `packages/rpa-core/src/capabilities/register-all.ts:49-56` 接收 `engine`,但只执行模块动态 import,最终仅 `return engine`。
|
||||||
|
- `packages/rpa-core/src/engine/registry.ts:33-34` 的 handlers/metas 是模块级 Map。
|
||||||
|
- `packages/rpa-core/src/engine/runtime.ts:260` 从全局 `getHandler()` 取 handler。
|
||||||
|
- `packages/rpa-core/src/engine/engine.ts:45-49` 的 `driver` 固定为 `new WindowsDriver()`,`createEngine()` 无注入参数。
|
||||||
|
- `packages/rpa-core/src/capabilities/desktop.ts:9`、`common-final.ts:11`、`system-extended.ts:36`、`window-extended.ts:7` 读取 `defaultEngine.driver`。
|
||||||
|
|
||||||
|
**影响:**调用者可能以为每个 Engine 有独立的 Driver、能力注册表和权限域,实际执行仍共享进程级注册表和默认 WindowsDriver。多 Agent、多桌面会话、测试并行、不同租户策略会互相污染。`createEngine()` 目前更像状态容器,不是完整隔离的执行实例。
|
||||||
|
|
||||||
|
**整改:**
|
||||||
|
1. 先确定目标架构:建议 `RpaEngine` 持有 Registry、Driver、HostServices、SessionContext。
|
||||||
|
2. `createEngine({ driver, host, registry, ... })` 显式注入。
|
||||||
|
3. handler 注册写入 `engine.registry`;Runtime 从 `ctx.engine` 或构造参数拿 handler,不读全局 Map。
|
||||||
|
4. 能力 handler 不捕获 `defaultEngine.driver`;改为从 `RunContext` 注入的 execution services 读取。
|
||||||
|
5. 过渡期如保留全局 API,明确标记 deprecated,不能让 `registerAll(engine)` 假装实例化注册。
|
||||||
|
|
||||||
|
**验收:**创建两个 Engine,分别注入 FakeDriverA/B 并注册不同能力集;并行运行时互不串用 Driver、handler、HostBridge、子流程源。
|
||||||
|
|
||||||
|
### P1-NEW-04:CLI 冒烟测试不能证明 Engine 注入和 Core 独立性
|
||||||
|
|
||||||
|
**证据:**
|
||||||
|
- `scripts/rpa-cli.ts:24-25` 同时 `import { ... } from "@rpa/core"` 和 `import "@rpa/core/capabilities"`(副作用全量注册)。
|
||||||
|
- `scripts/rpa-cli.ts:47-49` 随后调用 `createEngine()` / `registerAll(engine)`,但该参数没有绑定 Registry。
|
||||||
|
- `scripts/rpa-cli.ts:53-58` 通过全局 `getHandler()` 执行 handler,而不是通过 Engine 获取。
|
||||||
|
- `scripts/rpa-cli.ts:5-10` 声称 5 项代表能力,实际执行了 Window_GetActiveWindow、Window_GetScreenSize、Excel_Cs_NewV5、File_Write、File_Read、Http_Get 共 6 次 handler 调用。
|
||||||
|
|
||||||
|
**影响:**CLI 可以验证某些能力在当前 Node + 根目录依赖 + Windows 环境下能运行,但不能验证 Driver 注入、Engine 隔离、显式注册、无 Electron 依赖或独立包依赖闭包。文档将它当成“零宿主独立运行”的证明,结论过度。
|
||||||
|
|
||||||
|
**整改:**移除副作用 import;测试只使用显式注册;通过 Engine API 获取 handler 并运行;测试 FakeDriver 注入;将 CLI 验收拆成纯 Node、Windows Host、Electron Host 三种,不把联网 HTTP 作为唯一固定验收依赖。
|
||||||
|
|
||||||
|
### P1-NEW-05:迁移文档与任务清单互相矛盾,完成度表述不可信
|
||||||
|
|
||||||
|
**证据:**
|
||||||
|
- `docs/rpa-core-migration-guide.md:22` 写“改造已全部完成”;第 28 行写 Core 完全不依赖 Electron。
|
||||||
|
- `docs/rpa-core-refactor-tasks.md:17-91` 阶段 0 至阶段 6 的任务仍全部保留 `[ ]` 未勾选。
|
||||||
|
- 源码又存在动态 Electron import、Driver 注入缺失、包依赖缺项。
|
||||||
|
|
||||||
|
**影响:**接手者会按“已完成”理解架构状态,低估接入工作量;也无法判断哪些是设计目标、哪些是已验收能力。
|
||||||
|
|
||||||
|
**整改:**统一文档状态标记:`已实现(代码定位)`、`已测试(测试证据)`、`已在目标宿主验证`、`计划中`。每项完成必须附 commit / 测试命令 / 测试结果,不以文档描述代替代码事实。
|
||||||
|
|
||||||
|
## 4. 元素拾取与视觉定位:统一技术路线
|
||||||
|
|
||||||
|
元素拾取不应被等同为“找一个 AI 模型识别按钮”。办公 RPA 需要的是可执行、可复定位的 Target,而不仅是截图框。
|
||||||
|
|
||||||
|
### 4.1 建议识别优先级
|
||||||
|
|
||||||
|
1. **结构化可访问性接口**:Windows UIA;浏览器 DOM/CDP;macOS AX;Linux AT-SPI。
|
||||||
|
2. **OCR + 几何关系**:用于文字、标签、按钮文本与相邻控件关系,输出文字和 bounding box。
|
||||||
|
3. **免训练视觉候选检测**:当前模型实验可作为候选框生成器,但不要将模型类别直接当成业务语义。
|
||||||
|
4. **模板/局部视觉匹配**:对图标、无文本控件和自绘 UI 作兜底。
|
||||||
|
5. **人工确认**:候选冲突、低置信度或高风险动作时要求用户确认。
|
||||||
|
|
||||||
|
### 4.2 Selector 应保存“语义 + 定位证据”,而不是只存坐标
|
||||||
|
|
||||||
|
建议 Target 结构至少包含:
|
||||||
|
- `platform / app / process / window`:目标环境身份;
|
||||||
|
- `accessibility`:AutomationId、Name、ControlType、ClassName、祖先路径;
|
||||||
|
- `browser`:URL/域名、frame、DOM role/aria-label/test id 等;
|
||||||
|
- `ocr`:识别文本、语言、置信度、相对区域;
|
||||||
|
- `visual`:bbox、截图裁剪 hash、模型标识、相似度;
|
||||||
|
- `fallback`:最后一次屏幕坐标、DPI/缩放、窗口矩形、采集时间。
|
||||||
|
|
||||||
|
运行时定位顺序:结构化 selector → 同窗口/同 frame 限定查找 → OCR/视觉重定位 → 低置信度时停止并要求确认。禁止将拾取时 `screen=x,y,w,h` 当作长期稳定定位。
|
||||||
|
|
||||||
|
### 4.3 当前代码复核原则
|
||||||
|
|
||||||
|
- 之前报告指出的 `candidateId` 回填错位问题,在本次 ZIP 中已看到映射修复,不重复当作当前缺陷。
|
||||||
|
- `desktop-pick.ts` 中候选排序、筛选与执行 selector 的一致性仍需专项回归,尤其 Edge debug/CDP、DPI、多屏和 hover 高频刷新。
|
||||||
|
- `vision.ts` / `cv-vision.ts` 的 Electron 直接调用应迁到 Host 服务,不能放在可复用 Core 中。
|
||||||
|
- OCR 是语义补充,不应单独负责确定控件类型;模型候选类别也不能直接映射成“登录按钮”等业务含义。
|
||||||
|
|
||||||
|
## 5. Agent-Employee 集成目标架构
|
||||||
|
|
||||||
|
```text
|
||||||
|
Agent-Employee Workflow / Agent Runtime
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
RPA Adapter(唯一入口)
|
||||||
|
├─ Workflow/Step 状态映射
|
||||||
|
├─ Capability Gateway:授权、租约、审批、审计
|
||||||
|
├─ 参数/返回值 Schema 转换
|
||||||
|
├─ Trace / Event / Evidence 关联
|
||||||
|
└─ 取消、超时、幂等与重试策略
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
RPA Core(每次执行/会话实例)
|
||||||
|
├─ Flow Validator
|
||||||
|
├─ Runtime / Expression / Error Contract
|
||||||
|
├─ Engine-scoped Registry
|
||||||
|
├─ Capability Handlers
|
||||||
|
└─ Execution Context(AbortSignal / Trace / Permission / Evidence)
|
||||||
|
│
|
||||||
|
├──────── Host Services(宿主注入)
|
||||||
|
│ ├─ Browser Adapter
|
||||||
|
│ ├─ Screenshot / Display / DPI
|
||||||
|
│ ├─ Dialog / AI Config / Subflow Store
|
||||||
|
│ └─ Event / Evidence Sink
|
||||||
|
│
|
||||||
|
└──────── Computer Control Driver
|
||||||
|
├─ WindowsDriver(当前优先)
|
||||||
|
├─ MacDriver(后续)
|
||||||
|
└─ LinuxDriver(后续)
|
||||||
|
```
|
||||||
|
|
||||||
|
**权责原则:**
|
||||||
|
- Agent-Employee Workflow/Runtime 是宿主最终 Run/Step 状态的唯一权威来源;RPA Runtime 作为执行器或执行子流程,不另建第二套冲突状态系统。
|
||||||
|
- Capability Gateway 是所有带副作用能力的唯一授权入口。RPA Core 不再另建一套与其冲突的 allowlist。
|
||||||
|
- computer-control 是桌面观察/定位/动作契约的优先对齐对象;不要让 RPA Driver 与宿主契约长期并行。
|
||||||
|
- HostServices 与 Driver 按 Engine/Session 注入,不使用模块级全局 setter 承载多租户/多桌面会话状态。
|
||||||
|
- 高风险能力(命令执行、键鼠注入、文件删除/覆盖、发送邮件/消息、数据库写操作)必须有权限、审批/租约、审计和资源范围。
|
||||||
|
|
||||||
|
## 6. 当前问题优先级总表
|
||||||
|
|
||||||
|
| 优先级 | 问题 | 接入影响 | 建议门槛 |
|
||||||
|
|---|---|---|---|
|
||||||
|
| P0 | 超时只让外层 Promise reject,底层操作不一定停止 | 可能重叠执行、重复副作用 | 接入前修复 |
|
||||||
|
| P0 | 取消状态与 NodeState 不一致;取消可能被当失败/跳过 | 宿主状态与执行状态冲突 | 接入前修复 |
|
||||||
|
| P0 | retry 默认策略过宽,非幂等动作可能重试 | 重复点击/输入/发信/覆盖 | 接入前修复 |
|
||||||
|
| P0 | Engine 无 Driver 注入;能力使用 defaultEngine | 无法按会话隔离/测试替身 | 接入前修复 |
|
||||||
|
| P1 | Registry 模块级全局 Map,registerAll(engine) 不实例化注册 | 多 Agent/多权限域污染 | 接入前明确并改造 |
|
||||||
|
| P1 | HostBridge / SubflowSource 模块级单例 | 多宿主/多会话覆盖 | 接入前改造 |
|
||||||
|
| P1 | vision/cv-vision 动态依赖 Electron | Core 不能独立运行 | 迁移前修复 |
|
||||||
|
| P1 | Core manifest 缺失 canvas/pdf 依赖 | 单包安装不完整 | 迁移前修复 |
|
||||||
|
| P1 | CLI 依赖全局注册,不能证明注入/独立性 | 现有验收证据不足 | 重写验收 |
|
||||||
|
| P1 | Flow Graph 无完整 Validator | 多出口/非法边可能静默部分执行 | 接入前补校验 |
|
||||||
|
| P1 | CapabilityMeta 缺权限/副作用/幂等字段 | Gateway 映射不完整 | Gateway 对接前补齐 |
|
||||||
|
| P1 | Windows 专属能力散落在能力层 | 平台能力边界不清 | 平台适配前整理 |
|
||||||
|
| P1 | catalog platformTag 类型异常 | 平台过滤/界面标记不可信 | 修复并加 schema test |
|
||||||
|
| P1 | Agent-Employee 契约尚未逐接口编译验证 | 估算与实现存在偏差 | 联调前必须做 |
|
||||||
|
| P1 | BrowserSession 的 locate 与执行查找器语义不一致 | 拾取后 XPath/Shadow DOM 选择器可能执行失败 | 浏览器能力验收前修复 |
|
||||||
|
| P1 | iframe 未纳入拾取/定位/执行上下文 | iframe 内元素无法保证端到端可用,跨域 iframe 尤其明显 | 浏览器能力验收前明确支持边界 |
|
||||||
|
| P1 | CDP 单条命令无响应超时 | 浏览器命令可能永久 pending,流程卡死 | 接入前补统一超时与清理 |
|
||||||
|
| P1 | 外部浏览器“前台标签跟随”取 pages[0] | 多标签时可能切错目标页 | 多标签回归前修复 |
|
||||||
|
| P1 | AI 拾取取消未贯穿检测/OCR任务 | 取消后后台推理仍可能继续,OCR 等待无硬超时 | 拾取稳定性验收前修复 |
|
||||||
|
| P2 | 依赖护栏只扫静态 import,且只覆盖 TS/TSX | 边界回归容易漏检 | 尽快增强 |
|
||||||
|
| P2 | AI 请求没有调用方 AbortSignal | 拾取取消不能立即中止请求 | 交互体验整改 |
|
||||||
|
| P2 | 包没有独立 build/pack 验收 | 迁移消费方式不稳定 | 发布前补齐 |
|
||||||
|
|
||||||
|
## 7. 执行安全整改方案
|
||||||
|
|
||||||
|
### 7.1 取消必须贯穿整条调用链
|
||||||
|
|
||||||
|
`RunContext.signal` → Runtime → Handler → Host/Driver → PowerShell 子进程 / HTTP / Browser CDP。每层都必须接收 signal,并在取消时清理资源。
|
||||||
|
|
||||||
|
PowerShell 子进程需要持有 ChildProcess 引用,收到 abort 后执行 terminate/kill;仅设置 `execFile` timeout 不等于响应上游取消。浏览器动作、等待、网络请求、OCR/AI 请求也应尽量可取消。
|
||||||
|
|
||||||
|
### 7.2 超时不等于操作已停止
|
||||||
|
|
||||||
|
当前 `withTimeout()` 只让外层 Promise reject,不保证底层 handler 结束。超时后必须先 abort 底层任务并等待其停止/清理,再决定是否重试;不能直接开始第二次点击或输入。
|
||||||
|
|
||||||
|
### 7.3 重试必须基于能力语义
|
||||||
|
|
||||||
|
新增能力元数据:`sideEffect`、`idempotent`、`retryPolicy`、`permission`、`approvalRequired`、`resourceScope`。非幂等能力默认不重试;只有明确幂等、可安全重复的操作才能自动重试。对“发送消息/邮件、提交表单、点击付款、删除/覆盖文件”等默认禁止盲目重试。
|
||||||
|
|
||||||
|
### 7.4 状态机
|
||||||
|
|
||||||
|
至少统一 `pending → running → succeeded | failed | cancelled | skipped`。取消不能被转成普通失败,也不能误走错误分支。RunResult、NodeState、Agent-Employee Step 状态需要一份映射表和契约测试。
|
||||||
|
|
||||||
|
## 8. 平台适配路线
|
||||||
|
|
||||||
|
### 8.1 当前定位
|
||||||
|
|
||||||
|
当前实际是 Windows-first:PowerShell、Win32、UIA、Office COM、Windows 剪贴板和屏幕 API 占据大量路径。`platforms: ["win32"]` 的类型声明只是当前标记,不是跨平台实现。
|
||||||
|
|
||||||
|
### 8.2 建议分阶段
|
||||||
|
|
||||||
|
- **阶段 A:Windows 可靠性**:Windows 10/11、UIA、Edge/Chrome CDP、Office COM、DPI、多显示器、PowerShell 取消/超时、安装包资源。
|
||||||
|
- **阶段 B:Linux 桌面基础**:先选 X11 或 Wayland 的支持边界;AT-SPI、输入注入、截图、窗口管理分别实现;Wayland 必须明确权限与限制。
|
||||||
|
- **阶段 C:macOS 桌面基础**:Accessibility API、CGEvent/AppleScript、TCC 辅助功能和屏幕录制授权流程。
|
||||||
|
- **阶段 D:Office/文件能力矩阵**:Windows COM;macOS AppleScript/Office;Linux LibreOffice 或标记不支持。每项能力逐个标平台,不能按“同名软件存在”推断等价。
|
||||||
|
|
||||||
|
### 8.3 平台标记单一来源
|
||||||
|
|
||||||
|
`register-all.ts` 的能力分组、CapabilityMeta.platforms、catalog.platformTag 必须由一份平台能力表派生;当前发现 `catalog.json` 中 1016 项里 1013 项 `platformTag=null`,另 3 项是数字 `1`,与 `src/types.ts` 的 `string | null` 不符。`build-catalog.mjs` 的 `component.platformTag || ...` 会保留 truthy 数字,必须增加 schema 校验并修复源数据。
|
||||||
|
|
||||||
|
## 9. 推荐实施顺序与阶段门禁
|
||||||
|
|
||||||
|
### 阶段 0:先固化演示基线(Windows)
|
||||||
|
- 冻结一个端到端演示流程,准备稳定的测试软件、数据和账号。
|
||||||
|
- 把必需节点与不稳定节点分开;演示脚本不得依赖外网随机响应。
|
||||||
|
- 验收:连续完整执行多轮;每轮有运行日志、节点状态、失败截图/证据;取消后无残留子进程/后台动作。
|
||||||
|
|
||||||
|
### 阶段 1:Core 接入前 P0
|
||||||
|
- 贯通 AbortSignal 和底层取消。
|
||||||
|
- 取消状态正确映射。
|
||||||
|
- 重试策略按幂等性约束。
|
||||||
|
- Engine 真正支持 Driver/Host/Registry 注入。
|
||||||
|
- 验收:FakeDriver、超时、取消、重试、双 Engine 隔离测试全绿。
|
||||||
|
|
||||||
|
### 阶段 2:包边界闭环
|
||||||
|
- 移除 Core 动态 Electron import。
|
||||||
|
- 补齐/收敛 package dependencies。
|
||||||
|
- 依赖扫描升级为 AST 或覆盖动态 import/require。
|
||||||
|
- 重写 CLI,去掉副作用全量 import,验证 Engine-scoped registry。
|
||||||
|
- 验收:隔离安装、Node/Bun 纯 Core 加载、Host Mock 视觉路径。
|
||||||
|
|
||||||
|
### 阶段 3:Agent-Employee 契约对齐
|
||||||
|
- 获取目标仓库真实源码与当前 commit。
|
||||||
|
- 对 `workflow-engine`、`capability-gateway`、`computer-control`、`windows-plugin`、`browser-plugin` 的类型、生命周期、事件和错误模型逐接口对照。
|
||||||
|
- 输出 Adapter 映射表与 ADR,不根据迁移指南中的摘要直接开始实现。
|
||||||
|
- 验收:编译通过、Gateway 拒绝时 handler 零调用、事件/证据可追踪。
|
||||||
|
|
||||||
|
### 阶段 4:小范围垂直切片
|
||||||
|
- 先接一个纯文件能力、一个浏览器能力、一个桌面能力。
|
||||||
|
- 跑通授权、租约、审批、执行、取消、事件、证据回传。
|
||||||
|
- 验收后才开始分批迁移能力组。
|
||||||
|
|
||||||
|
### 阶段 5:能力规模化与跨平台
|
||||||
|
- 能力分组迁移、平台标记、风险元数据补齐。
|
||||||
|
- Linux/macOS 按驱动和能力矩阵逐步开放,不做虚假全平台开关。
|
||||||
|
|
||||||
|
## 10. 交付给接入团队的材料清单
|
||||||
|
|
||||||
|
1. 本总方案(本文件)。
|
||||||
|
2. `docs/RPACore二次源码审查与接入整改报告.md`:此前两轮源码审查细项与原始定位,作为附录参考。
|
||||||
|
3. `docs/rpa-core-migration-guide.md`:现有迁移指南,但其中“完全不依赖 Electron / 已全部完成”等表述应按本总方案纠正。
|
||||||
|
4. `docs/rpa-core-refactor-tasks.md`:原改造任务清单,当前勾选状态与迁移指南冲突,应重新按证据更新。
|
||||||
|
5. `docs/06-AI拾取模型选型与RPA执行方案.md`、`docs/图像与OCR问题检测报告.md`:拾取与视觉路线参考,需结合当前实现再次做 Windows 实机回归。
|
||||||
|
|
||||||
|
## 11. 本轮审查边界
|
||||||
|
|
||||||
|
- 已读取资料库中的最新 `PRA-only-demo-main(2).zip`,并检查 `packages/rpa-core`、核心运行时、注册入口、Driver、依赖护栏、CLI、catalog 和项目文档。
|
||||||
|
- 已核验 `check-deps.mjs` 的实现逻辑;该脚本的扫描方式确实漏掉动态 `import("electron")`。
|
||||||
|
- 未安装项目依赖;未执行 Vitest、TypeScript typecheck、Electron build、Bun test、打包和 Windows 实机 E2E。不得把这些项目标成“通过”。
|
||||||
|
- 当前没有 Agent-Employee 目标仓库真实源码作为本轮输入,因此本文件给出的是接入架构和待核验契约清单,不代表已完成目标仓库编译对接。
|
||||||
|
- ZIP 中不包含 `node_modules`;本轮静态结论以该 ZIP 源码快照为准。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 附录 A:此前二次源码审查细项
|
||||||
|
|
||||||
|
以下保留此前审查报告的完整细项,包括已修复问题回归、P0/P1/P2 代码定位、Graph Validator、HostBridge、Registry、平台标记、能力元数据和验收标准,避免统一文档时丢失前几轮审查证据。
|
||||||
|
|
||||||
|
# RPACore 二次源码审查与 Agent-Employee 接入整改报告
|
||||||
|
|
||||||
|
> 审查对象:资料库中的 `PRA-only-demo-main(2).zip`(约 154.8 MB)
|
||||||
|
> 审查日期:2026-11-06
|
||||||
|
> 审查范围:`packages/rpa-core/`、流程运行时、能力注册、Driver/HostBridge、能力目录、桌面拾取与打包配置、原有迁移文档
|
||||||
|
> 审查方式:源码静态审查 + 项目结构/JSON/脚本检查;未在 Windows 真实桌面环境执行端到端测试。
|
||||||
|
> 重要说明:本报告区分“源码已确认”“设计风险/需产品语义确认”“尚未运行验证”,不把静态推断写成运行结果。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 0. 总体结论
|
||||||
|
|
||||||
|
当前项目已经不是简单的 Electron 自动化 Demo:它有独立的 `packages/rpa-core` workspace、流程运行时、能力注册表、五域 Driver 类型、HostBridge、外部浏览器 CDP、桌面 UIA/视觉拾取以及较多能力测试。**现有技术栈不需要推翻,真正需要解决的是运行时语义、依赖注入闭环、能力治理和平台边界。**
|
||||||
|
|
||||||
|
但当前状态还不适合直接宣称“可无修改迁入 Agent-Employee”或“已具备跨平台能力”。建议定位为:**Windows-first 的 RPA 能力包,已建立初步包边界,仍需完成执行语义与宿主契约收口。**
|
||||||
|
|
||||||
|
### 本轮核验到的项目事实
|
||||||
|
|
||||||
|
- ZIP 内约 272 个条目、约 230 个文件;`packages/rpa-core/src` 包含 35 个 `*.test.ts` 文件。
|
||||||
|
- `packages/rpa-core` 的包名仍是 `@rpa/core`,`private: true`,入口直接指向 `src/index.ts`,尚未形成面向普通 Node/Bun 部署的独立构建产物契约。
|
||||||
|
- `node packages/rpa-core/scripts/check-deps.mjs` 本次执行通过;这只证明该脚本当前扫描规则未发现违规 import,不等于包完全没有宿主/平台耦合。
|
||||||
|
- `resources/catalog.json` 声明 1016 个组件:1013 个 `platformTag` 为 `null`,3 个 `platformTag` 是数字 `1`,与 `CatalogComponent.platformTag: string | null` 类型不一致。
|
||||||
|
- `electron-builder.config.mjs` 当前已经把 `resources/desktop-picker.ps1` 放入 `extraResources`;AI 候选覆盖层当前用 `candidateId` 回传并按 ID 查找原候选。这两项是旧版元素拾取报告中的 P0,本次最新快照中已看到对应修正,不应继续作为“当前未修复问题”重复上报。
|
||||||
|
|
||||||
|
### 建议优先级
|
||||||
|
|
||||||
|
| 优先级 | 主题 | 结论 |
|
||||||
|
|---|---|---|
|
||||||
|
| P0 | 超时、取消与重试的副作用安全 | 先处理;否则自动化可能出现重复点击/重复写入 |
|
||||||
|
| P0 | Engine/Driver/Registry/HostBridge 依赖闭环 | 接入前必须明确,否则宿主无法可靠隔离会话与权限域 |
|
||||||
|
| P1 | 流程图结构校验和执行语义 | 防止无效图被静默部分执行 |
|
||||||
|
| P1 | 能力元数据与 Capability Gateway | Agent 不应绕过授权直接调用高影响能力 |
|
||||||
|
| P1 | 平台目录、Driver 边界与 Windows 专属能力 | 不应把“抽象接口存在”描述成 Linux/macOS 已支持 |
|
||||||
|
| P1 | 包构建、Bun 兼容和集成验收 | 迁入前必须有目标运行时证据 |
|
||||||
|
| P2 | 依赖扫描增强、注册冲突治理、可观测性 | 提高长期维护质量 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. P0:执行安全与依赖闭环
|
||||||
|
|
||||||
|
### P0-01 超时只终止 Runtime 等待,不保证底层能力停止
|
||||||
|
|
||||||
|
**源码位置**:`packages/rpa-core/src/engine/runtime.ts:33-55, 279-307`;`packages/rpa-core/src/engine/types.ts:118-137`;`packages/rpa-core/src/driver/windows/powershell.ts:45-69`。
|
||||||
|
|
||||||
|
**确认事实**:`withTimeout()` 通过计时器拒绝外层 Promise;它没有创建并传递专属 AbortController,也没有在超时后通知 handler 停止。`RunContext` 虽然有 `signal`,但 `AutomationDriver` 的五域方法没有 `signal` 参数,PowerShell helper 也没有接受调用方 AbortSignal 的接口。PowerShell helper 自己有 30 秒等进程超时,但这与 Runtime 节点超时不是同一个取消链路。
|
||||||
|
|
||||||
|
**影响**:节点超时后 Runtime 可能已经开始重试,前一次 PowerShell/桌面操作却仍在执行。对点击、输入、删除、覆盖文件、发邮件、提交表单等非幂等动作,可能产生重复副作用。用户按取消后,运行状态结束也不代表底层操作已经停止。
|
||||||
|
|
||||||
|
**整改**:
|
||||||
|
1. 为每次节点尝试创建独立 `AbortController`,把 Run 级 signal 与 Node 级 timeout 合并。
|
||||||
|
2. Handler/Driver/PowerShell/HTTP/浏览器操作逐层接收 signal;可取消的操作必须真实中止。
|
||||||
|
3. 对不能中止的原生/外部动作,标记为 `nonCancellable`,超时后不得自动重试,必须等待进程退出或进入人工恢复状态。
|
||||||
|
4. 将重试策略从“按错误码猜测是否可重试”改为能力元数据声明:`idempotent`、`retryPolicy`、`sideEffectLevel`。
|
||||||
|
5. 默认重试次数设置保守上限;当前 `retryCount` 被允许到 100,且 `isRetryable()` 将大多数未知错误视为可重试,不适合作为有副作用能力的默认策略。
|
||||||
|
|
||||||
|
**验收**:构造一个执行 5 秒的 Fake Handler,节点 timeout 设 100ms;验证 100ms 后 handler 收到 abort 并停止;再构造“第一次执行延迟完成、第二次执行”的测试,证明不会发生并发重入。对文件写入/点击类能力增加“超时不自动重试”测试。
|
||||||
|
|
||||||
|
### P0-02 取消时节点状态可能记录为 failed,而非 cancelled
|
||||||
|
|
||||||
|
**源码位置**:`runtime.ts:326-351`;`types.ts:139-160`。
|
||||||
|
|
||||||
|
**确认事实**:`executeNode()` 捕获异常后,普通错误分支会将状态设为 `failed` 或 `skipped`;`RuntimeError("cancelled")` 没有在节点状态层单独映射为 `cancelled`。Run 层随后可能返回 `status: cancelled`,但对应 NodeState 仍显示 failed/skipped。
|
||||||
|
|
||||||
|
**影响**:Agent-Employee 若要做步骤级恢复、审批、审计和用户界面状态展示,会遇到 Run 已取消但 Step 显示失败的语义冲突。
|
||||||
|
|
||||||
|
**整改**:在 `executeNode` catch 中优先判断 `cancelled`,写入 `state.status = "cancelled"`,不执行 `skipOnError` 和普通重试;统一发出 cancelled 事件,并记录取消原因/发起方。
|
||||||
|
|
||||||
|
**验收**:在节点执行中取消;断言 Run 与对应 NodeState 都是 cancelled,后续节点不执行,且不进入 error route。
|
||||||
|
|
||||||
|
### P0-03 Engine 创建接口没有真正的 Driver 注入能力
|
||||||
|
|
||||||
|
**源码位置**:`engine.ts:24-53`;`driver/types.ts:1-10, 158-165`;`packages/rpa-core/README.md:50-57`。
|
||||||
|
|
||||||
|
**确认事实**:`RpaEngine.driver` 类字段固定 `new WindowsDriver()`;`createEngine()` 无参数。Driver 类型注释和 README 却声称 `createEngine()` 可注入自定义 Driver。
|
||||||
|
|
||||||
|
**影响**:不能注入 Agent-Employee `computer-control` 适配器,也不能在单测中使用 Fake Driver;非 Windows 平台创建 Engine 时仍会实例化 Windows 实现。
|
||||||
|
|
||||||
|
**整改**:提供 `createEngine({ driver, registry, hostBridge, subflowSource, ... })`,将默认 Driver 的选择放在平台工厂或宿主装配层;Engine 不应在跨平台包内部无条件创建 WindowsDriver。
|
||||||
|
|
||||||
|
**验收**:Fake Driver 注入测试覆盖 mouse/keyboard/window/screen/element;Linux/macOS 上若无 Driver,明确返回 `platform_unsupported`,不得误报初始化成功。
|
||||||
|
|
||||||
|
### P0-04 Registry 与 Engine 生命周期不一致,`registerAll(engine)` 参数没有生效
|
||||||
|
|
||||||
|
**源码位置**:`registry.ts:33-62`;`register-all.ts:49-57`;`runtime.ts:3, 260`。
|
||||||
|
|
||||||
|
**确认事实**:handlers/metas 是模块级 Map;`registerAll(engine)` 只动态 import 模块,能力模块通过全局 `register()` 写入同一个 Registry;`runFlow()` 用全局 `getHandler()` 查找 handler。传入的 Engine 不决定注册目标。
|
||||||
|
|
||||||
|
**决策背景**:原迁移指南明确选择过“保留全局注册表”。因此问题不是简单的代码疏漏,而是既有决策带来的隔离限制。
|
||||||
|
|
||||||
|
**整改建议**:
|
||||||
|
- 短期单后端、单能力集合:可保留进程级 Registry,但把 API 改成语义真实的 `loadCapabilities(groups)`,文档明确不支持 Engine 级能力隔离。
|
||||||
|
- 长期多 Agent/多租户/不同权限域:改成 `CapabilityRegistry` 实例,由 Engine 持有;显式注册函数接收 Registry,运行时从 Engine Registry 取 handler。
|
||||||
|
|
||||||
|
**验收**:至少有两个 Engine A/B 的注册隔离测试(若采用实例级方案);若暂时采用全局方案,测试需明确证明共享行为是预期设计,并验证重复注册、重复加载和冲突覆盖规则。
|
||||||
|
|
||||||
|
### P0-05 HostBridge/SubflowSource 是模块级单例
|
||||||
|
|
||||||
|
**源码位置**:`engine/host-bridge.ts`(HostBridge 与 SubflowSource 的模块级变量/set/get);`ai-service.ts:20-24`;`human-dialog.ts`;`window-extended.ts`。
|
||||||
|
|
||||||
|
**确认事实**:宿主桥通过模块级变量保存,`setHostBridge()` / `getHostBridge()` 读取同一进程级对象;子流程源也采用相同模式。
|
||||||
|
|
||||||
|
**影响**:多桌面会话、多 Engine、测试并行或多宿主容器时,后一次注入会覆盖前一次配置;AI 配置、DPI 换算、弹窗窗口工厂可能串到错误会话。
|
||||||
|
|
||||||
|
**整改**:把 HostServices 放入 Engine 构造依赖;若宿主明确只有一个全局桌面会话,可保留进程级服务,但应在 ADR 中限定部署模型,并提供 `reset/restore` 测试机制。不要让业务 handler 在模块导入时捕获全局桥对象。
|
||||||
|
|
||||||
|
**验收**:两个 Engine 注入不同 HostBridge,分别触发 AI 配置读取/DPI 转换/弹窗,结果不能交叉。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. P1:流程图语义与运行时完整性
|
||||||
|
|
||||||
|
### P1-01 执行器缺少运行前 Graph Validator
|
||||||
|
|
||||||
|
**源码位置**:`runtime.ts:178-219`;`types.ts:115-145`;`runtime.error-route.test.ts:51-75`。
|
||||||
|
|
||||||
|
**确认事实**:有边时,运行时选择一个起点(优先无入边 Trigger,其次任意无入边节点,再退回 `top[0]`),然后沿边递归执行;普通边通过 `edges.find(...)` 只取第一条非 error 边。访问过的节点由 `visited` 集合直接跳过。当前测试明确验证断开的 orphan 节点不执行,因此“孤立节点不执行”是现有行为,不应未经产品确认就当作 Bug。
|
||||||
|
|
||||||
|
**风险**:如果画布允许多个入口、一个节点多个普通出口、重复边、目标不存在、普通边形成环,运行时可能只执行其中一条路径或静默停止,且没有在执行前给出结构错误。流程编辑器允许的图结构与 Runtime 支持的图结构需要有统一契约。
|
||||||
|
|
||||||
|
**整改**:新增 `validateFlowGraph(flow)`,运行前检查:节点 ID 唯一;边 source/target 必须存在;入口数量与 Trigger 规则一致;普通出口基数符合节点类型;error handle 唯一;不允许意外环;容器子节点 slot 合法;孤立节点明确警告或拒绝。若业务确实支持多出口/并行,必须实现明确的调度语义,不能用 `find()` 随机取第一条。
|
||||||
|
|
||||||
|
**验收**:非法多出口、孤立节点、环、重复 ID、悬空边均在执行前得到稳定错误码;合法 IF/循环/异常分支测试保持通过。
|
||||||
|
|
||||||
|
### P1-02 `skipOnError`、错误分支与 Stop/Break/Continue 的状态语义需统一
|
||||||
|
|
||||||
|
**源码位置**:`runtime.ts:141-154, 203-211, 326-351`。
|
||||||
|
|
||||||
|
**风险**:目前错误路由、skipOnError、Break/Continue/Stop 都在不同层捕获;需要明确一个节点失败后何时走 error edge、何时 skip 并走 normal edge、何时终止整个 Run。特别要保证 cancelled 不会被当作普通可跳过错误。
|
||||||
|
|
||||||
|
**整改**:定义一张状态转移表并由测试固定:成功→normal;可跳过失败→skipped+normal;不可跳过失败且有 error edge→error;无 error edge→Run failed;cancelled→Run cancelled 且不走 error edge;Stop→按产品定义终止状态;Break/Continue 只能在循环上下文捕获。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. P1:能力元数据、安全治理与平台支持
|
||||||
|
|
||||||
|
### P1-01 能力目录平台标记数据异常
|
||||||
|
|
||||||
|
**源码位置**:`resources/catalog.json`;`src/types.ts:49-70`;`scripts/build-catalog.mjs:234-249`。
|
||||||
|
|
||||||
|
**确认事实**:当前 catalog 有 1016 个组件,其中 1013 个 `platformTag=null`,3 个值为数字 `1`(`Basic_BusinessOrderCount`、`Dialog_GetConfigV7`、`Dialog_GetConfigV5`),但类型要求 `string | null`。构建脚本的 `component.platformTag || ...` 会把 truthy 的数字 1 原样保留,无法靠类型声明自动纠正 JSON 数据。
|
||||||
|
|
||||||
|
**影响**:前端/Agent 不能可靠判断某个能力在哪个平台可执行;类型层与运行时 JSON 事实不一致。
|
||||||
|
|
||||||
|
**整改**:平台支持使用统一枚举 `win32 | darwin | linux` 或 `null`;生成 catalog 时做 schema 校验并拒绝非字符串值;平台能力映射只保留一个权威来源,运行时元数据与构建期目录从同一份数据派生。当前只声明 Windows 已实现,其他平台标记为未支持,不要为了“跨平台目标”提前填支持标记。
|
||||||
|
|
||||||
|
**验收**:catalog 构建后 1016 条均通过 schema 校验;每个能力都有明确平台状态;Windows 专属能力不会被误标为跨平台。
|
||||||
|
|
||||||
|
### P1-02 CapabilityMeta 不足以承载 Agent-Employee 权限治理
|
||||||
|
|
||||||
|
**源码位置**:`registry.ts:7-31`;能力目录 `CatalogComponent.risk` 在 `types.ts:65`。
|
||||||
|
|
||||||
|
**确认事实**:Registry 的元数据目前包含 summary/params/returns/timeout/errorCodes/needsScreenshot/platforms;目录只有 `risk: low | high`。未看到统一的权限范围、审批要求、幂等性、副作用类型、资源访问范围等完整治理元数据。
|
||||||
|
|
||||||
|
**影响**:Agent-Employee 的 `capability-gateway` 无法仅凭当前元数据可靠地决定授权、审批、租约范围和审计字段;若直接暴露全部 handler,会绕过宿主治理边界。
|
||||||
|
|
||||||
|
**整改**:在宿主适配层建立唯一 `CapabilityRunner`:授权 → 租约/策略校验 → 输入校验 → 执行 → 证据/审计。为能力补充结构化治理元数据,建议包含 `riskLevel`、`permissions`、`approvalRequired`、`sideEffect`、`idempotent`、`platforms`、`timeoutBudget`;字段最终以 Agent-Employee 契约为准。高影响能力不得通过全局 `getHandler()` 直接暴露给 Agent。
|
||||||
|
|
||||||
|
**验收**:命令执行、键鼠输入、文件删除/覆盖、邮件发送、网络写操作等均经过 Gateway;审批拒绝/租约过期时 handler 不会执行;审计可关联 Agent、Run、Capability、审批结果和执行结果。
|
||||||
|
|
||||||
|
### P1-03 Driver 抽象尚未形成平台隔离边界
|
||||||
|
|
||||||
|
**源码位置**:`driver/types.ts:133-165`;`driver/windows/windows-driver.ts`;`capabilities/desktop-uia.ts`;`capabilities/vision.ts`、`cv-vision.ts`、`remaining-core.ts`、`system-extended.ts`、`word.ts` 等。
|
||||||
|
|
||||||
|
**确认事实**:五域接口存在,但 ScreenDriver 目前只有 `size()`;注释明确截图等 System.Drawing 操作仍可留在能力层。ElementDriver 通过 `setElementDriverImplementation()` 模块级 setter 接到 WindowsDriver;多个能力仍直接调用 PowerShell、Win32、COM 或 Windows 剪贴板。
|
||||||
|
|
||||||
|
**影响**:接口“看起来五域齐全”不等于能力层已平台无关。未来增加 Mac/Linux Driver 时,若不先盘点调用点,平台差异会继续散落在能力 handler 中。
|
||||||
|
|
||||||
|
**整改**:按能力建立 Windows-only 白名单;把截图、剪贴板、窗口/元素观察、键鼠动作、Office COM 等原语逐项归入 Driver/Host adapter;对不适合抽象的能力显式标记平台限制。不要为了抽象而一次性重写全部能力,按高频基础原语先收口。
|
||||||
|
|
||||||
|
**验收**:Core 通用层静态扫描不允许出现未列入白名单的 Windows API/PowerShell/COM 调用;Fake Driver 可覆盖核心自动化能力。
|
||||||
|
|
||||||
|
### P1-04 跨平台目标与当前实现状态需要分开表达
|
||||||
|
|
||||||
|
**确认事实**:当前 `WindowsDriver` 使用 `powershell.exe`、`user32.dll`、Windows UIA、Office COM 等;外部浏览器启动和系统能力也存在 Windows 专属实现。没有看到可用的 `MacDriver` / `LinuxDriver`。
|
||||||
|
|
||||||
|
**建议表述**:项目目标是 Windows/Linux/macOS;当前可交付实现以 Windows 为主。Linux 需先明确 X11/Wayland 边界,macOS 需处理 Accessibility、Screen Recording 等系统授权;Office 自动化三端也不是同一套实现。不要将“接口预留”写成“已支持跨平台”。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. P1/P2:包工程化与质量护栏
|
||||||
|
|
||||||
|
### P1-05 `@rpa/core` 还不是完整的可发布包
|
||||||
|
|
||||||
|
**源码位置**:`packages/rpa-core/package.json`。
|
||||||
|
|
||||||
|
**确认事实**:包名为 `@rpa/core`,`private: true`,`main` / `types` / `exports` 直接指向 TypeScript 源文件;包脚本没有 build、pack 或 exports 产物检查。
|
||||||
|
|
||||||
|
**影响**:在当前 workspace/Bun 环境中可能可以直接消费 TS 源码,但不能据此推断它可被普通 Node.js、生产打包器或独立服务稳定消费。Agent-Employee backend 使用 Bun 的事实仍需在目标仓库实测。
|
||||||
|
|
||||||
|
**整改**:确定宿主采用源码 workspace 依赖还是构建产物依赖。若源码消费,锁定 Bun 版本并建立 Bun test/typecheck/运行测试;若产物消费,补 `tsc/tsup` 构建、`exports` 的 types/import/default 条目、`files` 白名单和 `npm pack --dry-run` 检查。
|
||||||
|
|
||||||
|
### P2-01 依赖护栏是正则扫描,不是完整依赖边界证明
|
||||||
|
|
||||||
|
**源码位置**:`packages/rpa-core/scripts/check-deps.mjs:20-55`。
|
||||||
|
|
||||||
|
**确认事实**:脚本扫描 `.ts/.tsx`,用正则匹配静态 `import/export ... from`;没有覆盖 `import("electron")`、CommonJS `require()`、`createRequire()`、运行时拼接加载、`.mjs/.js` 文件,也没有校验包外部依赖 allowlist。当前脚本执行通过。
|
||||||
|
|
||||||
|
**整改**:保留快速扫描,同时增加 TypeScript AST 扫描;对 `import()`、require、createRequire 和禁用模块路径进行检查;另加 `package.json` 依赖审计和构建后 bundle 检查。脚本通过只能写成“当前规则扫描通过”,不要写成“彻底无 Electron 依赖”。
|
||||||
|
|
||||||
|
### P2-02 Registry 重复注册与元数据覆盖规则不明确
|
||||||
|
|
||||||
|
**源码位置**:`registry.ts:36-41`。
|
||||||
|
|
||||||
|
**确认事实**:同 ID 再次注册会直接覆盖 handler;如果第二次注册没有 meta,旧的 meta 不会删除,可能出现 handler 已更新但 meta 仍是旧值。
|
||||||
|
|
||||||
|
**整改**:定义重复注册策略(拒绝/显式覆盖/版本化);覆盖 handler 时同步处理 meta;注册冲突输出可审计信息。
|
||||||
|
|
||||||
|
### P2-03 `registerAll()` 可能发生部分注册
|
||||||
|
|
||||||
|
**源码位置**:`register-all.ts:49-56`。
|
||||||
|
|
||||||
|
**确认事实**:按传入顺序逐组 import;如果前面几个组已加载、后续遇到未知组才抛错,会留下部分全局注册状态。
|
||||||
|
|
||||||
|
**整改**:先校验全部 group 名称,再执行加载;长期方案采用显式 Registry,支持事务式注册或失败回滚。
|
||||||
|
|
||||||
|
### P2-04 AI 请求缺少调用方取消信号
|
||||||
|
|
||||||
|
**源码位置**:`capabilities/ai-service.ts:39-76`。
|
||||||
|
|
||||||
|
**确认事实**:AI 请求使用 `AbortSignal.timeout(timeoutMs)`,但 `ChatCompletionOptions` 没有调用方 `signal` 字段。交互式请求默认超时 45 秒;UI 拾取取消时不能从该 API 直接中止 HTTP 请求。
|
||||||
|
|
||||||
|
**影响**:界面退出后请求仍可能继续占用网络/服务端资源;连续识别可能造成无用请求并发。
|
||||||
|
|
||||||
|
**整改**:增加 `signal?: AbortSignal`,把用户取消与内部 timeout 合并;新请求开始时按业务策略取消旧请求;对 408/429/5xx 以外的错误明确重试策略。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. 上一版元素拾取报告的回归核验
|
||||||
|
|
||||||
|
以下项目在 2026-11-01 的旧报告中列为高优先级,本次针对最新 ZIP 复核后状态如下:
|
||||||
|
|
||||||
|
| 旧问题 | 最新源码核验 | 当前结论 |
|
||||||
|
|---|---|---|
|
||||||
|
| `desktop-picker.ps1` 未列入 electron-builder `extraResources` | `electron-builder.config.mjs:26-32` 已包含 `resources/desktop-picker.ps1` | 已修复,仍需安装包实测资源路径 |
|
||||||
|
| 候选框排序后用数组下标回填错位 | `desktop-pick.ts:238-253, 596-616` 使用 `candidateId` 识别和回查候选 | 已看到修复,不应再列为当前 Bug |
|
||||||
|
| 视觉坐标执行路径与 UIA 执行路径不一致 | 旧报告指出过此风险;本次未完成所有 selector source 到执行器分发的逐函数回归 | 待专项复核,不在本报告宣称已修复 |
|
||||||
|
| AI 请求超时/取消 | Core AI API 目前有 timeout,但没有外部 AbortSignal | 仍有取消链路缺口,见 P2-04 |
|
||||||
|
|
||||||
|
元素拾取总体仍建议维持“UIA 快路径 → 局部 OCR/CV → 低频 AI 兜底”的方向。普通 hover 不应触发大模型;AI 候选应绑定稳定 candidateId,并最终形成可重新定位的 ElementDescriptor,而不是只保存一次性屏幕坐标。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. 推荐目标架构与接入边界
|
||||||
|
|
||||||
|
```text
|
||||||
|
Agent-Employee workflow-engine
|
||||||
|
|
|
||||||
|
v
|
||||||
|
RPA Adapter / CapabilityRunner
|
||||||
|
- 权限授权、租约、审批、审计
|
||||||
|
- Run/Step 状态映射
|
||||||
|
- 输入输出 schema 适配
|
||||||
|
|
|
||||||
|
v
|
||||||
|
RPA Core Engine (per execution/session)
|
||||||
|
- Flow Validator + Runtime
|
||||||
|
- CapabilityRegistry(建议长期实例级)
|
||||||
|
- RunContext (signal / trace / evidence / variables)
|
||||||
|
|
|
||||||
|
+---- Capability Handlers
|
||||||
|
| - pure Node capabilities
|
||||||
|
| - Windows-only capability allowlist
|
||||||
|
|
|
||||||
|
+---- AutomationDriver (injected)
|
||||||
|
| - WindowsDriver
|
||||||
|
| - LinuxDriver (future)
|
||||||
|
| - MacDriver (future)
|
||||||
|
|
|
||||||
|
+---- HostServices (injected)
|
||||||
|
- Browser adapter
|
||||||
|
- Dialog / DPI / config
|
||||||
|
- Subflow repository
|
||||||
|
- Event / evidence sink
|
||||||
|
```
|
||||||
|
|
||||||
|
**职责原则**:
|
||||||
|
- workflow-engine 是宿主权威 Run/Step 生命周期的候选所有者;RPA Core 提供流程执行算法或节点执行器。最终只能有一个权威状态源。
|
||||||
|
- Capability Gateway 是所有高影响能力的唯一授权入口;Core 不另建一套冲突的权限系统。
|
||||||
|
- computer-control 是桌面观察/定位/动作契约的优先对齐对象;RPA Core 的 Driver 不应与宿主契约长期并行演化。
|
||||||
|
- HostServices/Browser/子流程存储由宿主注入,Core 不直接依赖 Electron BrowserWindow 或应用全局状态。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. 整改顺序与验收门槛
|
||||||
|
|
||||||
|
### 第一阶段:执行安全(接入前)
|
||||||
|
- [ ] P0-01 节点级 AbortSignal 贯通 Runtime → Handler → Driver/PowerShell/HTTP/Browser。
|
||||||
|
- [ ] P0-02 取消状态映射一致,取消不触发 skip/error route/retry。
|
||||||
|
- [ ] P0-03 非幂等能力默认不重试;能力元数据声明幂等性与重试策略。
|
||||||
|
- [ ] P1-01 Graph Validator 覆盖入口、出口、环、悬空边、孤立节点和容器 slot。
|
||||||
|
|
||||||
|
### 第二阶段:依赖闭环(接入前)
|
||||||
|
- [ ] P0-04 明确 Registry 是进程级还是 Engine 级,并更新 API/ADR。
|
||||||
|
- [ ] P0-05 Engine 构造时注入 Driver 与 HostServices。
|
||||||
|
- [ ] P1-02 `registerAll()` 与实际注册作用域一致,重复注册规则明确。
|
||||||
|
- [ ] P1-03 建立 Capability Gateway 适配层和权限元数据映射。
|
||||||
|
|
||||||
|
### 第三阶段:平台与包工程化(联调阶段)
|
||||||
|
- [ ] P1-04 catalog 平台标记修复并增加 schema 校验。
|
||||||
|
- [ ] P1-05 完成 Driver/Windows-only 能力白名单。
|
||||||
|
- [ ] P1-06 选定 Bun 源码消费或构建产物消费方案,执行 Bun typecheck/test/运行验证。
|
||||||
|
- [ ] P2-01 AST 依赖检查覆盖动态 import/require。
|
||||||
|
- [ ] P2-02 AI 取消信号与拾取流程取消联动。
|
||||||
|
|
||||||
|
### 接入验收最低标准
|
||||||
|
1. Core 在目标 Bun 版本下 typecheck/test 通过;不能以 Node 测试代替 Bun 验证。
|
||||||
|
2. Fake Driver 注入成功,核心能力不启动真实 PowerShell。
|
||||||
|
3. 两个 Engine 的依赖/状态隔离行为符合 ADR。
|
||||||
|
4. 所有有副作用能力经过 Capability Gateway,权限拒绝时 handler 零调用。
|
||||||
|
5. 取消/超时/重试测试证明不会重复执行非幂等动作。
|
||||||
|
6. Graph Validator 能阻止非法流程在执行前运行。
|
||||||
|
7. catalog 平台字段通过 schema 校验,平台支持范围与真实实现一致。
|
||||||
|
8. Windows 10/11 实机完成 UIA、浏览器、Office、DPI、多屏幕、安装包资源和取消场景回归。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. 需要与 Agent-Employee 团队共同确认的 ADR
|
||||||
|
|
||||||
|
| ADR | 必须确认的问题 | 建议 |
|
||||||
|
|---|---|---|
|
||||||
|
| Registry 生命周期 | 一个 backend 是否存在多个 Agent/租户/权限域? | 长期采用 Engine 级 Registry;短期单进程固定能力集可临时全局共享 |
|
||||||
|
| Runtime 所有权 | workflow-engine 还是 RPA Core 持有 Run/Step 最终状态? | 宿主持有权威状态,Core 负责执行并回传事件 |
|
||||||
|
| Driver 契约 | RPA Driver 是否直接实现 computer-control? | 对齐宿主契约,避免双份抽象 |
|
||||||
|
| HostServices 作用域 | 一个进程是否会管理多个桌面会话? | 按 Engine/Session 注入,不使用全局 setter |
|
||||||
|
| 能力权限 | 哪些能力需要审批/租约/资源范围? | Capability Gateway 唯一入口 |
|
||||||
|
| 包消费方式 | Bun 直接消费 TS 源码还是 Core 构建后消费? | 以 backend 生产部署方式决定,并建立 CI 证明 |
|
||||||
|
| 平台范围 | 本轮交付是否只要求 Windows? | 当前明确 Windows-first,Linux/macOS 分阶段验收 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9. 审查限制与下一步
|
||||||
|
|
||||||
|
- 本次成功读取了资料库中的最新 ZIP,并对关键文件做静态核验。
|
||||||
|
- 本次执行了 `check-deps.mjs`,结果通过。
|
||||||
|
- ZIP 未包含 `node_modules`,当前审查环境没有安装项目依赖,因此没有执行 Vitest、TypeScript typecheck、Electron build 或 E2E;这些结果均标记为“未验证”,不能写成通过。
|
||||||
|
- 没有 Agent-Employee 目标仓库源码可供本轮逐接口编译对接;其契约映射仍以原迁移指南中记录的接口信息为基础,正式集成前必须在目标仓库核对真实类型定义。
|
||||||
|
- 没有在真实 Windows 10/11 桌面执行 UIA、Office COM、Edge/CDP、多屏/DPI、安装包和 AI 取消链路。
|
||||||
|
|
||||||
|
**最终建议**:先修 P0 的取消/超时/重试和 Engine 依赖注入,再决定 Registry 作用域;不要先把 1016 个能力整体搬入宿主。先用一条小而完整的垂直切片(一个流程、一个浏览器能力、一个桌面能力、一个文件能力)跑通 Gateway → Runtime → Driver → Event/Evidence,再逐批迁移能力。
|
||||||
|
|
||||||
|
|
||||||
|
## 12. 第三轮深挖:Runtime、浏览器执行链与元素拾取
|
||||||
|
|
||||||
|
> 本节是对同一源码快照的进一步静态追踪。这里区分“代码路径已确认”和“运行时表现仍需实机复现”;没有把未执行的 Edge/Office/UIA 测试写成已复现故障。
|
||||||
|
|
||||||
|
### 12.1 P1:BrowserSession 的定位与执行查找器不一致,拾取结果不保证可执行
|
||||||
|
|
||||||
|
**源码证据:**
|
||||||
|
- `src/main/browser/session.ts:61-85`:`selectorSpec()` 接受 `css`、`xpath`、`text`、`aria` 等类型,并可接收拾取器产生的 `css` selector。
|
||||||
|
- `src/main/browser/session.ts:243-291`:`withElementOnce()` 对 XPath 使用 `document.evaluate()`;对 CSS `>>>` 分段逐层进入 `shadowRoot`。
|
||||||
|
- `src/main/browser/session.ts:463-503`:`locateOnce()` 对 `text`、`aria` 有分支,但其余类型统一执行 `document.querySelectorAll(spec.value)`。因此 `xpath` 会被当作 CSS selector;`css` 中的 `>>>` 也会被当作普通 CSS 字符串。
|
||||||
|
- `src/main/browser/session.ts:562-565`:`click()` 首先调用 `locate()`,之后才调用 `withElement()`。
|
||||||
|
- `src/main/browser/session.ts:623-627`:`type()` 同样先调用 `locate()`。
|
||||||
|
- `src/main/browser/session.ts:672-681`:`getText(selector, attribute)` 的属性分支再次直接调用 `document.querySelector(spec.value)`。
|
||||||
|
|
||||||
|
**结论:**
|
||||||
|
1. `xpath://...` 即使能通过 `withElement()` 找到,也会在 `click()` / `type()` 前置的 `locate()` 阶段走 CSS 查询路径。
|
||||||
|
2. 拾取器会生成包含 Shadow DOM 路径分隔符 `>>>` 的 selector;`withElement()` 支持该语法,但 `locate()` 与 `getText(attribute)` 不支持。
|
||||||
|
3. 因此“拾取器生成了 selector”不等于“所有浏览器操作都能消费这个 selector”。这属于明确的实现不一致,不是单纯的测试缺失。
|
||||||
|
|
||||||
|
**整改:**抽出唯一的 `resolveElement(selector, context)`,统一支持 CSS/XPath/Text/ARIA/Shadow DOM;`locate/click/type/getText/getRegion/getElementInfo` 全部复用同一个解析器。不要让每个方法各自拼 DOM 查询表达式。
|
||||||
|
|
||||||
|
**验收:**同一批 fixture 分别覆盖 CSS、XPath、text、ARIA、open Shadow DOM、多层 Shadow DOM;每个 selector 至少通过 locate、click、type、getText、getRegion。测试断言拾取产出的 selector 能被执行器直接消费。
|
||||||
|
|
||||||
|
### 12.2 P1:iframe 未进入浏览器拾取与执行上下文模型
|
||||||
|
|
||||||
|
**源码证据:**
|
||||||
|
- `PICKER_BOOTSTRAP_SOURCE`(`src/main/browser/session.ts:1390` 起)将拾取脚本注入当前页面 `document`,使用 `document.elementFromPoint`、`event.composedPath()` 和 `document.querySelectorAll()`。
|
||||||
|
- `selectorSpec()` / `withElementOnce()` / `locateOnce()` 均以当前 target 的主 `document` 为根,没有 frameId、frame URL、frame path 或 frame-specific execution context。
|
||||||
|
- `getElement()` 生成的 CSS 路径只描述当前 document / ShadowRoot 树,不包含 iframe 边界。
|
||||||
|
|
||||||
|
**结论:**当前代码有 Shadow DOM 的部分穿透设计,但没有 iframe frame tree 的处理。跨域 iframe 内元素不能由顶层页面脚本直接访问;同源 iframe 也没有被当前 picker 主动递归遍历。不能把“支持 Shadow DOM”宣传成“支持 iframe”。
|
||||||
|
|
||||||
|
**整改:**基于 CDP `Page.getFrameTree` 建立 frame registry;为每个可访问 frame 建立 execution context,selector reference 携带 frameId 与 frame 内 selector;跨域 iframe 通过 CDP frame execution context 定位,而不是依赖顶层 JS 的同源访问。对于无法接管的特殊 frame,明确返回 unsupported / inaccessible。
|
||||||
|
|
||||||
|
**验收:**同源 iframe、跨域 iframe、嵌套 iframe、iframe 内 Shadow DOM 分别测试拾取、定位、点击、输入;页面导航和 frame 重建后 frameId 必须刷新。
|
||||||
|
|
||||||
|
### 12.3 P1:外部 CDP 单条命令没有超时和 pending 清理
|
||||||
|
|
||||||
|
**源码证据:**
|
||||||
|
- `packages/rpa-core/src/browser/transport.ts:96-117`:`send()` 将请求写入 `pending`,等待对应 response;没有单次请求 timer / AbortSignal。
|
||||||
|
- 只有 WebSocket close/error 时 `failAllPending()` 才会清理 pending(`transport.ts:76-78, 160-165`)。
|
||||||
|
- `transport.ts:83-90`:`connect()` 用 `Promise.race()` 做连接超时,但超时分支没有清理 timer,也没有关闭已经创建的 WebSocket transport。
|
||||||
|
|
||||||
|
**影响:**CDP 某条命令若既不返回响应、WebSocket 又没有断开,调用会一直 pending;上层 Runtime 的 `withTimeout()` 只能让节点 Promise 超时,不能清掉 transport 内部 pending 项。连接超时后底层 socket 也可能继续存活。
|
||||||
|
|
||||||
|
**整改:**`send(method, params, { timeoutMs, signal })` 为每个 request 建 timer;超时后删除 pending 并 reject;连接超时必须清 timer、关闭 socket;所有 timer 在成功、失败、close 三条路径都清理。超时错误要保留 method、targetId、requestId。
|
||||||
|
|
||||||
|
**验收:**模拟不返回 response、WS 断线、连接握手超时、主动 close;断言 Promise 在期限内结束、pending 数回到 0、socket 不泄漏。
|
||||||
|
|
||||||
|
### 12.4 P1:所谓“跟随前台标签页”实际上取 `/json/list` 的第一项
|
||||||
|
|
||||||
|
**源码证据:**
|
||||||
|
- `src/main/browser/session.ts:205-213` 的 `followForegroundTab()` 调用 `listPageTargets()` 后直接取 `pages[0]`。
|
||||||
|
- `src/main/browser/session.ts:1293-1305` 的 picker `followForegroundTab()` 同样取 `pages[0]`。
|
||||||
|
- `packages/rpa-core/src/browser/external.ts:242-250` 的 `listPageTargets()` 只是按 `/json/list` 返回顺序过滤并映射,没有查询实际窗口焦点或活动 target。
|
||||||
|
- `listTabs()` 的 `active` 字段只是比较 `page.id === this.activeTargetId`(`session.ts:1186-1193`),代表当前 CDP session 绑定页,不代表用户实际看到的前台页。
|
||||||
|
|
||||||
|
**结论:**注释中的“前台标签页”与实现语义不一致。多标签场景下 `/json/list` 第一项不保证是当前活动标签页,可能切错页面,造成操作落在用户看不到的页面。
|
||||||
|
|
||||||
|
**整改:**明确产品语义到底是“用户前台页”还是“流程当前绑定页”。若要求真实前台页,使用可验证的活动 target 机制(并处理 Chromium 版本差异);若只是流程绑定页,删除“前台跟随”逻辑和误导性注释,要求通过 targetId 明确切页。
|
||||||
|
|
||||||
|
**验收:**至少 3 个标签页,交替切换前台页,比较用户可见页、CDP 当前页、RPA active 标记;点击新标签、页面跳转、关闭标签时检查绑定状态。
|
||||||
|
|
||||||
|
### 12.5 P1:AI 桌面拾取取消未贯穿检测和 OCR,存在后台任务继续运行的路径
|
||||||
|
|
||||||
|
**源码证据:**
|
||||||
|
- `src/main/desktop-pick.ts:480-490`:`detection` 先启动截图与 `detectUiElements()`,OCR `recognizePane()` 并行启动;这两个任务都没有接收 AbortSignal。
|
||||||
|
- `desktop-pick.ts:506-528`:取消 watcher 只与 `detection` 做 `Promise.race()`;取消先发生时只结束外层等待,不会停止检测/截图/OCR底层工作。
|
||||||
|
- `desktop-pick.ts:557-589`:OCR 后台任务独立运行并更新 overlay。
|
||||||
|
- `desktop-pick.ts:613-614`:用户确认候选后直接 `await detected.ocrTask`,没有额外硬超时。
|
||||||
|
- `src/main/local-vision/pane-ocr-service.ts:88-112`:`recognizePane()` 等待模型初始化和 `ocr.detect()`,接口没有 signal / timeout。
|
||||||
|
|
||||||
|
**影响:**用户取消时,CPU/GPU 推理或 OCR 仍可能继续;overlay 已销毁时后台检测后续仍可能访问窗口对象;用户确认后 OCR 若卡住,拾取流程可能超过 UI 的 180 秒候选等待期限继续挂起。
|
||||||
|
|
||||||
|
**整改:**为拾取建立统一 `AbortController`,贯通截图、检测、OCR、overlay 更新和 UIA enrichment;第三方模型不能中止时,至少在取消后丢弃结果、禁止更新 UI,并限制并发任务数。OCR 结果不应成为确认回填的无限等待条件:设定独立预算,超时允许以 AI 框 + UIA 结果回填,并将 OCR 标记为未完成。
|
||||||
|
|
||||||
|
**验收:**分别在截图、模型加载、检测、OCR、UIA enrichment 阶段取消;断言取消后不再更新 UI、不再启动后续步骤、overlay/hotkey/临时图片均清理;OCR 超时仍能按明确降级策略完成或失败。
|
||||||
|
|
||||||
|
### 12.6 P2:浏览器选择器属性读取路径存在二次解析偏差
|
||||||
|
|
||||||
|
`getText(selector, attribute)` 先经 `locate()` 成功,再在属性分支中直接 `document.querySelector(spec.value)`。即使 locate 未来补齐 XPath / Shadow DOM,属性分支仍会绕过统一解析器。应与 12.1 一并修复,不要单独打补丁。
|
||||||
|
|
||||||
|
### 12.7 当前测试证据不足:浏览器底层实现没有被集成测试覆盖
|
||||||
|
|
||||||
|
**源码核验:**当前 ZIP 中存在 `src/main/capabilities/browser*.test.ts`,但相关测试主要验证能力 handler 向 `BrowserLike` 转发参数及 mock 行为;未发现针对 `src/main/browser/session.ts` / `packages/rpa-core/src/browser/transport.ts` 的真实 DOM / CDP transport 单元测试。`browser-common.test.ts` 对 XPath 的测试验证的是把 `xpath://button` 传给 mock 的 `getElement`,并不验证 BrowserSession 能否解析 XPath。
|
||||||
|
|
||||||
|
**建议新增测试层:**
|
||||||
|
1. `selector-resolver.test.ts`:纯 DOM fixture,统一覆盖 CSS/XPath/Text/ARIA/Shadow DOM/iframe。
|
||||||
|
2. `cdp-transport.test.ts`:mock WebSocket,覆盖乱序响应、超时、close、断线和 pending 清理。
|
||||||
|
3. `browser-session.integration.test.ts`:使用真实 Chromium/Electron fixture 验证拾取 selector 到 click/type 的端到端一致性。
|
||||||
|
4. `browser-multitab.test.ts`:多 target 切换、前台页识别、页面导航与 target 生命周期。
|
||||||
|
5. `desktop-picker-cancel.test.ts`:各阶段取消、OCR 超时、overlay/hotkey/temp file 清理。
|
||||||
|
|
||||||
|
### 12.8 本轮深挖结论与修复顺序
|
||||||
|
|
||||||
|
| 顺序 | 事项 | 等级 | 建议处理 |
|
||||||
|
|---|---|---|---|
|
||||||
|
| 1 | 统一 selector resolver,修复 XPath / Shadow DOM 的 locate 执行不一致 | P1 | 先修,直接影响浏览器自动化成功率 |
|
||||||
|
| 2 | CDP request timeout、AbortSignal、pending 清理 | P1 | 先修,避免浏览器节点永久挂起 |
|
||||||
|
| 3 | iframe/frameId 支持边界与实现 | P1 | 先明确 MVP 支持范围,再实现 |
|
||||||
|
| 4 | 修正前台标签语义,不能把 pages[0] 当 active | P1 | 多标签演示前修复 |
|
||||||
|
| 5 | AI 拾取取消贯穿检测/OCR/资源清理 | P1 | 桌面拾取稳定性验收前修复 |
|
||||||
|
| 6 | 增加 BrowserSession/CDP/Picker 真实集成测试 | P1 | 修复后必须补证据 |
|
||||||
|
|
||||||
|
**本轮仍未执行:**Vitest、TypeScript typecheck、Electron build、真实 Edge/Chrome CDP 测试、Windows UIA/Office/DPI 实机测试。上述是源码级缺陷/风险定位,不等于已经在用户机器上复现。
|
||||||
87
docs/dev-conventions.md
Normal file
87
docs/dev-conventions.md
Normal file
@ -0,0 +1,87 @@
|
|||||||
|
# RPA Studio 开发规范(v1 · 面向 rpa-core 抽离)
|
||||||
|
|
||||||
|
> 生效日期:2026-11-04。
|
||||||
|
> 背景:本平台继续作为日常开发载体;所有新代码必须同时满足"当前可用"和"将来可直接搬入
|
||||||
|
> `packages/rpa-core`(最终迁入 Agent-Employee)"。迁移路线与任务清单见
|
||||||
|
> `docs/rpa-core-refactor-tasks.md`,本规范是它的日常执行约束。
|
||||||
|
|
||||||
|
## 核心思想
|
||||||
|
|
||||||
|
**迁移不是将来的改造,而是现在每次写代码时的选择。** 遵守本规范写的能力,抽包时是搬运;
|
||||||
|
违反规范写的能力,抽包时是返工。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 一、分层归属(硬性规则)
|
||||||
|
|
||||||
|
| 层 | 目录 | 判别标准 | 未来去向 |
|
||||||
|
|---|---|---|---|
|
||||||
|
| **L1 包内层** | `src/main/engine/`、`src/main/capabilities/` 大部分、`src/main/browser/external.ts`、`transport.ts` | 不依赖 Electron 窗口即可运行 | rpa-core 包 |
|
||||||
|
| **L2 宿主层** | `src/main/index.ts`、`overlay`、`desktop-pick`、`vision-lab`、`fixture/`、`local-vision/`、`flows`、`run/`、`settings`、`logger`、`browser/session.ts` 内置浏览器部分 | 需要 Electron 窗口/应用生命周期 | 宿主适配层 |
|
||||||
|
|
||||||
|
**判别口诀**:这个能力在没有窗口的 CLI 里能跑吗?能 → L1;不能 → L2。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 二、L1 层禁令(现有代码不追溯,新代码必须遵守)
|
||||||
|
|
||||||
|
1. **禁止 `import ... from "electron"`**
|
||||||
|
需要剪贴板/对话框/截屏/屏幕信息时:先看 Node 等价物(fs/os/child_process),
|
||||||
|
确实需要宿主能力时在 RunContext 或参数中注入,并在代码处注释 `// HostBridge 候选`。
|
||||||
|
2. **禁止横向依赖宿主模块**
|
||||||
|
L1 内禁止 `import "../settings"`、`"../logger"`、`"../flows"` 等。配置走参数/ctx,日志走 `ctx.log`。
|
||||||
|
3. **禁止私起 PowerShell 进程**
|
||||||
|
所有 PowerShell 调用必须使用 `capabilities/desktop.ts` 导出的 `powershell()` / `psString()`。
|
||||||
|
禁止自己 `execFile("powershell.exe", ...)`。原因:将来统一切换为常驻 worker(任务 3.5)。
|
||||||
|
4. **禁止新增模块级可变全局状态**
|
||||||
|
不写模块顶层 `const cache = new Map()` / `let globalXxx`。需要跨调用状态时挂在显式 store
|
||||||
|
或 RunContext 上(目标形态:`createEngine()` 实例持有)。
|
||||||
|
5. **禁止把 PowerShell 逻辑写成不可拆的内联串**
|
||||||
|
复杂脚本提取为独立函数/常量,便于将来搬进 WindowsDriver。
|
||||||
|
|
||||||
|
## 三、能力编写规范(tool 友好,为 Agent-Employee 准备)
|
||||||
|
|
||||||
|
1. **结构化进出**:参数从 `args` 对象取具名字段,返回结构化对象或原始值;
|
||||||
|
禁止返回"为 UI 拼接的字符串"(渲染层需要展示时自行格式化)。
|
||||||
|
2. **必有超时**:涉及外部交互(进程/网络/窗口等待)的能力必须有 timeout 参数和默认值。
|
||||||
|
3. **错误契约**:失败抛 `RuntimeError(code, message, { recommendation })`;
|
||||||
|
code 用稳定英文枚举(必须登记在 `packages/rpa-core/src/engine/error-codes.ts`
|
||||||
|
的 `RUNTIME_ERROR_CODES`,任务 4.3 起机器可读),message 中文,recommendation 给可执行建议。禁止裸 throw string。
|
||||||
|
4. **平台标记(单一源,三端优先 win)**:平台归属只有一处权威——注册分组表
|
||||||
|
(迁入前固化为 `register-all.ts` 的 `PLATFORM_GROUPS`,现阶段按分组在 catalog 构建时统一标 `win32`)。
|
||||||
|
能力文件**不得**自带平台标记、**不得**散写 `process.platform` 判断(例外白名单单独维护);
|
||||||
|
`meta.platforms` 与 catalog 的 `platformTag` 都从分组表派生,一致性由测试守卫。
|
||||||
|
现阶段所有能力只标 `win32`,`darwin`/`linux` 仅在类型枚举里占位,**不填任何数据**;
|
||||||
|
将来支持新平台 = 新增分组 + 表里加一行,不做别的。
|
||||||
|
5. **可测试**:每个能力有同名 `*.test.ts`(vitest),与实现同目录。
|
||||||
|
6. **结构化 meta(任务 4.1 起,新能力强制)**:`register(ids, handler, meta)` 第三参
|
||||||
|
必填 `CapabilityMeta`(summary / params / returns / timeoutMs / errorCodes /
|
||||||
|
needsScreenshot / platforms)。老能力逐步补齐,review 时放行不带 meta 的旧文件改动。
|
||||||
|
|
||||||
|
## 四、新能力验收 checklist
|
||||||
|
|
||||||
|
- [ ] 归属判定正确(L1/L2 口诀)
|
||||||
|
- [ ] 无 electron / 宿主模块 import(L1 时)
|
||||||
|
- [ ] PowerShell 只经 `powershell()` 助手(如需要)
|
||||||
|
- [ ] 无新增模块级全局状态
|
||||||
|
- [ ] 结构化参数/返回 + 超时 + RuntimeError 错误契约
|
||||||
|
- [ ] catalog 注册完整(含 platforms 标记)
|
||||||
|
- [ ] 有测试,`npm run test` + `npm run typecheck` 全绿
|
||||||
|
|
||||||
|
## 五、渐进收紧节奏
|
||||||
|
|
||||||
|
规范随改造阶段自动收紧,不必一步到位:
|
||||||
|
|
||||||
|
| 里程碑(见任务清单) | 规范变化 |
|
||||||
|
|---|---|
|
||||||
|
| 现在 | 本文第二节禁令生效(靠 review 执行) |
|
||||||
|
| 阶段 2 完成 | L1 禁 electron 升级为 ESLint no-restricted-imports 强制 |
|
||||||
|
| 阶段 3 完成 | `powershell()` 收口进 `driver/windows/`,L1 禁止直接引用 |
|
||||||
|
| 阶段 4 完成 | 无 schema 的新能力不允许进 catalog |
|
||||||
|
|
||||||
|
## 六、明确不动的部分
|
||||||
|
|
||||||
|
- 现有 77 处 PowerShell 调用点:**不做预防性重构**,统一等阶段 3 的 driver 抽象一次性处理。
|
||||||
|
之前只修 bug、只加能力,不重写。
|
||||||
|
- 渲染层(React)按产品需求正常开发,不为其迁移性投资。
|
||||||
|
- `local-vision/`、`overlay/` 等宿主层模块正常开发,不受 L1 禁令约束。
|
||||||
79
docs/next-work-plan.md
Normal file
79
docs/next-work-plan.md
Normal file
@ -0,0 +1,79 @@
|
|||||||
|
# RPA 迁移后续工作完整清单(2026-11-06 定稿)
|
||||||
|
|
||||||
|
> 前提:阶段 0 前置作业 + 阶段 1 收口 + 本仓库代码活已 100% 完成(含 C# actuate 原型、A++ 字段、NodeState 两态)。本清单是"接下来做什么"的唯一权威账本,按可动性分四档。更新规则:完成一项就划掉并记提交号;新工作先进"待评估"再入档。
|
||||||
|
|
||||||
|
## 三个"不知道"的设计提案(已定稿,发对面确认即可,不再阻塞)
|
||||||
|
|
||||||
|
### 提案 1(原 Q1):平台字段怎么表达
|
||||||
|
|
||||||
|
对面 plugin-contract 的 `CapabilityDefinition` 增可选字段 `platforms?: readonly ("win32"|"darwin"|"linux")[]`:
|
||||||
|
|
||||||
|
- **缺省 = 全平台**——旧插件清单不加字段,零破坏;
|
||||||
|
- `assertPluginManifest` 校验值域,未知平台值直接拒绝;
|
||||||
|
- plugin-worker 认领动作时按宿主 `process.platform` 过滤,不匹配的能力对宿主不可见;
|
||||||
|
- **RPA 插件清单不手工维护**:rpa-adapter 从 rpa-core catalog 的 `platformTag` 自动生成(687 win32 / 326 null 单一事实源)。
|
||||||
|
|
||||||
|
### 提案 2(原 Q2):utilityExtended 权限归属
|
||||||
|
|
||||||
|
utilityExtended 在对面源码中无同名物(纯 RPA 侧能力域)。**映射到对面 13 项 PermissionScope 的 `desktop.process`**:
|
||||||
|
|
||||||
|
- 理由:RPA 命令执行发生在用户电脑 Electron 宿主进程内,"宿主即桌面进程",与 `desktop.process` 语义吻合;
|
||||||
|
- 该域全部标 `highImpact + idempotencyRequired`(走审批,与 ADR-R1/R6 一致);
|
||||||
|
- 备选:若对面坚持 desktop.process 严格指"桌面进程管理",则新增 `desktop.command` scope(对面 plugin-contract 加一个 union 成员 + Set 一行)。
|
||||||
|
|
||||||
|
### 提案 3:晨间助手截图时机 —— **已搁置(2026-11-06 用户拍板,不再作为工作项)**
|
||||||
|
|
||||||
|
不设专门时机,若未来重启则全挂既有证据节奏:observe 基线 + 人审弹窗前证据 + actuate 动作后自动双证据。当前不要求对面任何配合。
|
||||||
|
|
||||||
|
### 提案 4:rpa.browser 备案单(发给对面即可登记)
|
||||||
|
|
||||||
|
| 项 | 值 |
|
||||||
|
|---|---|
|
||||||
|
| pluginId | `rpa.browser` |
|
||||||
|
| capabilities | `rpa.flow.execute`(RPA 流程整体入口,方案 A)|
|
||||||
|
| runner | 双 runner:对面 PlaywrightBrowserRunner(AI 智能体场景)+ 我方 CDP 接管(RPA 场景主力,ADR-R4)|
|
||||||
|
| PermissionScope | `browser.domain`(白名单复用对面 BrowserContextPolicy 三重复检)+ `filesystem.download`(下载类流程)|
|
||||||
|
| manifest 类型 | browser 能力按对面现行 browser-plugin 口径(域名白名单强校验)|
|
||||||
|
| 证据 | snapshot/截图走对面证据体系(SHA-256 落盘),不自建 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 第一档:本周就能做(不依赖对面)
|
||||||
|
|
||||||
|
- [ ] **W1 发提案包**:上面 4 个提案 + 6 项 ADR 结论,一条消息发对面负责人,等书面确认(Q1/Q2 提案已写入 V2 §10)
|
||||||
|
- [x] ~~W2 C# actuate 合入对面主干~~:**已完成(2026-11-06,对面 b646d4d9)**——三新文件+Program.cs 分发改造+RunnerResponse 可选字段;dotnet build 0w0e;零桌面冒烟 5 例全过(policy_invalid/desktop_session_mismatch/window_not_found/observe 回归/request_invalid);document-facts 刷新 10 区块 factRevision=2026-11-06.2;ci-gate fast 仅余 opencode fork 漂移旧账
|
||||||
|
- [x] ~~W3 rpa.browser 侧准备~~:**已完成(2026-11-06,本仓库 08e798c + 对面 440f7f4f)**——①transport 单命令超时(默认 30s,稳定码 timeout,pending 不再泄漏);②selectorOf 三份重复定义收敛为 src/main/browser/selector.ts(键并集补 uiWindowSelector;browser-common 位置感知版有意保留);宿主 49/49+引擎 32/32 全绿
|
||||||
|
|
||||||
|
## 第二档:等对面书面确认后立即动(预计 1-2 天内解锁)
|
||||||
|
|
||||||
|
- [x] ~~W4 git subtree 迁包~~:**已完成(2026-11-06,对面 8085ff47+4e345304)**——rpa-core → `code/backend/packages/rpa-core`(@agent-worker/rpa-core),测试按对面规范移 test/,48/48 全绿;feat/rpa-core-migration 共 5 笔提交(含 attempt 831e25f9、actuate b646d4d9、transport 440f7f4f)已推远端,待主控审核
|
||||||
|
- [ ] **W5 rpa-adapter 骨架**(对面仓库新建):CapabilityRunner(gateway.execute → ref→componentId → RunContext → handler)+ 权限映射表(34 域 × 13 scope,utilityExtended→desktop.process)+ 状态映射直接 import 本方 `toStepStatus()` + 719 能力"未经 authorize 不得触达 handler"全量冒烟
|
||||||
|
- [ ] **W6 platform 字段实现**(对面 plugin-contract):按提案 1,registry 校验 + worker 过滤 + 契约测试
|
||||||
|
|
||||||
|
## 第三档:阶段 3+(前置:W4/W5 完成)
|
||||||
|
|
||||||
|
- [ ] **W7 垂直切片**:1 文件能力 + 1 HTTP 能力全链(create/start → authorize → adapter → handler → 事件/证据回传 → 中途取消);Run 终态与 RunResult 一致性断言
|
||||||
|
- [ ] **W8 A++ 检查点最小验证**:崩溃 → resumeToken 重入 → 幂等键去重(`checkpointId`/`idempotencyKey` 已定稿,只差实现)
|
||||||
|
- [ ] **W9 双 ADR 落地**:R4 浏览器(W3 产物接对面白名单)+ R3 桌面(W2 产物接 TS 宿主适配器——actuate 前先 observe 解析精确标题)
|
||||||
|
- [ ] **W10 TS 宿主 actuate 适配器**:native-observer.ts 扩展(请求侧精确标题解析、响应侧证据逐项复核、宿主超时 ≥15s)
|
||||||
|
|
||||||
|
## 第四档:规模化与收尾(阶段 5/6)
|
||||||
|
|
||||||
|
- [ ] **W11 34 域分批接入**:files/data/http/ftp → office/excel/word → desktop/utility 高影响(末批 + 审批矩阵抽查)
|
||||||
|
- [ ] **W12 ocr-smoke 素材治理**(迁移时**必做**,非可选):素材入库 `test-fixtures/ocr/` + evidenceDir 改 `os.tmpdir()`——硬编码 `D:/office-test2` 在对面机器必然失效
|
||||||
|
- [ ] **W13 平台扩展**:Linux/macOS 驱动按 ADR-R3 原生通道(阶段 6)
|
||||||
|
- [ ] **W14 demo 侧冻结**:本 demo 仓库转入只读维护
|
||||||
|
|
||||||
|
## 挂账小项(低优先)
|
||||||
|
|
||||||
|
- [x] ~~晨间助手截图时机~~:已搁置(2026-11-06 用户拍板),见提案 3 说明
|
||||||
|
- [ ] `.workbuddy/tmp/actuate-build` 编译沙盒:留作 W2 合入前复验环境,合入后可删
|
||||||
|
|
||||||
|
## 里程碑对照
|
||||||
|
|
||||||
|
| 里程碑 | 依赖 | 预计解锁 |
|
||||||
|
|---|---|---|
|
||||||
|
| Q1/Q2/备案 书面确认 | W1 | 第二档全部 |
|
||||||
|
| 对面仓库可提交 | W1 | W2/W4/W5/W6 |
|
||||||
|
| 垂直切片全链绿 | W4-W7 | 阶段 4(W8-W10)|
|
||||||
|
| 高影响域过审批矩阵 | W11 | 阶段 5 完成即"接入就绪" |
|
||||||
56
docs/pr-feat-rpa-core-migration.md
Normal file
56
docs/pr-feat-rpa-core-migration.md
Normal file
@ -0,0 +1,56 @@
|
|||||||
|
# PR 草稿:feat/rpa-core-migration(线4·RPA插件迁入)
|
||||||
|
|
||||||
|
> 用途:对面仓库开 PR 时直接粘贴。按对面 `.github/PULL_REQUEST_TEMPLATE.md`(699df63d 版)字段填写,同时补齐迁入提交 8085ff47 单行提交说明未承载的细节。
|
||||||
|
|
||||||
|
## 基本信息
|
||||||
|
|
||||||
|
- 分支:`feat/rpa-core-migration`(未直接向 main 提交/合并)
|
||||||
|
- 基线 SHA:`1a56a084`(origin/main 最新,直系子提交,无分叉)
|
||||||
|
- 最终提交 SHA:`4e345304`
|
||||||
|
- 关联需求/ADR/待办:线4·RPA融合第一步——迁入 RPA 核心包;设计依据见自仓库 `docs/rpa-core-migration-guide-v2.md`
|
||||||
|
- 前置 PR/任务:无(独立迁入)
|
||||||
|
- 后置 PR/任务:`rpa-adapter` 适配层(线4,depends-on: 本 PR;待 Q5 配置面/审批上游确认后开工)
|
||||||
|
- 共享入口 owner:`code/backend/package.json` workspace 清单(本 PR 未改它,仅 bun.lock 新增依赖锁);catalog 资源目录 `code/backend/resources/` owner=线4
|
||||||
|
|
||||||
|
## 代码逻辑
|
||||||
|
|
||||||
|
- 输入如何进入:RunRequest(流程 JSON + 输入参数)→ `createEngine` → `registerAll` 能力注册表 → 引擎逐节点执行。能力覆盖桌面(Windows 原生驱动)/浏览器/Excel/PDF/HTTP/文件/屏幕等,注册表 700+ 条目,零 Electron 依赖。
|
||||||
|
- 状态如何变化:NodeState.status 流转(idle/pending/running/succeeded/failed/skipped/waiting_user/unknown);RunEvent 事件流携带 `checkpointId`/`idempotencyKey`(=runId:nodeId)/`resumable`/`executor` 审计字段。
|
||||||
|
- 谁最终判权:本包不做权限判定。执行授权由宿主下发(对面 gateway 租约 + 每节点单独授权,粒度拍板已落档);包内仅做 capabilityVersion 三段 semver 校验。
|
||||||
|
- 数据如何保存/回读:包内无独立落库;检查点与运行记录由宿主持久化,包只产出事件与状态。
|
||||||
|
- 失败/重试/取消/并发/撤权:节点级稳定错误码;幂等键 runId:nodeId 跨重入稳定,支持检查点续跑(resumable 门控:idempotent===false 不作锚点);撤权由宿主停止派发,包内无自主重试循环。
|
||||||
|
- 本次明确不包含:rpa-adapter、AI 调度、双端 executor 路由落地、管理端/客户端 UI、Bun 冒烟(决策10:延期到接宿主时)。
|
||||||
|
- 触及路径:`code/backend/packages/rpa-core/**`(新增 150 文件:src 128 + test 44 等)、`code/backend/resources/`(catalog.json / catalog-overrides.json / capability-duplicates.json 伴生资源)、`code/backend/bun.lock`、`项目资料/开发方案/当前事实.json`。
|
||||||
|
- 明确不触及路径:`code/frontend/**`、Java 服务端、Flyway 迁移、权限目录、OpenAPI/生成器、部署 Compose。
|
||||||
|
- 重复实现检索:`git log origin/main -- code/backend/packages`(无既有 RPA 包);`rg -n "rpa-core|createEngine|registerAll" code`(无重复实现)。复用结论:唯一 owner=线4,无同事并行认领。
|
||||||
|
|
||||||
|
## 契约与影响
|
||||||
|
|
||||||
|
- SQL/迁移/权限/配置:无 SQL/迁移/权限变化;新增 workspace 包 `@agent-worker/rpa-core` 及 bun.lock 条目。
|
||||||
|
- 文档事实:`当前事实.json` 已同提交刷新,`document-facts --check` 通过(代码/事实/文档同提交原则已满足)。
|
||||||
|
- 回滚方式:revert 两个提交即可整包移除;无迁移、无生成物、无跨包引用残留。
|
||||||
|
- 依赖的迁移/制品/运行版本:**bun 1.4.3**(1.4.4 的 patchedDependencies 打不上 photon-node 补丁,与团队锁定版本一致,不是新要求)。
|
||||||
|
|
||||||
|
## 验证证据
|
||||||
|
|
||||||
|
- `bun test`(backend workspace):48/48 全绿(engine + excel + pdf + platform-groups + register-all + compat-audit)。
|
||||||
|
- `node code/tooling/document-facts.mjs --check`:通过。
|
||||||
|
- `node code/tooling/ci-gate.mjs --mode fast`:通过。
|
||||||
|
- `node code/tooling/check-workspace`:产品源码树测试文件违规清零。
|
||||||
|
- 真实环境证据:本 PR 为代码迁入,桌面能力(Windows 原生输入/PrintWindow)需真实 Windows 宿主验收,属后置 rpa-adapter 联调范围;CI 覆盖注册表/catalog/引擎层。
|
||||||
|
- 冲突解决记录:无冲突(基于 origin/main 最新直系提交)。
|
||||||
|
|
||||||
|
## 未完成与风险
|
||||||
|
|
||||||
|
- opencode fork 快照漂移:main 既有问题(ci-gate fast 模式旧账),与本 PR 无关,不阻塞审查。
|
||||||
|
- Windows 桌面能力实机验证未做:随 rpa-adapter 联调补齐。
|
||||||
|
- 迁入提交 8085ff47 提交说明仅一行(早于 699df63d 模板新规),细节以本 PR 描述为准;第二笔 4e345304 已按新模板填写。
|
||||||
|
|
||||||
|
## 提交清单确认
|
||||||
|
|
||||||
|
- [x] 已从最新 origin/main 同步(1a56a084)
|
||||||
|
- [x] 未直接向 main 提交或合并
|
||||||
|
- [x] 本 PR 只包含一个可审查的纵切目标(RPA 核心包迁入+规范化)
|
||||||
|
- [x] 已声明前置/后置任务、共享 owner 和依赖顺序
|
||||||
|
- [x] 没有与其他 PR 重复实现;共享入口只有一个 owner
|
||||||
|
- [x] 已按第五条线工作流完成自审,等待主控审核和合并
|
||||||
66
docs/rpa-core-migration-guide-v2-diff.md
Normal file
66
docs/rpa-core-migration-guide-v2-diff.md
Normal file
@ -0,0 +1,66 @@
|
|||||||
|
# 迁入指南 V1 → V2 差异清单(含审查报告修正记录)
|
||||||
|
|
||||||
|
> 配套文档:`docs/rpa-core-migration-guide-v2.md`(V2 正文)。
|
||||||
|
> 本文记录三件事:① V1 各断言在真实源码下的判定;② 审查报告(docs/RPA-Review-Full.md)与真实源码不一致处的修正及理由;③ V2 新增内容清单。
|
||||||
|
> 核查基线:2026-11-06 双侧真实工作区源码。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 一、V1(rpa-core-migration-guide.md)逐项判定
|
||||||
|
|
||||||
|
分类口径:**保留** / **需要修改** / **过时或不再适用** / **尚未实现(V1 写成了现状或反之)**。
|
||||||
|
|
||||||
|
| # | V1 断言/内容 | 判定 | 真实源码依据(2026-11-06) |
|
||||||
|
|---|---|---|---|
|
||||||
|
| 1 | 包现状:719 能力、34 域、宿主桥 4 方法、五域 Driver | **保留** | 实测 capabilityCount()=719;DOMAINS/PLATFORM_GROUPS 34 域;host-bridge.ts 仍 4 方法+SubflowSource 2 方法 |
|
||||||
|
| 2 | "完全不依赖 Electron,check:deps 谁加就报错" | **过时(不实)** | vision.ts:18,66,432 + cv-vision.ts:51,213 共 5 处动态 `import("electron")`;check-deps.mjs:20 正则 `import\s+` 对 `import("electron")` 零匹配(实测),护栏从未能拦住 |
|
||||||
|
| 3 | "npm run rpa-cli 证明能独立跑" | **需要修改** | rpa-cli.ts:25 仍有副作用全量 import、:53-59 全局 getHandler 直调,只能证明能力可跑,不能证明注入/隔离/无 Electron |
|
||||||
|
| 4 | 第 -1 步:"对面 7 个契约包每个只有 1-9 个声明文件,都是空壳" | **过时(核心叙事失效)** | workflow-engine 25 文件完整实现;capability-gateway 321 行实现;browser-plugin 已接 playwright-core(358 行 Runner);windows-plugin 新增 native-observer.ts(292 行);runtime-governance 有 readiness 投影;windows-runner 17 文件(C# FlaUI observe-only)。仍空壳的只剩 computer-control Runner 与 WindowsRunner.click/type |
|
||||||
|
| 5 | "迁入本质 = 拿 rpa-core 实现去填对面的契约" | **需要修改** | 改为"接底座 + 让路":不重复造状态机/浏览器/UIA 观察,RPA 补的是执行半边(WindowsRunner.click/type、CDP 接管场景)与 719 能力规模 |
|
||||||
|
| 6 | 第 0 步作业 #2"重新生成 catalog 让 platformTag 生效(当时没生效)" | **已完成(V1 写成待办)** | 2026-11-05 已重生成:win32=687 / null=326;残留 3 个数字 1 脏值待清洗 |
|
||||||
|
| 7 | 第 0 步作业 #1 Bun 冒烟 | **尚未实现** | bun 未安装,未跑过任何 Bun test |
|
||||||
|
| 8 | 第 0 步作业 #3 注册方式决策 | **需要修改** | 并入 ADR-R1(全局表 vs engine-scoped);rpa-cli 仍用旧方式 |
|
||||||
|
| 9 | 第 1 步:搬包位置 `code/backend/packages/rpa-core`、改名 `@agent-worker/rpa-core`、git subtree、demo 冻结 | **保留** | backend package.json workspaces 含 `packages/*`、全包 `@agent-worker/*` 前缀、`workspace:*` 源码消费——核实无误 |
|
||||||
|
| 10 | 第 2 步 A:capability-gateway"对照后改,不自建 allowlist" | **需要修改** | gateway 已实现(authorize/execute/租约 60s/幂等/高影响审批),工作是写 adapter 而非"对照设计" |
|
||||||
|
| 11 | 第 2 步 A:workflow-engine"状态模型对齐" | **需要修改** | 对面状态机已实现并稳定(含 waiting_approval/waiting_user/unknown 态);RPA 侧缺 waiting_user/unknown;集成形态二选一(ADR-R2) |
|
||||||
|
| 12 | 第 2 步 A:computer-control/windows-plugin "五域 driver 与契约对照" | **需要修改** | 契约依旧(Runner 无实现),但出现新硬约束:对面安全模型禁止任意 PowerShell(native-observer.ts 约束注释 + *.exe 白名单)→ ADR-R3 |
|
||||||
|
| 13 | 第 2 步 B:内置浏览器"最大变数,对面还没定型" | **过时(变数已收敛)** | PlaywrightBrowserRunner 已实现(白名单三重检+证据);变数变为"双路径取舍"(ADR-R4) |
|
||||||
|
| 14 | 第 2 步 B:宿主桥 4 方法 + SubflowSource 2 方法注入 | **保留**(落点补充) | 对面 desktop 主进程无现成宿主桥,但有等价物(identity-login-window/execution-context/zoom.ts);新增 ScreenCaptureHost 注入面(vision 迁移所需) |
|
||||||
|
| 15 | 第 2 步 B:"AI 能力接对面模型网关" | **保留并具体化** | model-gateway 已实现(stream + ModelControlPlane 每次调用授权);RPA ai-service 仍读 HostBridge.loadAiConfig,改造点明确 |
|
||||||
|
| 16 | 第 3 步 7 条验收 | **需要修改** | 扩充为 10 条总验收(加 gateway 零调用、取消传播、双 Engine 隔离、Graph Validator、纯 Node 加载) |
|
||||||
|
| 17 | 三端计划"搬家前 5 件小事" | **已完成(V1 写成待办)** | ① PLATFORM_GROUPS 单一源+守卫测试 ✅ ② 驱动自动选+platform_unsupported ✅ ③ COM 分组核查 ✅ ④ 5 个测试补 skipIf ✅ ⑤ driver/platform.ts 收拢 ✅(2026-11-05 落地) |
|
||||||
|
| 18 | 三端计划"搬家之后按平台写驱动" | **保留**(补约束) | 驱动实现统一走 ADR-R3 原生通道;Linux 容器不能跑桌面 Runner(对面 README 约束) |
|
||||||
|
| 19 | 附 10 决策 #2"保留全局注册表"、#9"注册双入口过渡" | **需要修改(重开 ADR)** | 与对面多租户/插件模型冲突,重开 ADR-R1;其余 8 个决策维持 |
|
||||||
|
| 20 | "浏览器:外部浏览器(CDP)进包,内置浏览器留宿主"(决策 5) | **保留**(补 ADR) | 仍成立,但需 ADR-R4 明确与 Playwright 的关系 |
|
||||||
|
|
||||||
|
## 二、审查报告(RPA-Review-Full.md)与真实源码不一致的修正
|
||||||
|
|
||||||
|
以真实源码为准,逐条记录修正理由:
|
||||||
|
|
||||||
|
| # | 审查报告断言 | 真实源码(2026-11-06) | 修正理由 |
|
||||||
|
|---|---|---|---|
|
||||||
|
| 1 | §3 P1-NEW-03 / 附录 P0-03:Engine 完全无注入,`driver` 固定 `new WindowsDriver()` | **部分过时**:driver 已改惰性 getter,非 win32 抛 platform_unsupported(engine.ts:47-61 + error-codes.ts:43 + engine-driver.test.ts) | 报告基于 11-06 前的 ZIP 快照;此后落地了"驱动按平台自动选择"。但"createEngine() 无注入参数"这一半仍属实 |
|
||||||
|
| 2 | §8.3 / 附录 P1-01:catalog 1016 项里 1013 项 platformTag=null、3 项数字 1,"平台标记从未生效" | **部分过时**:现为 win32=687 / null=326 / 数字 1=3 | PLATFORM_GROUPS 单一源落地后重生成过 catalog。"3 个数字 1 脏值透传(build-catalog.mjs:246 对 truthy 不清洗)"仍属实,保留为待办 |
|
||||||
|
| 3 | §12.4:"跟随前台标签页无条件取 pages[0]" | **需限定范围**:followForegroundTab 本身仍取 pages[0](session.ts:211),但 withElementOnce/locateOnce 已改两段式(当前页优先、失败才跟随,:296-307/:518-523);evaluate 默认 follow=true 仍会先切 | 该缺陷的暴露面已从"所有元素操作"缩小到"跟随函数本身+evaluate 默认路径",严重度下调,但未根除 |
|
||||||
|
| 4 | §6 P0 行"Engine 无 Driver 注入;能力使用 defaultEngine" | **属实且更严重**:desktop.ts:9 在模块加载时捕获 `defaultEngine.driver`,非 win32 上 import 即抛错(比报告写的"使用时才碰到"更早暴露) | 复核确认,问题加重一档 |
|
||||||
|
| 5 | §3 P1-NEW-02:包依赖缺 @napi-rs/canvas/pdfjs-dist/@cantoo/pdf-lib | **属实(复核确认)**:rpa-core/package.json:17-32 仍缺这 3 项,main/exports 直指 src | 无修正,维持 P1 |
|
||||||
|
| 6 | §12.1:locate 与 withElement 对 xpath/Shadow DOM 支持不一致 | **属实(复核确认)**:locateOnce :465-517 仅 text/aria/else 三分支 | 无修正,维持 P1 |
|
||||||
|
| 7 | §12.3:CDP send() 无单命令超时/AbortSignal | **属实(复核确认)**:transport.ts:96-118 | 无修正,维持 P1 |
|
||||||
|
| 8 | 附录 P2-04:AI 请求无调用方 signal | **属实(复核确认)**:ChatCompletionOptions(ai-service.ts:39-48)无 signal 字段 | 无修正,维持 P2 |
|
||||||
|
| 9 | §12.5:桌面拾取无 AbortSignal | **属实(复核确认)**:desktop-pick.ts 全文零 signal 命中(取消是热键+轮询自有机制) | 无修正,维持 P1 |
|
||||||
|
| 10 | §2.2 "尚未完成 Agent-Employee 真实契约的编译级对接;当前没有 Agent-Employee 目标仓库真实源码作为本轮输入" | **已过时**:本机已有 D:/code/Agent-Employee 真实工作区,V2 已完成逐接口核查 | V2 的核查基线即基于该仓库 |
|
||||||
|
| 11 | 附录 §8 ADR 表"Driver 契约:RPA Driver 是否直接实现 computer-control?建议对齐宿主契约" | **升级为阻塞项**:对面的安全约束(禁任意 PowerShell、*.exe 白名单)与 RPA 驱动实现方式正面冲突,不是"对齐一下"而是二选一(V2 ADR-R3) | native-observer.ts 设计约束注释明示"不执行任意 PowerShell",为 V1 快照中没有的新事实 |
|
||||||
|
| 12 | 报告整体:审查对象是资料库 ZIP 快照(无 node_modules、未装依赖) | **V2 基线升级**:真实工作区 + 依赖已装 + 可实测(正则盲区构造用例、catalog python 统计、tsx 实测 capabilityCount=719) | 结论可信度提升;仍保留"未做实机 E2E/联调"的边界 |
|
||||||
|
|
||||||
|
## 三、V2 相对 V1 的新增内容
|
||||||
|
|
||||||
|
1. **双侧事实清单**(V2 §2/§3):24 项 RPA 侧 + 12 项宿主侧,全部带 `文件:行号` 证据与状态标记——V1 无此层。
|
||||||
|
2. **宿主侧巨变披露**:plugin-registry/plugin-worker(对面真实插件宿主)、model-gateway、Java 控制面(Spring Boot 454 文件、OpenAPI+Outbox、端口分工)、evidence-store——V1 完全未提。
|
||||||
|
3. **6 项 ADR + 3 项 Q 待确认**(V2 §10),其中 R3(桌面安全模型)、R6(部署拓扑)为 V1 未识别的新阻塞项。
|
||||||
|
4. **逐接入点七要素详单**(V2 §7,A1-A10):RPA 位置/AE 位置/兼容性/问题/改哪侧/方案/验收——V1 只有粗粒度对齐表。
|
||||||
|
5. **模块对应关系表**(V2 §6):双侧模块一一对应 + 关系 + 现状。
|
||||||
|
6. **分阶段实施计划**(V2 §8):0-6 阶段每阶段带硬验收门;新增"阶段 0 事实确认会"。
|
||||||
|
7. **风险清单 10 项**(V2 §9):新增 R1 安全模型冲突、R2 部署拓扑、R10 底座演进过期风险。
|
||||||
|
8. **能力治理映射方案**:719 能力 ↔ PermissionScope 13 项的映射生成策略(脚本从 catalog 生成 + 高风险域人工审)——V1 只说"不自建 allowlist"。
|
||||||
|
9. **宿主桥注入面扩容预案**:4 方法 → +ScreenCaptureHost/DisplayCoordinateService(vision 迁移必需),V1 的 4 方法清单不再完整。
|
||||||
|
10. **明确"未实现/设计中"口径**:computer-control Runner、WindowsRunner.click/type、MacDriver/LinuxDriver、Graph Validator、bun 冒烟、ScreenCaptureHost 全部标为未实现,防止把计划写成现状(用户红线 #7)。
|
||||||
406
docs/rpa-core-migration-guide-v2.md
Normal file
406
docs/rpa-core-migration-guide-v2.md
Normal file
@ -0,0 +1,406 @@
|
|||||||
|
# Agent-Employee RPA 迁入指南 V2(基于双侧真实源码核查)
|
||||||
|
|
||||||
|
> 文档性质:对 `docs/rpa-core-migration-guide.md`(V1)的全面修订版。
|
||||||
|
> 核查基线:2026-11-06,双侧**真实工作区源码**(非快照 ZIP、非审查报告转述)——
|
||||||
|
> RPA 侧 `D:/code/PRA-only-demo`(main,工作区干净)、宿主侧 `D:/code/Agent-Employee`。
|
||||||
|
> 与 V1 的逐条差异及审查报告修正记录见 `docs/rpa-core-migration-guide-v2-diff.md`。
|
||||||
|
> 状态标记约定:**已实现**(代码在库)/ **部分实现** / **仅契约**(有类型无实现)/ **设计中** / **未实现** / **待确认**(不允许猜测)。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 0. V2 核心结论(先读这个)
|
||||||
|
|
||||||
|
1. **宿主底座已大变,V1 的"空壳契约"叙事大部分失效。** V1 说对面 7 个契约包每个只有 1-9 个声明文件、windows-runner 是 0 源文件空壳。实测(2026-11-06):7 个包中 5 个已有实质实现(workflow-engine 25 文件、capability-gateway 321 行、browser-plugin 已接 Playwright、windows-plugin 新增原生观察适配器、runtime-governance 有 readiness 投影),windows-runner 有真实 C# 原生实现(.NET 10 + FlaUI UIA3,observe-only)。**仍无实现的只剩两处:computer-control 的 Runner、windows-plugin 的 WindowsRunner.click/type。**
|
||||||
|
→ 迁入的本质从 V1 的"拿实现去填契约"变为**"接底座 + 给已有实现让路"**:不重复造 workflow 状态机、不重复造 Playwright 浏览器、不重复造 UIA 观察。
|
||||||
|
2. **RPA 侧接入前 P0 整改已于 2026-11-06 完成(迁移阶段 1 收口)**:节点尝试级 AbortController(超时即真中止底层操作)、取消映射 cancelled 不走 error route、重试改 `idempotent` 元数据门控、图校验器执行前拒非法流程图、`createEngine(options)` 注入参数(driver/hostBridge/subflowSource/registry)、视觉能力 5 处动态 `import("electron")` 清零改走 HostBridge.screen 注入、check-deps 覆盖动态 import、catalog 脏值修复、rpa-cli 显式注册重写。★ 标记明细见 §2/§7/§8。**接入前本仓库代码活已全部完成(2026-11-06)**:①NodeState 补 `waiting_user`/`unknown` 两态 + `toStepStatus` 全量映射表(与对面 StepStatus 8 态同构,含契约测试);②A++ 事件负载字段定稿——`node.finished` 增 `checkpointId`/`resumable`/`idempotencyKey`(幂等键=runId:nodeId 跨重入稳定,非幂等能力不作为续跑锚点),契约测试覆盖。附带:补 sm-crypto 类型声明(typecheck 归零)。**对接面工作全部在对面仓库**:rpa-adapter、subtree 迁包、垂直切片;本方仅待对面答复 Q1/Q2 与 rpa.browser 备案。PowerShell 编码差异验证留待阶段 2 迁包后(bun 本机冒烟已于 2026-11-06 完成:bun 1.4.4,engine 37/37 全绿,exceljs/pdf 行为无差异;对面锁 1.4.3 为 patch 级差异)。
|
||||||
|
3. **迁入位置与名称维持 V1 结论**:`code/backend/packages/rpa-core` → `@agent-worker/rpa-core`,backend workspace(bun@1.4.3,workspaces 含 `packages/*`)放进去即生效,backend 全家统一 `@agent-worker/*` 前缀、`workspace:*` 源码消费。
|
||||||
|
4. **两条硬边界不变且加严**(用户红线):
|
||||||
|
- 所有带副作用的能力必须经 capability-gateway(授权→租约→幂等→执行→审计),**不得绕行**;
|
||||||
|
- rpa-core **不得依赖 Electron 主进程**(当前仍有 5 处动态 import,是接入阻塞项)。
|
||||||
|
5. **6 项 ADR 全部拍板(2026-11-06)**,仅剩 2 个小问题(Q1/Q2)需对面口头确认,详见 §10。已定基调:
|
||||||
|
- **R3 桌面执行**:C# 路线——五域执行逻辑移植进 windows-runner 原生进程,PowerShell 通道仅限本 demo。
|
||||||
|
- **R6 部署拓扑**:按能力域拆分——纯 Node 域进 backend plugin-worker,桌面/视觉/弹窗域装用户电脑 Electron 宿主,同一包按分组加载。
|
||||||
|
- **R1 能力清单**:全局一份,权限隔离全交对面网关按租户管。
|
||||||
|
- **R4 浏览器**:按场景分工——RPA 场景用咱们 CDP 接管(复用登录态/真实页面),对面 Playwright 留给 AI 智能体;白名单与证据共用对面一套。
|
||||||
|
- **R5 消费方式**:源码 git subtree 并入对面仓库统一编译,Bun 锁 1.4.3。
|
||||||
|
- **R2 集成形态(含 A++ 补记)**:方案 A——整个 RPA 流程注册为 `rpa.flow.execute` 单能力,事件流旁路+人审升级两补丁;事件流再升级为检查点协议(崩溃后带 resumeToken 断点续跑),权威账本始终单 step。两侧改动清单见 §7-A5。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. 核查方法与证据基线
|
||||||
|
|
||||||
|
- 双侧源码逐文件核查(两个独立探查通道 + 关键点人工复核),关键结论均带 `文件:行号` 证据。
|
||||||
|
- 实测过的事实:`import("electron")` 逃过 check-deps 正则(构造用例 0 匹配);catalog.json platformTag 分布(null=326 / win32=687 / 数字 1=3);capabilityCount()=719;对面 capability-gateway 租约 TTL=60s、幂等表上限 4096。
|
||||||
|
- 未验证(不写"通过"):Windows 实机 E2E、Bun 环境 test、对面真实链路联调。
|
||||||
|
- 审查报告(`docs/RPA-Review-Full.md`)与真实源码不一致处,以源码为准并记录修正理由(见 diff 文档第 2 节)。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. RPA 侧现状事实清单(2026-11-06 实测;★=阶段 1 已于当日修复)
|
||||||
|
|
||||||
|
| # | 事项 | 状态 | 证据 |
|
||||||
|
|---|---|---|---|
|
||||||
|
| 1 | Driver 惰性 getter + 非 win32 抛 `platform_unsupported` | 已实现 | engine.ts;error-codes.ts:43;engine-driver.test.ts |
|
||||||
|
| 2 | **Driver 可注入** | ★已实现 | `createEngine({driver})` + `engine.setDriver()`;desktop/common-final/system-extended/window-extended 的模块级捕获改惰性 `getDriver()` |
|
||||||
|
| 3 | Registry 全局 Map、last-wins、runtime 读全局 getHandler | 属实(ADR-R1 短期形态固化;新增 `engine.execute` 语义真实入口) | registry.ts;engine.ts execute() |
|
||||||
|
| 4 | PLATFORM_GROUPS 单一源(34 域全 win32)+ buildDomainPlatformMap 真实实现 + 守卫测试 | 已实现 | register-all.ts;platform-groups.test.ts |
|
||||||
|
| 5 | withTimeout 监听 signal 但无 AbortController——超时不中止底层 | ★已修复 | 尝试级 AbortController 汇聚 run 取消+节点超时,handler 收到的 ctx.signal 即尝试信号 |
|
||||||
|
| 6 | NodeState 类型含 cancelled,但 executeNode catch 把取消标成 failed | ★已修复 | 取消映射 cancelled(不适用 skipOnError、不进异常分支);runtime.cancel.test.ts |
|
||||||
|
| 7 | 重试上限 100、黑名单式 isRetryable | ★已门控 | CapabilityMeta.idempotent===false 时无视 retryCount |
|
||||||
|
| 8 | Graph Validator | ★已实现 | engine/graph-validator.ts(重复ID/悬空边/自环/环/孤儿/不可达),runFlow 前置校验 |
|
||||||
|
| 9 | 动态 `import("electron")` 5 处(vision.ts×3、cv-vision.ts×2) | ★已修复 | HostBridge.screen 服务注入(requireScreenService),Core 恢复零 Electron |
|
||||||
|
| 10 | check-deps.mjs 对 `import("electron")` 零匹配(盲区实测) | ★已修复 | 新增动态 import()/require() 扫描;合成用例命中=1,真实源码 0 违规 |
|
||||||
|
| 11 | 包依赖缺 3 项:@napi-rs/canvas、pdfjs-dist、@cantoo/pdf-lib | ★已修复 | rpa-core/package.json dependencies 补齐 |
|
||||||
|
| 12 | main/exports 直指 src 源码、private | 属实(决策项,与对面源码消费一致) | rpa-core/package.json |
|
||||||
|
| 13 | rpa-cli 副作用全量 import + 全局 getHandler | ★已重写 | 显式按域注册(5 域)+ engine.execute |
|
||||||
|
| 14 | CDP transport 单命令无超时/无 AbortSignal/pending 仅断连清理 | 属实(未修,A7 迁移前修) | transport.ts:96-118 |
|
||||||
|
| 15 | 两段式元素定位(当前页优先,失败才跟前台)| 已实现 | session.ts |
|
||||||
|
| 16 | locate() 不支持 xpath / Shadow DOM(与 withElement() 不一致)| 属实(未修,A7 迁移前修) | session.ts |
|
||||||
|
| 17 | HostBridge/SubflowSource 模块级单例 | 属实(ADR-R1 范畴;screen 服务已并入 HostBridge) | host-bridge.ts |
|
||||||
|
| 18 | AI ChatCompletionOptions 无 signal 参数 | 属实(未修,P2) | ai-service.ts |
|
||||||
|
| 19 | 桌面拾取无 AbortSignal(热键+轮询+deadline 自有机制)| 属实(未修,P1) | desktop-pick.ts |
|
||||||
|
| 20 | catalog platformTag:win32=687 / null=326 / 数字 1=3(脏值未清洗) | ✅ 已清洗(2026-11-06) | build-catalog.mjs 增白名单归一化(非 win32/darwin/linux 字符串一律重派生),3 条脏值重派生为 win32,全字段 diff 仅 3 条变化 |
|
||||||
|
| 21 | catalog overrides(fields/removeFields/addFields) | 已实现(+782 个 options 收割修复重生成回归) | catalog-overrides.mjs |
|
||||||
|
| 22 | 能力 719 个 / 目录组件 1016(vendor 997 + native 16 + trigger 3)/ fields 4299 / 34 域 | 实测 | platform-groups.test.ts;catalog.json counts |
|
||||||
|
| 23 | 平台守卫测试 5 文件 + driver/platform.ts 唯一入口 | 已实现 | capabilities/*.test.ts;driver/platform.ts |
|
||||||
|
| 24 | Driver 五域方法均无 AbortSignal 参数 | 属实(未修;运行时取消已能中止监听 signal 的能力内部操作) | driver/types.ts |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. 宿主侧(Agent-Employee)现状事实清单
|
||||||
|
|
||||||
|
| 契约包/服务 | V1 描述 | 2026-11-06 实测 | 状态 |
|
||||||
|
|---|---|---|---|
|
||||||
|
| workflow-engine | 9 个声明文件,空壳 | **25 文件完整实现**:WorkflowEngine 接口(create/start/beginStep/executeStep(runner)/completeStep/pause/resume/cancel/requestApproval/decideApproval)、状态机迁移表、Memory+PostgreSQL 双存储、recovery、REST API、worker(并发上限/取消/排空) | 已实现 |
|
||||||
|
| capability-gateway | 空壳契约 | **321 行完整实现**:`authorize(ref, subjectId, policy) → CapabilityLease`(TTL 60s)、`execute(lease, runner)`、幂等键预占/结算(上限 4096)、高影响审批精确 actionId 放行、结果 leaseId/actionId 回显校验 | 已实现 |
|
||||||
|
| browser-plugin | 纯契约 | **已接 playwright-core ^1.64.1**:PlaywrightBrowserRunner 358 行(navigate/click/type/snapshot),域名白名单三重复检(导航前/落点后/动作前),证据 SHA-256 落盘;定位仅 css/aria/text(无 xpath) | 已实现 |
|
||||||
|
| windows-plugin | 纯契约 | 契约 + **新增 native-observer.ts(292 行受信宿主适配器)**:spawn C# 观察进程、证据 realpath/SHA-256/PNG 魔数校验;`WindowsRunner{click;type;readText;snapshot}` 接口仍无实现;manifest `rpa.windows`:type=highImpact,全部 idempotencyRequired | 部分实现 |
|
||||||
|
| computer-control | 纯契约 | **未变**:纯类型 + 3 纯函数(selectPreferredTarget 降级链、assertComputerActionAllowed、assertComputerResultEvidence);5 种 target kind、6 种 action、CoordinateSpace(DPI/缩放契约已就绪) | 仅契约 |
|
||||||
|
| runtime-governance | 空壳 | readiness 投影实现(readiness.ts 117 行),无插件生命周期契约 | 部分实现 |
|
||||||
|
| plugin-contract | 纯类型 | 类型底座 + `assertPluginManifest` 校验实现;**PermissionScope 13 项**(browser.domain/storage、desktop.window/input/process、filesystem.workspace/download、network.http、office.document、messaging.send/read、media.generate、monitoring.capture);CapabilityLease/ActionResult/CapabilityRef(`pluginId:capabilityId`) | 部分实现 |
|
||||||
|
| services/windows-runner | 0 源文件空壳 | **17 文件真实 C# 实现**(.NET 10 + FlaUI UIA3 + PrintWindow,stdin/stdout JSON 协议,**只认 observe**,click/type 未实现);TS 宿主适配器校验证据 | 部分实现(观察半边) |
|
||||||
|
| plugin-registry / plugin-worker | (V1 未提) | registry=清单登记+版本链校验(无动态 loader);plugin-worker=常驻执行进程,PG LISTEN/NOTIFY 认领 workflow/session 动作、租约续期、跨进程取消传播、崩溃接管→unknown。**这就是对面当前的插件宿主** | 已实现 |
|
||||||
|
| model-gateway | (V1 未提) | 21 文件实现包:ModelGateway.stream 流式调用 + ModelControlPlane(每次物理调用向 Java 控制面重新授权,短授权 60s) | 已实现 |
|
||||||
|
| Java 控制面 | (V1 未提) | `code/backend/control-plane/`:Spring Boot 4.1.1 / Java 21 / 454 个 Java 文件 / 9 Maven 模块;与 TS 面经 OpenAPI + Outbox + 短授权 grant 通信;端口 8887(Java)/8881(Admin)/8880(TS 执行面) | 已实现 |
|
||||||
|
| frontend/apps/desktop | Electron+SolidJS | 确认。**无 RPA 宿主桥**,但有等价物:identity-login-window(弹窗)、execution-context.ts(grant 桥)、zoom.ts(缩放);DPI/坐标契约在 computer-control CoordinateSpace 与 C# Protocol.cs | 部分实现 |
|
||||||
|
|
||||||
|
对面安全模型要点(直接影响 RPA 桌面路线):windows-plugin 策略校验只允许 `*.exe` 进程名、不允许路径;native-observer.ts 设计约束明言"不执行任意 PowerShell";桌面交互 Runner 留宿主、不进 Linux 容器。
|
||||||
|
|
||||||
|
**对面协作门禁(2026-11-06 新增,本项目补丁必须遵守)**:对面已建立代码与文档事实一致性门禁——`code/tooling/document-facts.mjs` + `项目资料/开发方案/当前事实.json`(factRevision 2026-11-06.1)。规则:代码、迁移、合同、端口、运行事实的任何变化,必须与事实文件、README 机器生成区块**同一提交**;CI 的 `document-facts --check` / `ci-gate.mjs` 会重扫源码输入并阻断漂移。**因此本项目未来合入对面仓库的任何补丁(含 C# actuate 补丁包、subtree 迁包)须:①同一提交内更新 `当前事实.json` 与相关 README 区块;②提交前跑 `node code/tooling/document-facts.mjs --check` 与 `node code/tooling/ci-gate.mjs --mode fast`。** 另:对面 backend/README Windows 行已明确下一步为"click/type、视觉/OCR 或随本轮客户端交付"——R3 C# actuate 路线与其路线图一致,可据此作为提案依据。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. 原指南(V1)内容分类
|
||||||
|
|
||||||
|
| V1 内容 | 分类 | 处理 |
|
||||||
|
|---|---|---|
|
||||||
|
| 包现状描述(719 能力、34 域、宿主桥 4 方法、Driver 五域) | **保留**(微调) | 仍准确;宿主桥维持 4 方法不变 |
|
||||||
|
| "完全不依赖 Electron、check:deps 天天检查" | **过时/不实** | 5 处动态 import 逃过护栏;改为"静态依赖已清、动态残留待清",护栏必须升级 |
|
||||||
|
| 第 -1 步对面契约包全部空壳的叙事 | **过时** | 按 §3 改写:5/7 已实现,"填契约"变"接底座" |
|
||||||
|
| 第 0 步三件作业(bun 冒烟 / catalog 重生成 / 注册方式决策) | ✅ 全部完成 | catalog 重生成完成且脏值已清洗(platformTag win32=690 / null=326,数字脏值归零);bun 1.4.4 本机冒烟 37/37;注册方式决策并入 ADR-R1 |
|
||||||
|
| 第 1 步搬包(git subtree → code/backend/packages/rpa-core、改名 @agent-worker/rpa-core、demo 侧不双活) | **保留** | 双侧命名与 workspace 布局核实无误 |
|
||||||
|
| 第 2 步 A 类契约对齐表 | **需要修改** | capability-gateway/workflow-engine 从"对照后改对面"变"按已实现接口写 adapter";新增 plugin-contract/registry/worker 三层对接 |
|
||||||
|
| 第 2 步 B 类宿主注入表(浏览器=最大变数) | **需要修改** | 浏览器变数已收敛:对面有 Playwright 实现;变数改为"双路径取舍"(ADR-R4) |
|
||||||
|
| 第 3 步 7 条验收 | **需要修改** | 扩充为 §9 验收标准(加 gateway 零调用、取消传播、双 Engine 隔离等) |
|
||||||
|
| 三端计划(PLATFORM_GROUPS 单一源、驱动自动选、分组核查、测试守卫、平台判断收拢) | **已完成,从"待办"改"现状"** | 五件小事全部落地;平台硬约束一节保留为长期规范 |
|
||||||
|
| 附:10 个关键决策 | **大部分保留** | 决策 2"保留全局注册表"与决策 9"双入口"需按 ADR-R1 重新确认;其余不变 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. 接入目标架构
|
||||||
|
|
||||||
|
```text
|
||||||
|
┌─ Agent-Employee(宿主:权威状态与治理的唯一来源)──────────────────────┐
|
||||||
|
│ │
|
||||||
|
│ Java 控制面 control-plane(:8887) ──OpenAPI/Outbox/短授权 grant──┐ │
|
||||||
|
│ │ │
|
||||||
|
│ TS 执行面 apps/api(:8880, Bun) │ │
|
||||||
|
│ ├─ workflow-engine ← Run/Step 权威状态机(PG 存储+recovery) │ │
|
||||||
|
│ ├─ capability-gateway← 授权→租约(60s)→幂等→执行→审计【唯一入口】│ │
|
||||||
|
│ ├─ plugin-registry ← 清单登记 + 版本链校验(无动态 loader) │ │
|
||||||
|
│ ├─ model-gateway ← AI 流式调用 + 每次调用控制面授权 │ │
|
||||||
|
│ └─ services/plugin-worker ← 常驻执行进程(LISTEN/NOTIFY 认领) │ │
|
||||||
|
│ ├─ browser-plugin → PlaywrightBrowserRunner(已实现) │ │
|
||||||
|
│ └─ windows-plugin → windows-runner(C# FlaUI, observe-only)│ │
|
||||||
|
│ │
|
||||||
|
│ frontend/apps/desktop(Electron+SolidJS,用户桌面宿主) │ │
|
||||||
|
│ └─ RPA 宿主桥实现落点(弹窗/DPI/AI 配置/子流程仓库) │ │
|
||||||
|
└────────────────────────────────────────────────────────────────┘
|
||||||
|
▲
|
||||||
|
│ CapabilityRunner(gateway 包皮,唯一执行通道)
|
||||||
|
│ Run/Step 事件回传(RPA 不建第二套状态)
|
||||||
|
┌─ RPA 迁入物(执行器与能力供给方)──────────────────────────────────────┐
|
||||||
|
│ @agent-worker/rpa-core(packages/rpa-core 迁入,Bun 源码消费) │
|
||||||
|
│ ├─ Flow Runtime(作为 workflow-engine 的执行引擎候选 / 步骤级 runner)│
|
||||||
|
│ ├─ 719 能力 handler(纯 Node 部分可在 plugin-worker 服务器侧跑; │
|
||||||
|
│ │ 桌面/视觉部分只能跑在用户桌面宿主进程——见 ADR-R6) │
|
||||||
|
│ ├─ WindowsDriver(补齐对面 WindowsRunner.click/type 执行缺口,ADR-R3)│
|
||||||
|
│ └─ CDP 外部浏览器会话(与 PlaywrightBrowserRunner 的取舍,ADR-R4) │
|
||||||
|
│ @agent-worker/rpa-adapter(新增薄适配层) │
|
||||||
|
│ ├─ CapabilityRunner:authorize/execute 包裹 → ref↔componentId 映射 │
|
||||||
|
│ ├─ RunResult/NodeState ↔ WorkflowStatus/StepStatus 映射表 │
|
||||||
|
│ └─ 宿主注入装配:HostBridge(4) + SubflowSource(2) + ScreenCaptureHost│
|
||||||
|
└────────────────────────────────────────────────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
**权责原则(沿审查报告,逐条可执行):**
|
||||||
|
- workflow-engine 是 Run/Step 状态唯一权威;RPA Runtime 只回传事件,不落第二份终态。
|
||||||
|
- capability-gateway 是唯一授权入口;RPA 侧 `getHandler()` **不对 adapter 之外暴露**(rpa-cli 顺手重写以作表率)。
|
||||||
|
- computer-control 是桌面契约对齐对象;RPA 不新造并行的桌面抽象。
|
||||||
|
- 宿主服务一律注入,禁用模块级 setter 承载多会话状态(现状 HostBridge 单例是过渡,见 ADR-R1)。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. 模块对应关系表
|
||||||
|
|
||||||
|
| RPA 侧(packages/rpa-core) | Agent-Employee 侧 | 关系 | 现状 |
|
||||||
|
|---|---|---|---|
|
||||||
|
| `engine/runtime.ts` 流程执行 | `workflow-engine` WorkflowEngine/StepRun | RPA=执行算法候选,AE=权威状态 | 已拍板方案 A(ADR-R2) |
|
||||||
|
| `engine/registry.ts` 全局注册表 | `plugin-registry` + `capability-gateway` CapabilityRef | ref↔componentId 映射层(rpa-adapter) | 映射待建 |
|
||||||
|
| 719 能力 handler | `plugin-contract` CapabilityDefinition + PermissionScope 13 项 | 能力清单生成 + 权限映射 | 待建(脚本生成) |
|
||||||
|
| `RunContext.signal` | `CapabilityExecutionContext.signal` | 均 AbortSignal,直通 | 兼容 |
|
||||||
|
| `NodeState`(idle/running/succeeded/failed/skipped/cancelled) | `StepStatus`(+waiting_user/unknown/pending) | 映射表;RPA 缺 waiting_user/unknown | 需补 |
|
||||||
|
| `human-dialog` 审批弹窗 | `WorkflowApprovalRequest`/decideApproval | 人审节点挂宿主 approval 流 | 需 adapter |
|
||||||
|
| `driver/windows/*`(PowerShell 五域) | `windows-runner` C#(observe-only)+ `WindowsRunner{click,type}` 契约 | RPA 补执行半边;安全模型冲突 | ADR-R3 |
|
||||||
|
| `browser/*` CDP 会话(BrowserLike) | `browser-plugin` PlaywrightBrowserRunner | 双路径取舍 | ADR-R4 |
|
||||||
|
| `ai-service.ts` loadAiConfig | `model-gateway`(ModelGateway.stream + ModelControlPlane) | RPA AI 能力改走宿主网关 | 需改造 |
|
||||||
|
| `host-bridge.ts` 4 方法 | frontend desktop main 进程(弹窗/坐标/AI 配置等价物已有) | 参考实现照抄本仓库 src/main/index.ts | 落点明确 |
|
||||||
|
| `SubflowSource` 2 方法 | 对面流程存储 | 包一层 | 小 |
|
||||||
|
| vision/cv-vision 截图+DPI | `computer-control` CoordinateSpace + C# PrintWindow | 宿主注入 ScreenCaptureHost | 待建 |
|
||||||
|
| 运行证据/截图 | `evidence-store`(S3) + `monitoring.capture` 权限 | 证据统一落对面 | 待接 |
|
||||||
|
| `check-deps.mjs` 依赖护栏 | backend 包边界 | 保留并升级(动态 import) | 需增强 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. 逐接入点详单
|
||||||
|
|
||||||
|
每项按固定七要素:RPA 位置 / AE 位置 / 兼容性 / 问题 / 改哪侧 / 方案 / 验收。
|
||||||
|
|
||||||
|
### A1 Engine 与 Driver 注入
|
||||||
|
|
||||||
|
- **RPA 位置**:`packages/rpa-core/src/engine/engine.ts:47-66`(惰性 getter + `createEngine()` 无参);`capabilities/desktop.ts:9`、`common-final.ts:11`、`system-extended.ts`、`window-extended.ts`、`office.ts:15`、`ftp.ts`、`ocr-caps.ts:11` 捕获 `defaultEngine.*`(其中 desktop.ts:9 在**模块加载时**取 `.driver`,非 win32 下 import 即抛错)。
|
||||||
|
- **AE 位置**:对面不关心 driver 细节;对齐契约是 `computer-control` 的 `ComputerRunner{observe;execute}`(computer-control/src/index.ts:99-102)与 `windows-plugin` 的 `WindowsRunner`。
|
||||||
|
- **兼容性**:部分。平台错误语义(platform_unsupported)已就绪;注入面缺失。
|
||||||
|
- **问题**:宿主无法注入测试替身或 computer-control 适配器;能力文件模块级捕获引擎状态(excelHandles/ftpClient/ocrInstance/driver),多会话共享。
|
||||||
|
- **改哪侧**:RPA。
|
||||||
|
- **方案(★ 已实现,2026-11-06 阶段 1-A1)**:`createEngine(options?: { driver?, hostBridge?, subflowSource?, registry? })`,全部可选、缺省保持现行为;能力文件改为经 RunContext/engine 参数取 driver 与资源表(模块级捕获改为首用时向 engine 取);幂等句柄表(excelHandles/ftpClient/ocrInstance)迁入 engine 实例字段(部分已是)。
|
||||||
|
- **验收**:FakeDriver 注入测试覆盖五域;两个 Engine 各注入不同 FakeDriver 并行执行互不串用;非 win32 上 import 全部 34 域不抛错、调用桌面能力时才抛 platform_unsupported。
|
||||||
|
|
||||||
|
### A2 Registry 注册机制与实例隔离
|
||||||
|
|
||||||
|
- **RPA 位置**:`engine/registry.ts:36-45`(模块级 Map、last-wins、meta 可残留旧值);`runtime.ts:260`(全局 getHandler);`register-all.ts`(34 域 DOMAINS + PLATFORM_GROUPS + buildDomainPlatformMap)。
|
||||||
|
- **AE 位置**:`plugin-registry`(清单登记+版本链,无动态 loader)、`capability-gateway` CapabilityRef(`pluginId:capabilityId` + version)。
|
||||||
|
- **兼容性**:结构性错位。对面一切能力调用都带 CapabilityRef 与版本;RPA 是字符串 id 全局表。
|
||||||
|
- **问题**:进程级注册在对面"多租户 + plugin-worker 常驻进程"模型下无法按租户隔离;componentId 与 CapabilityRef 无映射;同 id 覆盖无审计。
|
||||||
|
- **改哪侧**:短期 RPA 侧固化 ADR + 新建 adapter 映射;长期 RPA 侧 engine-scoped registry。
|
||||||
|
- **方案**:①ADR-R1 先定部署模型(单后端单能力集 → 全局表合法,把 `registerAll` 文档语义改为"加载到进程注册表",不假装实例隔离);②rpa-adapter 维护 `componentId ↔ (pluginId="rpa", capabilityId=componentId, version=catalog 版本)` 映射,由 catalog.json 生成并做完整性测试;③重复注册策略定为"显式覆盖 + 输出 warn 审计行"。
|
||||||
|
- **验收**:映射表覆盖 719 能力(或明确排除清单);gateway 侧按 ref 解析到正确 handler;重复注册产生可审计日志。
|
||||||
|
|
||||||
|
### A3 Runtime 执行、超时、取消、重试
|
||||||
|
|
||||||
|
- **RPA 位置**:`runtime.ts:33-56`(withTimeout 只 reject 外层)、`runtime.ts:334-339`(取消标 failed)、`runtime.ts:58-60,276`(isRetryable 黑名单、retryCount≤100)、`driver/types.ts:87-119`(五域方法无 signal)。
|
||||||
|
- **AE 位置**:`workflow-engine` StepRun.attempt/leaseEpoch/idempotencyKey、`workflow-worker` 跨进程取消传播与崩溃接管→unknown、`capability-gateway` execute 的 `ctx.signal?`。
|
||||||
|
- **兼容性**:不兼容。对面以幂等键 + 租约纪元管理副作用安全;RPA 超时/取消/重试语义与它冲突。
|
||||||
|
- **问题**:超时后底层 PowerShell/HTTP/CDP 仍执行(重复副作用);取消状态不一致;非幂等能力可被重试 100 次。**这是接入的第一阻塞项。**
|
||||||
|
- **改哪侧**:RPA。
|
||||||
|
- **方案(★ 已实现,2026-11-06 阶段 1-A3)**:①节点尝试创建独立 AbortController,合并 Run 级 signal 与节点 timeout,贯通 handler→driver→PowerShell 子进程/HTTP/CDP(driver 五域方法加可选 `signal` 参数);②executeNode catch 优先映射 cancelled → `state.status="cancelled"`,不走 skipOnError/error route/retry;③重试改元数据驱动:CapabilityMeta 增 `idempotent`/`sideEffect`/`retryPolicy`,非幂等默认 0 次重试;④新增 `validateFlowGraph`(ID 唯一、边合法、入口/出口基数、环、悬空边)。
|
||||||
|
- **验收**:构造 5 秒 Fake Handler + 100ms 超时,断言 handler 收到 abort 并停止、无并发重入;执行中取消后 Run 与 NodeState 均为 cancelled;对 File 写入/点击类能力验证"超时不自动重试";非法图执行前得到稳定错误码。
|
||||||
|
|
||||||
|
### A4 Capability Gateway(能力注册与权限治理)
|
||||||
|
|
||||||
|
- **RPA 位置**:`registry.ts`(handler + meta 无权限字段);`resources/catalog.json`(仅 `risk: low|high`)。
|
||||||
|
- **AE 位置**:`capability-gateway/src/index.ts:75-252`(已实现 authorize/execute/租约 60s/幂等/高影响审批);`plugin-contract` PermissionScope 13 项。
|
||||||
|
- **兼容性**:接口契合度高——`CapabilityRunner = (input, ctx{lease,capability,signal?}) => Promise<ActionResult>`,RPA handler `(ctx, node, args, component)` 包一层即可;ActionResult 需回显 leaseId/actionId(adapter 统一填)。
|
||||||
|
- **问题**:①719 能力 ↔ 13 项 PermissionScope 无映射;②CapabilityMeta 无 sideEffect/idempotent/permission/approvalRequired 字段;③存在绕行面(rpa-cli、子流程直接 getHandler)。
|
||||||
|
- **改哪侧**:新建 adapter(AE 仓库内包)+ RPA 补元数据。
|
||||||
|
- **方案**:①rpa-adapter 实现 CapabilityRunner:gateway.execute(lease, runner) → ref→componentId → 构造 RunContext(signal 直通)→ handler → 结果包装 ActionResult;②权限映射表按 34 域 × PermissionScope 初稿生成(files→filesystem.workspace、http→network.http、desktop/windowExtended→desktop.input/window、utilityExtended→独立高影响 scope【待确认:现表无 process 执行域对应,需对面确认】、message→messaging.send);③高风险域标 highImpact 走审批(System_ExecuteCommand、键鼠、文件删除/覆盖、发消息);④meta 增治理字段(与对面 CapabilityDefinition 字段对齐)。
|
||||||
|
- **验收**:审批拒绝/租约过期/权限不在白名单时 handler **零调用**(gateway 已有测试语义,adapter 层加 719 全量冒烟:未经 authorize 不得触达 handler);审计可关联 Agent/Run/Capability/审批结果。
|
||||||
|
- **租约粒度拍板(2026-11-06 用户拍板:每个节点执行都要慎重)**:gateway 租约 TTL=60s(index.ts:124),而方案 A 整条流程可达数分钟——**解法定为"外层一个 step、内层每节点一次授权"**:流程内每个节点执行前由 adapter 单独走一次 authorize+execute(60s/节点绰绰有余),高危节点逐节点审批。与 A++ 检查点协议天配:事件负载的 `nodeId + idempotencyKey(runId:nodeId)` 正好构成 gateway 幂等键所需的作用域材料(workflow 来源要求 run+step+attempt,adapter 以 nodeId 充当 step 定位)。每节点授权同时天然实现"慎重"语义:任何节点执行前都过权限白名单+审批门,无绕行。
|
||||||
|
- **版本管理设计提案(2026-11-06,adapter 层落地,不改引擎)**:对面版本约束为硬性三处——①每次调用必带 `capabilityVersion`,与注册表不符直接拒绝(gateway index.ts:171);②幂等键含能力版本与配置版本;③插件 manifest 强制三段 semver(plugin-contract index.ts:277,`v1.0`/四段式均不通过)。我方现状:catalog 997/1016 项带上游四段版本(如 `App_Run@3.3.5.0`),但注册表无版本机制、包版本 0.1.0 从未随能力变化推进。提案:①对外能力 version = rpa-core 包 semver(0.1.0 起,仅对外行为变化才升版);②catalog 四段版本归一化为三段后进 CapabilityDefinition(版本仅作标识,行为一致性由同包源码保证);③configVersion 用 catalog+overrides 内容哈希(与幂等键 configVersion 段正好衔接)。**对迁包零影响**:catalog 版本字段原样迁移,归一化发生在 adapter 生成清单时。
|
||||||
|
|
||||||
|
### A5 Workflow Engine 执行契约
|
||||||
|
|
||||||
|
- **RPA 位置**:`engine/runtime.ts`(RunResult/NodeState)、`engine/types.ts:142`、human-dialog 审批弹窗、SubflowSource。
|
||||||
|
- **AE 位置**:`workflow-engine/src/index.ts:278-311`(WorkflowEngine 接口)、`workflow-runtime.ts`、`workflow-worker.ts`、`approval-policy.ts`;`executeStep(..., runner)` 的 runner 类型即 `Parameters<CapabilityGateway["execute"]>[1]`。
|
||||||
|
- **兼容性**:状态映射大体可行;RPA 缺 `waiting_user`/`unknown` 两态;对面 StepStatus 无 RPA 的 `idle`。
|
||||||
|
- **问题**:两套状态权威冲突风险;集成形态二选一未定(整体作为一个 capability vs 节点级映射为 step)。
|
||||||
|
- **改哪侧**:adapter + 双侧小改。
|
||||||
|
- **方案**:**已拍板(ADR-R2)方案 A + 两补丁**——整个 RPA 流程注册为**一个** capability(`rpa.flow.execute`),workflow-engine 的一个 step 调它,RunResult→ActionResult、节点级事件→event-bus 事件流(旁路,不进权威账本)、human-dialog→requestApproval 挂对面 approval 流、取消→signal 直通。补丁 1:节点事件流保透明;补丁 2:人审/高危节点照常升级对面审批。方案 B(节点=step 1:1)的触发条件留档:仅当对面产品确认需要"逐步控制 RPA"(单步执行/任意步回退)时局部启用,届时事件流已备好数据不返工。NodeState 增 `waiting_user`/`unknown` 两态(unknown 用于崩溃恢复对齐)。
|
||||||
|
- **A++ 检查点补记(2026-11-06,本方负责人确认)**:在方案 A 骨架内将节点事件流升级为检查点协议(决策全文见 §10-R2)。
|
||||||
|
- **RPA 侧改动**:①事件负载增 `nodeId`/`status`/`checkpointId`/`idempotencyKey`/`resumable` 字段;②`rpa.flow.execute` 接受可选 `resumeToken`,重入时从最近检查点续跑(幂等句柄表已迁 engine 实例字段,地基现成);③幂等键生成与传递贯通(复用 A3 的 `idempotent` 门控,仅补 key 生成与透传)。
|
||||||
|
- **对面侧改动**:①workflow-engine 对 `rpa.flow.execute` 所在 step 的重试/恢复策略允许带 resumeToken 重入(不新建 step、不改租约语义);②event-bus 消费端识别检查点事件并入审计链;③崩溃恢复时把 resumeToken 经 adapter 回传(复用 workflow-recovery 的 unknown→恢复路径)。
|
||||||
|
- **落地节奏**:阶段 2 只做事件负载字段设计定稿(payload 预留,近零成本);阶段 3 垂直切片做最小验证(崩溃→重入续跑);阶段 4/5 规模化时强制执行。
|
||||||
|
- **验收**:状态映射契约测试(含 waiting_user/unknown/cancelled 全分支);一次含审批节点、一次执行中取消、一次进程崩溃恢复的端到端演练;对面 Run 状态终态与 RPA RunResult 永远一致;崩溃重入演练——杀掉执行进程后带 resumeToken 续跑,已完成节点不重复执行、幂等键防重复副作用。
|
||||||
|
|
||||||
|
### A6 Computer Control 与 Windows 桌面执行
|
||||||
|
|
||||||
|
- **RPA 位置**:`driver/windows/*`(PowerShell 封装五域)、`capabilities/desktop-uia.ts`、`element.ts`。
|
||||||
|
- **AE 位置**:`computer-control`(仅契约+3 纯函数)、`windows-plugin`(WindowsRunner{click;type;readText;snapshot} 无实现、native-observer.ts observe-only)、`services/windows-runner/native/*.cs`(FlaUI UIA3 observe-only)。
|
||||||
|
- **兼容性**:**安全模型冲突**——RPA 驱动 spawn 任意 PowerShell;对面白名单只许 `*.exe`、明文"不执行任意 PowerShell"、证据要求 SHA-256。
|
||||||
|
- **问题**:RPA 的执行半边(click/type)正是对面缺口,但实现方式过不了对面安全门。
|
||||||
|
- **改哪侧**:双侧(ADR-R3 已拍板:C# 扩展 actuate,见 §10)。
|
||||||
|
- **方案**:推荐——把 RPA WindowsDriver 的五域执行逻辑**移植进 windows-runner 的 C# 原生进程**( FlaUI/CGEvent 等价实现 + 现成协议加 actuate 操作),RPA 侧 driver 抽象保留、Windows 实现改为可指向"原生进程通道"或"本地 PowerShell 通道"(后者仅 demo 使用,明确标注不走宿主治理);策略校验/证据断言复用对面 `assertComputerActionAllowed`/`assertComputerResultEvidence`。
|
||||||
|
- **验收**:click/type 经策略校验(白名单进程/窗口、allowInput)+ 证据落盘;宿主拒绝时零副作用;同目标 observe→actuate→observe 闭环一致。
|
||||||
|
|
||||||
|
### A7 浏览器(Browser Plugin vs RPA CDP 会话)
|
||||||
|
|
||||||
|
- **RPA 位置**:`packages/rpa-core/src/browser/*`(transport/external,进包部分)+ `src/main/browser/session.ts`(接管/拾取,留守部分);已知缺陷:transport 单命令无超时(:96-118)、locate 不支持 xpath/Shadow DOM、evaluate 默认 follow=true。
|
||||||
|
- **AE 位置**:`browser-plugin`(PlaywrightBrowserRunner 已实现:白名单三重检、AbortSignal 透传、证据 SHA-256;定位 css/aria/text,无 xpath)。
|
||||||
|
- **兼容性**:双实现并行,语义不同(Playwright 自管浏览器 vs RPA 接管用户已开浏览器/复用登录态)。
|
||||||
|
- **问题**:重复投资;RPA CDP 链路自身有 P1 缺陷;对面定位器无 xpath(RPA 拾取器产出 xpath/css/aria/text 混合 selector)。
|
||||||
|
- **改哪侧**:ADR-R4 已拍板(按场景分工):RPA 场景走咱们 CDP 接管(主力),对面 Playwright 留给 AI 智能体场景。
|
||||||
|
- **方案**:①RPA 浏览器能力迁前先修 transport 单命令超时 + 统一 resolveElement(一次修复覆盖 locate/click/type/getText);②CDP 接管路线接对面 BrowserContextPolicy 白名单与证据体系(不自建白名单);③与对面确认 rpa.browser 插件清单承载咱们的 runner;④PlaywrightBrowserRunner 保持现状服务 AI 智能体场景,双方不重复投资。
|
||||||
|
- **验收**:同一定位用例集(css/aria/text/xpath)在咱们 CDP 路径全部通过;白名单拒绝 URL 时导航前即拦截(走对面策略对象);证据落 evidence-store。
|
||||||
|
|
||||||
|
### A8 平台适配(Windows / Linux / macOS)
|
||||||
|
|
||||||
|
- **RPA 位置**:`register-all.ts` PLATFORM_GROUPS(34 域全 win32)、`driver/platform.ts`、`engine/engine.ts:47-61`、5 个 skipIf 测试;catalog **win32=690 / null=326,数字脏值已归零(2026-11-06 白名单归一化)**。
|
||||||
|
- **AE 位置**:`plugin-contract` Platform 类型;windows-runner 明确"留在宿主、不进 Linux 容器"(README)。
|
||||||
|
- **兼容性**:兼容(两边都是 win32-first 务实路线)。
|
||||||
|
- **问题**:①~~catalog 3 个 `platformTag: 1` 脏值透传~~ **已修复(2026-11-06)**:build-catalog.mjs 白名单归一化,上游 shizai 的数字 1 重派生为 win32(三条均为包内真实现,语义正确);②对面 CapabilityDefinition 未见 platform 字段(**待确认**:插件清单如何表达平台限制)。
|
||||||
|
- **改哪侧**:RPA(清洗);对面(清单平台字段,待确认后)。
|
||||||
|
- **方案**:build-catalog 归一化——源值不在 `win32|darwin|linux|null` 白名单的一律重派生;三端扩展节奏沿用 V1 第三节(Linux xdotool/AT-SPI、Mac AX/CGEvent/TCC、Office 分端实现),驱动实现统一走 ADR-R3 的原生进程通道。
|
||||||
|
- **验收**:catalog 全量过 schema 校验(1016/1016);非 win32 上 Core 可加载、桌面能力明确报 platform_unsupported;Mac/Linux CI 跑纯逻辑子集。
|
||||||
|
|
||||||
|
### A9 Electron 依赖与 Core 独立性(含包闭包)
|
||||||
|
|
||||||
|
- **RPA 位置**:vision.ts:18,66,432、cv-vision.ts:51,213(动态 `import("electron")`);check-deps.mjs:20(正则盲区实测);rpa-core/package.json:17-32(缺 @napi-rs/canvas、pdfjs-dist、@cantoo/pdf-lib;main 直指 src、private)。
|
||||||
|
- **AE 位置**:backend 执行面是 Bun(无 Electron);Electron 只在 frontend/apps/desktop;截图/坐标对面等价物是 C# PrintWindow + CoordinateSpace 契约。
|
||||||
|
- **兼容性**:不兼容。视觉能力在对面 backend 进程必然失败。
|
||||||
|
- **问题**:Core 不是 Electron-free;护栏查不出动态 import;单包安装缺 3 个依赖。
|
||||||
|
- **改哪侧**:RPA。
|
||||||
|
- **方案(★ 已实现,2026-11-06 阶段 1-A9)**:①抽 `ScreenCaptureHost`/`DisplayCoordinateService` 接口并入 HostBridge 注入面(弹窗/DPI×2/AI 配置 4 方法 → 增至 6 方法,或独立 host services 包——实现期定,倾向并入 HostBridge 减少 API 面);vision/cv-vision 只依赖接口,宿主未注入时抛 `host_service_unavailable`;②check-deps 升级为覆盖 `import()`/`require()`/`createRequire()`(先补正则,中期 AST);③补齐 3 个依赖声明(pdf.js 与 canvas 为原生/大件,评估 optionalDependencies + 能力级降级);④消费方式锁定 **Bun 源码消费**(对面全家 workspace:* 源码依赖,与现状一致),锁 bun@1.4.3 做 CI 冒烟。
|
||||||
|
- **验收**:无可解析 electron 的纯净 Node/Bun 环境 Core 全量 import 成功;视觉能力未注入 Host 时返回 host_service_unavailable、注入 Fake Host 后可测;隔离目录仅按包声明装依赖跑通 PDF/OCR 代表能力。
|
||||||
|
|
||||||
|
### A10 插件生命周期、异常隔离与资源清理
|
||||||
|
|
||||||
|
- **RPA 位置**:无插件概念;资源清理散在能力内(PowerShell 子进程、ws、FTP/OCR 句柄);历史上有 catch{} 吞错模式(已部分修复)。
|
||||||
|
- **AE 位置**:`plugin-contract` PluginManifest(apiVersion/minCoreVersion/entrypoint/rollbackVersion)+ assertPluginManifest;`plugin-registry` 版本链校验;`plugin-worker`(崩溃接管→unknown、租约续期、跨进程取消)。
|
||||||
|
- **兼容性**:RPA 需要一张插件"外壳"才能进对面宿主。
|
||||||
|
- **问题**:719 能力不可能逐个成插件;RPA 无崩溃恢复语义(对面 unknown 态)。
|
||||||
|
- **改哪侧**:adapter 为主。
|
||||||
|
- **方案**:RPA 以**单一插件**接入(manifest id 建议 `rpa.core`,能力按域分批暴露为 CapabilityDefinition 清单,随 catalog 生成);异常隔离靠节点级 try/catch + AbortSignal(A3);资源清理建统一 finally 清单(子进程 kill、ws close、临时文件删除)并纳入取消路径;崩溃恢复复用对面 workflow-recovery(NodeState.unknown 对齐)。
|
||||||
|
- **验收**:杀掉 plugin-worker 进程后 Run 转入 unknown 并可恢复;取消后无残留子进程/连接/临时文件;manifest 通过 assertPluginManifest 与 registry 版本链。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. 分阶段实施计划
|
||||||
|
|
||||||
|
> 原则:先 RPA 侧收口(便宜、不依赖对面),再垂直切片,再规模化。不为了架构统一搞大重构;每阶段有硬验收门。
|
||||||
|
|
||||||
|
| 阶段 | 内容 | 前置 | 验收门 |
|
||||||
|
|---|---|---|---|
|
||||||
|
| **0 事实确认**(双侧,1 次评审会) | 向对面同步 6 项已拍板结论(R1-R6)并预估 C# actuate 扩展量;确认 2 个小问题(Q1 platform 字段 / Q2 utilityExtended 权限归属);确认事件流接口与 rpa.browser 清单备案 | 无 | Q 有书面结论 |
|
||||||
|
| **1 RPA 侧收口**(本仓库)✅ **已完成(2026-11-06,提交 9b76dc6/1da0082/1a20dd7/5e8fc96)** | A3 四件(取消贯通/状态映射/重试元数据/Graph Validator)+ A1 注入参数 + A9(Electron 残留、check-deps 升级、依赖闭包)+ A2 的 ADR 固化 + catalog 脏值清洗 + rpa-cli 重写(显式注册、经 engine 取 handler) | 无 | 审查报告 §7 第一/二阶段清单全绿;纯 Node 环境 Core 全量加载 |
|
||||||
|
| **2 迁入与 adapter 骨架** | git subtree 迁包改名 @agent-worker/rpa-core;bun@1.4.3 test 冒烟(exceljs/pdf/PowerShell 编码);新建 @agent-worker/rpa-adapter(CapabilityRunner + ref 映射 + 状态映射表);A++ 事件负载字段设计定稿(nodeId/检查点/幂等键,见 §7-A5);demo 侧冻结 | 阶段 1 + ADR-R1/R2 | bun test 纯逻辑全绿;未经 gateway 不可达 handler;719 映射完整性测试 |
|
||||||
|
| **3 垂直切片** | 一个文件能力 + 一个 HTTP 能力走通:workflow-engine create/start → gateway authorize → adapter → rpa-core handler → 事件/证据回传 → cancel 中途取消;方案 A(rpa.flow.execute 整体注册)+ A++ 检查点续跑最小验证(崩溃→resumeToken 重入,见 §7-A5) | 阶段 2 | 取消/审批/审计全链有证据;对面 Run 终态与 RunResult 一致 |
|
||||||
|
| **4 浏览器与桌面双 ADR 落地** | ADR-R4(已拍板):RPA 浏览器能力修完 transport/resolver,接对面白名单与证据体系后作为 RPA 场景主力 runner;ADR-R3:windows-runner C# 扩展 actuate(RPA 五域逻辑移植) | 阶段 3 | §7-A6/A7 验收条目 |
|
||||||
|
| **5 能力规模化** | 34 域按风险分批:files/data/http/ftp → office/excel/word(COM 仅 win32)→ desktop/utility 高影响(末批+审批);CapabilityDefinition 与权限映射随批生成 | 阶段 4 | 每批过 gateway 全量冒烟 + 审批矩阵抽查 |
|
||||||
|
| **6 平台扩展** | V1 第三节三端计划原样生效(Linux/macOS 驱动按 ADR-R3 原生通道) | 阶段 5 | 三平台 CI 矩阵;产品按 platformTag 置灰 |
|
||||||
|
|
||||||
|
**不搬的东西**(维持 V1 清单):界面渲染层、悬浮框/取色器、本地视觉 onnxruntime、流程存储、内置浏览器编排——由宿主提供或对面重做;其中"本地视觉"明确**不进 backend**(原生依赖 + 桌面截图依赖),留在宿主桌面进程。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9. 风险清单与总验收标准
|
||||||
|
|
||||||
|
### 风险清单
|
||||||
|
|
||||||
|
| # | 风险 | 等级 | 缓解 |
|
||||||
|
|---|---|---|---|
|
||||||
|
| R1 | ~~桌面安全模型冲突~~ → **已缓解(ADR-R3 拍板 C# 路线)**:残余风险转为 C# 移植工作量与工期 | 中 | 阶段 0 与对面确认 actuate 扩展量;RPA 五域逻辑先行整理成移植清单 |
|
||||||
|
| R2 | ~~部署拓扑错配~~ → **已缓解(ADR-R6 拍板按域拆分)**:残余风险为宿主进程加载子集的工程实现 | 中 | 阶段 2 按 PLATFORM_GROUPS 分组做宿主子集加载验证 |
|
||||||
|
| R3 | Registry 全局单例在多租户 backend 的隔离缺陷 | 高 | 短期按 ADR-R1 限定部署模型;长期 engine-scoped registry 列入 backlog |
|
||||||
|
| R4 | 浏览器双路径重复投资 | 低(已缓解) | R4 已拍板按场景分工(RPA=CDP 接管,AI 智能体=Playwright),白名单与证据共用一套,无重复投资 |
|
||||||
|
| R5 | 719 能力的治理映射与元数据补齐工作量失控 | 中 | 脚本从 catalog 生成初稿 + 人工只审高风险域;分批(阶段 5) |
|
||||||
|
| R6 | Bun 与 Node 行为差异(exceljs/pdf-lib/PowerShell 编码) | 中(部分缓解) | exceljs/pdf 已于 2026-11-06 本机 bun 1.4.4 冒烟通过(37/37);PowerShell 编码差异留待阶段 2 迁包后清单化 |
|
||||||
|
| R7 | demo 侧与 backend 双活跃漂移 | 中 | git subtree + "迁移后 demo 冻结"纪律(V1 决策 3) |
|
||||||
|
| R8 | 两套状态机语义漂移(NodeState vs StepStatus) | 中 | 映射表 + 契约测试常驻 CI;A++ 检查点协议(ADR-R2 补记)统一崩溃恢复语义 |
|
||||||
|
| R9 | catalog 源数据(shizai)上游已删,重生成受阻 | 低 | 已从对面历史恢复至 temp/catalog-source;计划快照入库 |
|
||||||
|
| R10 | 对面底座继续快速演进,本文行号/接口过期 | 中 | 本文标注核查日期;联调前对关键接口做增量 diff |
|
||||||
|
|
||||||
|
### 总验收标准(迁移完成的定义)
|
||||||
|
|
||||||
|
1. Bun@1.4.3 下包 typecheck/test 全绿(不以 Node 测试替代)。
|
||||||
|
2. FakeDriver/Host 注入可用,双 Engine 隔离行为符合 ADR-R1。
|
||||||
|
3. 全部已暴露能力经 capability-gateway:审批拒绝/租约过期/无权限时 handler 零调用,审计四元组(Agent/Run/Capability/审批)可追踪。
|
||||||
|
4. 取消/超时/重试测试证明:超时即中止底层、取消不进 error route、非幂等零自动重试、无并发重入。
|
||||||
|
5. Graph Validator 拒绝非法流程图于执行前。
|
||||||
|
6. 纯 Node/Bun 环境(无 electron 可解析)Core 全量加载;视觉路径 Host Mock 可测。
|
||||||
|
7. 隔离目录按包声明安装依赖,代表能力(文件/Excel/HTTP/PDF)可跑。
|
||||||
|
8. 垂直切片端到端演练通过:含一次审批、一次中途取消、一次进程崩溃恢复(unknown→恢复)。
|
||||||
|
9. Windows 10/11 实机回归:UIA/浏览器/Office/DPI/多屏/安装包资源 + 取消场景。
|
||||||
|
10. catalog 1016 项全过平台 schema 校验;能力平台范围与真实实现一致。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 10. 决策与待确认清单
|
||||||
|
|
||||||
|
### 已拍板(2026-11-06,本方负责人确认)
|
||||||
|
|
||||||
|
| ADR | 问题 | 决定 |
|
||||||
|
|---|---|---|
|
||||||
|
| R1 | Registry 作用域:backend 是否多租户并存 | **全局一份能力清单**(菜单所有租户共用),权限隔离完全由 capability-gateway 按租户管;长期将清单挂到 engine 实例(已从对面源码确认 WorkflowIdentity 带 tenantId,但清单本身无需按租户复制) |
|
||||||
|
| R3 | 桌面执行安全模型:RPA PowerShell 驱动 vs 对面"受信 C# 原生进程"红线 | **C# 扩展 actuate**:把 RPA 五域执行逻辑移植进 windows-runner 原生进程;PowerShell 通道仅限本 demo,不进宿主。**原型已产出并编译验证(2026-11-06)**:`contrib/windows-runner-actuate/` 补丁包(协议扩展 + NativeInput + ActuateService + 逐原语移植对照表 + 合入指引含门禁合规清单);对面 csproj 原样合编 0 警告 0 错误(.NET 10.0.403),协议冒烟 5 例通过(安全门错误码/双路分发/observe 无回归);对面 README Windows 行"click/type 或随本轮交付"与该路线吻合 |
|
||||||
|
| R4 | 浏览器双路径 | **按场景分工**:RPA 场景走咱们的 CDP 接管路线(复用用户登录态/真实页面——RPA 核心价值);对面 PlaywrightBrowserRunner 留给 AI 智能体场景(无头自管浏览器)。约束:①咱们 CDP 路线必须接对面 BrowserContextPolicy 白名单与证据体系,不自建;②迁移前修完 transport 超时与定位器不统一两缺陷;③需对面在插件清单给咱们的 runner 备案(rpa.browser) |
|
||||||
|
| R5 | 包消费方式 | **源码并入 + 锁版本**:git subtree 进对面仓库统一编译,Bun 锁 1.4.3(与对面全家 workspace:* 源码消费现状一致) |
|
||||||
|
| R6 | 部署拓扑:rpa-core 各域跑在哪 | **按能力域拆分部署**:纯 Node 域→backend plugin-worker;桌面/视觉/弹窗域→用户电脑 Electron 宿主;同一包按 PLATFORM_GROUPS 分组加载,不维护两份代码。**补记(2026-11-06 用户拍板:双端并存+透明调度)**——同一能力本地/后端两驻地并存,SaaS 与本地部署两形态通用,用户无感知;防乱设计见下方专条 |
|
||||||
|
| R2 | 集成形态:RPA 流程=单个 capability(方案 A)还是节点=step 映射(方案 B) | **方案 A + 两补丁**:RPA 流程=`rpa.flow.execute` 单能力,对面一个 step 一次租约一次结算;节点级事件走 event-bus 旁路(保透明、不进权威账本);人审/高危节点照常升级对面审批。**不用 B 的理由**(记录在案):双账本一致性(崩溃窗口两边分歧、破坏对面 recovery 的单账本假设)、控制流节点无副作用无法映射 CapabilityRef、审批/重试粒度与业务语义错位。**B 的触发条件**:对面产品确认需要"逐步控制 RPA"(单步执行/任意步回退)时局部启用,事件流已备好数据不返工。**A++ 补记(2026-11-06 二次拍板)**:方案 A 骨架不变,节点事件流从"透明度日志"升级为**检查点协议**(nodeId+状态+幂等键+可恢复标记),崩溃后 `rpa.flow.execute` 带 resumeToken 重入、从最近检查点续跑,权威账本仍为单 step、无双账本;B 的隐藏触发条件(长流程断点续跑,RPA 桌面场景高发)由此覆盖而无需节点=step。业界对照:UiPath Orchestrator=Job 黑盒+日志旁路(≈A);Camunda/Zeebe External Task 的"单引擎假设"不适用于双引擎集成;AWS Step Functions callback≈补丁 2。两侧改动清单与落地节奏见 §7-A5。**租约粒度三次拍板(2026-11-06:每个节点执行都要慎重)**:外层单 step 不变,内层**每节点执行前单独 authorize+execute**(60s 租约/节点),高危节点逐节点审批——节点级治理粒度 + step 级账本粒度两全,实现方式见 §7-A4 |
|
||||||
|
| V1 | 能力版本管理 | **adapter 层承接对面硬约束(2026-11-06 设计定稿)**:对面三处硬约束(调用必带 capabilityVersion 且不符即拒 / 幂等键含版本 / manifest 强制三段 semver);我方 catalog 997 项带四段版本但注册表无版本机制。定案:对外能力 version=rpa-core 包 semver(仅对外行为变化才升版);catalog 四段版本归一化三段;configVersion=catalog+overrides 内容哈希。详见 §7-A4;对迁包零影响 |
|
||||||
|
|
||||||
|
### 待对面对确认(仅剩小问题,不猜测)
|
||||||
|
|
||||||
|
| Q# | 问题 | 评审要点 |
|
||||||
|
|---|---|---|
|
||||||
|
| Q1 | CapabilityDefinition 是否需要 platform 字段(对面清单未见) | **本方设计提案(2026-11-06)**:对面 plugin-contract 增可选 `platforms?: readonly ("win32"\|"darwin"\|"linux")[]`,缺省=全平台(旧清单零破坏);registry 的 `assertPluginManifest` 校验值域;plugin-worker 认领动作时按宿主 `process.platform` 过滤。RPA 侧清单由 rpa-adapter 从 catalog `platformTag` 自动生成,不手工维护第二份。与对面确认后照此实现 |
|
||||||
|
| Q2 | utilityExtended(命令执行)对应哪个 PermissionScope | **本方设计提案(2026-11-06)**:utilityExtended 域在对面源码无同名物,属纯 RPA 侧概念。提案映射到 `desktop.process`——RPA 命令执行发生在用户电脑 Electron 宿主内,宿主即桌面进程,语义吻合;该域标 highImpact + idempotencyRequired 走审批。若对面认为 desktop.process 严格指"桌面进程管理",备选新增 `desktop.command` scope。13 项 PermissionScope 其余映射维持 §7-A4 初稿 |
|
||||||
|
| Q4 | 高危审批的"上游服务"成熟度 | gateway 只按 actionId 精确放行,注释明言"审批有效性判定(是否过期、是否本人审批)由上游负责"(capability-gateway index.ts:13-14),但该上游服务的实现与闭环未见。RPA 人审弹窗方案(human-dialog→requestApproval→绑定 actionId)依赖它,须确认其签发/过期/防代批机制现状。**源码核查(2026-11-06):上游已实现非设计**——即服务端 workflow-engine 包:approval-policy 纯函数(actionId=runId:stepId:attempt 重试不被旧审批放行 / expiresAt 过期判定 / 仅 high 风险可发起)+ workflow-api 路由(request_approval / decide_approval,403 发起人不能审批自己=防代批、409 缺审批、410 已过期)+ Postgres 持久化;桥=plugin-worker 取 approvedActionIds 喂 gateway。缺口:审批人身份仍为开发态 token(PRODUCT_APPROVER_TOKEN,生产形态未接完);RPA 挂接的对齐点见下方"审批对象三层"专条 |
|
||||||
|
| Q5 | 租户权限配置面 | 每租户/员工的 `allowedPermissions` 白名单由谁配置、走哪条链路(entitlement-contract / identity-store 成熟度),adapter 对接前须核一眼 |
|
||||||
|
|
||||||
|
> 已回答:Q3(human-dialog 弹窗落点)已由 R6 回答——用户电脑 Electron 宿主窗口;仅样式/交互细节与对面 UI 对齐。
|
||||||
|
|
||||||
|
### 审批对象三层设计(2026-11-06 设计思考,待与上游对齐)
|
||||||
|
|
||||||
|
"审批什么、谁来批"须拆三层,混用会导致对面防代批机制(403)误伤合法场景:
|
||||||
|
|
||||||
|
| 层 | 对象 | 谁来批/定 | 时机与频率 | 载体 |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| ① 能力白名单 | 租户/员工能用哪类能力 | 管理员配置 | 事前一次,长期生效 |对面 Q5 链路(entitlement),**不产生审批单** |
|
||||||
|
| ② 流程上架 | 流程定义版本 + 它声明的能力清单 | 上游审批人 | 事前一次,多次运行复用 | 审批单(或管理端流程发布流程,待对齐用哪个) |
|
||||||
|
| ③ 单次高危动作 | 某次运行中的具体高危节点 | **分两种**:有人值守=用户知情同意;无人值守=上游审批人 | 事中,逐动作 | 有人值守=human-dialog 弹窗(consent,**不走审批单**,否则"自己批自己"被对面 403 挡死);无人值守=workflow-engine 审批单,运行暂停等批 |
|
||||||
|
|
||||||
|
推论(修正此前对齐清单):
|
||||||
|
- **actionId 粒度问题(attempt 维度)只存在于无人值守路径**;有人值守路径由"租约 60s + 每节点弹窗"兑现(§R2 三次拍板),与审批单无关。
|
||||||
|
- 日常有人值守运行**不需要**每次调 workflow-api;只有定时/远程触发的无人值守高危动作才走 request_approval 暂停等批——这正面回答了"发起方通道"问题。
|
||||||
|
- 对上游的对齐问题收敛为:②流程上架审批放哪个系统;③无人值守路径复用 workflow-engine 时 actionId(runId:stepId:attempt)与 rpa-core 幂等键(runId:nodeId)的映射(nodeId:attempt 是否补入 RunEvent);审批人身份生产化(trusted-gateway-headers / 管理端账号)。
|
||||||
|
|
||||||
|
### R6 补记:双端透明调度设计(2026-11-06 用户拍板:前端留、后端也留)
|
||||||
|
|
||||||
|
同一能力允许本地(用户电脑宿主)与后端(plugin-worker)两驻地并存,调度对用户无感知,SaaS 与本地部署两形态通用(本地部署时"后端"即客户自有机房,规则不变)。防乱靠三条纪律与一张确定性路由表:
|
||||||
|
|
||||||
|
**三条纪律:**
|
||||||
|
|
||||||
|
1. **一个脑决策**:路由只在宿主侧 Resolver 一处发生(本质是 A1 注入机制的扩展——registry 条目的 handler 绑定二选一:本地实现 or 自动生成的远端代理,远端代理走对面 gateway 授权/租约/审计,不绕行)。能力代码与流程定义里**禁止**出现任何驻地 if-else,规则全部来自 catalog 元数据。
|
||||||
|
2. **一份契约两处实现**:能力签名/参数/返回/错误码同一套(catalog 单一源已保证);本地与远端实现跑同一套契约测试;远端代理自动生成而非手写第二份逻辑,杜绝漂移。
|
||||||
|
3. **数据不动原则**:能力在哪个驻地执行由"数据已经在哪里"决定(代码就数据,不搬数据)。需要搬数据时必须是流程中的显式节点(上传/下载),**路由器不得隐式搬运用户本地文件到后端**——用户无感知的是"在哪算",不是"数据去哪"(隐私红线)。
|
||||||
|
|
||||||
|
**路由规则表(按序匹配,确定性,不留裁量空间):**
|
||||||
|
|
||||||
|
| 序 | 条件 | 执行驻地 |
|
||||||
|
|---|---|---|
|
||||||
|
| 1 | platformTag 限宿主平台 / 需要桌面会话 / 弹窗人审 | 仅本地(无远端选项) |
|
||||||
|
| 2 | 输入数据在本地(本地文件路径) | 本地 |
|
||||||
|
| 3 | 输入数据在云端(对面 tenant-files 等) | 后端 |
|
||||||
|
| 4 | 数据两驻地均有副本 | 默认本地(隐私+省带宽);流程可显式声明 `prefer: backend`(如夜间批量) |
|
||||||
|
| 5 | 本地不可用(宿主离线)且能力无本地数据依赖且标可远端 | 后端兜底 |
|
||||||
|
| 6 | 均不可用 | 节点失败,错误码 `location_unavailable` |
|
||||||
|
|
||||||
|
**用户无感知 ≠ 审计无感知**:`node.finished` 事件携带 `executor: "local" | "backend"` 字段(已定稿进 RunEvent 并有契约测试,当前恒为 `local`,远端代理落地后由代理覆盖)——用户界面不展示,审计与排障必须可分辨。该字段同时是对面 event-bus 入审计链的取用点。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 11. 本指南边界
|
||||||
|
|
||||||
|
- 全部结论基于 2026-11-06 双侧工作区源码静态核查 + 少量脚本实测(正则盲区、catalog 统计、capabilityCount);未执行 Windows 实机 E2E 与对面联调,未运行的测试不写成通过。
|
||||||
|
- 行号会随代码演进过期,以函数/接口名为准。
|
||||||
|
- 本轮只改文档,不改业务源码(用户指令);阶段 1 起的代码改动另行排期。
|
||||||
184
docs/rpa-core-migration-guide.md
Normal file
184
docs/rpa-core-migration-guide.md
Normal file
@ -0,0 +1,184 @@
|
|||||||
|
# rpa-core 迁入 Agent-Employee 指南(通俗版)
|
||||||
|
|
||||||
|
## 这是一份什么文档?
|
||||||
|
|
||||||
|
我们花了两周把 RPA 的"能力层"(跑流程用的几百个能力 + 流程引擎)从现在这个 demo 项目里抽出来,
|
||||||
|
做成了一个独立软件包,叫 `@rpa/core`,放在本仓库的 `packages/rpa-core/` 目录。
|
||||||
|
|
||||||
|
**为什么要抽出来?** 因为这个包以后要搬到另一个项目 Agent-Employee 里用。
|
||||||
|
抽成独立包的好处:搬过去时不用改一行代码,两边也不用维护两份一样的代码。
|
||||||
|
|
||||||
|
**这份文档讲什么?** 讲三件事:
|
||||||
|
1. 这个包现在长什么样(现状)
|
||||||
|
2. 到时候怎么搬过去(操作步骤)
|
||||||
|
3. 以后要支持 Mac 和 Linux 怎么办(三端计划)
|
||||||
|
|
||||||
|
> 配套文档:`docs/rpa-core-refactor-tasks.md`(当时的改造任务清单)、`docs/dev-conventions.md`(日常开发规范)、`packages/rpa-core/README.md`(包的技术说明书)。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 一、现状:包已经是什么样了
|
||||||
|
|
||||||
|
改造已全部完成并合入 main 分支(6 个阶段提交,`4139400..a8084dc`)。现在:
|
||||||
|
|
||||||
|
| 你可能想知道的 | 答案 |
|
||||||
|
|---|---|
|
||||||
|
| 包在哪? | `packages/rpa-core/`,包名 `@rpa/core` |
|
||||||
|
| 里面有什么? | 流程引擎(跑流程的循环/变量/错误处理)+ 719 个能力(文件/Excel/HTTP/数据库/邮件/桌面操作等) |
|
||||||
|
| 它依赖 Electron 吗? | **完全不依赖**。有脚本 `npm run check:deps` 天天检查,谁往里加 Electron 代码就报错 |
|
||||||
|
| 它怎么用 Electron 的功能? | 见下面"宿主桥"的解释 |
|
||||||
|
| 怎么证明它能独立跑? | `npm run rpa-cli`:不启动 Electron,直接跑通 5 个代表能力(桌面点击/Excel/HTTP/文件/屏幕) |
|
||||||
|
|
||||||
|
**几个名词解释(后文会反复出现):**
|
||||||
|
|
||||||
|
- **宿主**:使用这个包的应用程序。现在宿主是本 demo(Electron 应用),以后宿主是 Agent-Employee。
|
||||||
|
- **宿主桥(HostBridge)**:包偶尔需要"只有应用程序才有"的功能,比如弹一个窗口、读取 AI 配置。
|
||||||
|
包不直接去拿,而是定了一个接口说"谁来用我,谁把这几个功能递给我"。
|
||||||
|
宿主启动时调用 `setHostBridge(...)` 把功能递进来。目前一共只有 4 个功能要递:
|
||||||
|
弹窗、两组屏幕坐标换算、读 AI 配置。另有 `setSubflowSource(...)` 递子流程仓库(2 个方法)。
|
||||||
|
- **驱动(Driver)**:真正干"动鼠标、敲键盘、找窗口"这些活的模块。现在的实现叫 `WindowsDriver`(用 PowerShell 实现)。
|
||||||
|
能力层不直接碰 PowerShell,而是喊"driver,帮我把鼠标移到 x,y"——所以以后要做 Mac 版,只需要再写一个 `MacDriver`,能力层一行不用改。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 二、到时候怎么搬:三步走
|
||||||
|
|
||||||
|
### 第 -1 步:先知道对面已经有什么(2026-11-05 实地调研)
|
||||||
|
|
||||||
|
Agent-Employee(`D:\code\Agent-Employee`)是 Electron + SolidJS 客户端 + Bun 执行面 + Java 控制面的结构,
|
||||||
|
前端 `code/frontend`、后端 `code/backend` 是**两个独立的 bun workspace**(bun@1.4.3,workspaces 含 `apps/*`、`packages/*`、`services/*`)。
|
||||||
|
|
||||||
|
关键事实:**对面已经为我们要搬的东西铺好了契约,但还没有实现**。`code/backend/packages/` 下有一批契约包(每个只有 1-9 个声明文件):
|
||||||
|
|
||||||
|
| 对面现有契约包 | 它声明了什么 | 与 @rpa/core 的关系 |
|
||||||
|
|---|---|---|
|
||||||
|
| `workflow-engine` | workflow / step / approval / failure 状态模型(9 个文件,依赖 plugin-contract、capability-gateway、event-bus) | rpa-core 的 runtime 是它的**执行引擎候选**,迁入时需做状态模型对齐(审批/失败/证据如何挂到我们 RunResult/NodeState 上) |
|
||||||
|
| `capability-gateway` | 所有带副作用的能力执行前必须过:授权 → 租约签发 → 租约校验 → 执行约束;契约含 `CapabilityRunner` / `authorize()` / `execute()` | **权限治理(6.4)的现成落点**。rpa-core 能力执行要包进这个网关,而不是自建 allowlist |
|
||||||
|
| `computer-control` | 跨平台观察、定位、动作抽象与桌面会话策略 | rpa-core 五域 driver 是它的**实现候选**;三端设计(第三节)应对齐这份契约 |
|
||||||
|
| `windows-plugin` | Windows 目标、桌面会话策略、输入类高影响能力、执行器接口 | WindowsDriver 的接入契约 |
|
||||||
|
| `browser-plugin` | 浏览器动作、域名白名单、高影响能力审批边界、证据机制 | demo 侧浏览器能力的对面契约 |
|
||||||
|
| `runtime-governance` / `plugin-contract` | 运行时治理 / 插件契约 | 治理对接的参考 |
|
||||||
|
| `services/windows-runner` | Windows 交互 Runner(**空壳,0 个源文件**) | 对面 README 已明确"Windows 交互 Runner 留在宿主,不放入 Linux 容器"——与 rpa-core 桌面能力只能跑在 Windows 交互会话的约束天然一致 |
|
||||||
|
|
||||||
|
**结论:迁入的本质不是"新写一套宿主适配",而是"拿 rpa-core 的实现去填对面的契约"。**
|
||||||
|
这比原计划省了适配层设计,但多了一步契约对齐(接口签名互相对照,差异列清单评审)。
|
||||||
|
|
||||||
|
### 第 0 步:搬家之前,在 demo 里先补三件作业
|
||||||
|
|
||||||
|
这些事当时刻意延后了,搬家前必须补上(都在本仓库做,改起来便宜):
|
||||||
|
|
||||||
|
1. **Bun 冒烟测试**。Agent-Employee 用 Bun 跑代码,我们一直用 Node。
|
||||||
|
搬过去第一件事:用 `bun test` 跑包里的纯逻辑测试,确认 exceljs、pdf-lib 这些库在 Bun 下行为一致,
|
||||||
|
尤其是调 PowerShell 的部分(编码处理容易有差异)。
|
||||||
|
2. **重新生成能力目录**。跑一次 `npm run catalog`,让每个能力带上 `platformTag: "win32"` 标记
|
||||||
|
(标记逻辑早就写好了,当时因为数据源不在这台机器上没生效)。
|
||||||
|
3. **决定用哪种注册方式**。包支持两种注册能力的方式:
|
||||||
|
旧方式 `import "@rpa/core/capabilities"`(一导入全注册),新方式 `await registerAll(engine)`(想注册哪几组就注册哪几组)。
|
||||||
|
建议 Agent-Employee 直接用新方式,旧的以后可以废掉。
|
||||||
|
|
||||||
|
### 第 1 步:搬包(半天)
|
||||||
|
|
||||||
|
1. **评审**:按 Agent-Employee 团队的流程提交一份架构决策记录(ADR),内容就是本文档第一节 + 对面契约对照表 + 包的 README,不用另写。
|
||||||
|
2. **搬代码**:把 `packages/rpa-core/` 整个目录用 git 方式搬进 **`code/backend/packages/rpa-core`**
|
||||||
|
(执行面归 backend workspace;对面 bun@1.4.3 的 workspaces 已含 `packages/*`,放进去即生效)。
|
||||||
|
用 `git subtree` 之类保留提交历史,**千万不要复制文件夹另起炉灶**——两份代码会越走越远。
|
||||||
|
3. **包名决策**:对面 backend 包统一用 `@agent-worker/*` 前缀(如 `@agent-worker/plugin-contract`)。
|
||||||
|
建议迁入时改名 `@agent-worker/rpa-core`(一次 commit 完成 rename,包内代码不用动)。
|
||||||
|
4. **处理 demo 侧**:搬走之后 demo 要么改成只读引用,要么归档。**两边不能同时活跃改代码。**
|
||||||
|
|
||||||
|
### 第 2 步:契约对齐 + 宿主适配(核心工作量)
|
||||||
|
|
||||||
|
分两类工作:
|
||||||
|
|
||||||
|
**A. 契约对齐(对面已铺好,我们填实现)**——逐个对照,差异列清单评审后改:
|
||||||
|
|
||||||
|
| 对面契约 | 对齐内容 |
|
||||||
|
|---|---|
|
||||||
|
| `capability-gateway` | 把 rpa-core 的能力执行包进 `authorize() / execute()` 流程(`CapabilityRunner` 接口);危险能力(System_ExecuteCommand、键鼠注入等)走它的租约与审批,**不再自建 allowlist** |
|
||||||
|
| `workflow-engine` | 状态模型对齐:我们 `RunResult/NodeState`(running/succeeded/failed/...)对照它的 workflow/step/approval/failure;人工审批节点(human-dialog)要能挂进它的 approval 流 |
|
||||||
|
| `computer-control` / `windows-plugin` | 五域 driver 的观察/定位/动作抽象与它们契约互相对照;WindowsDriver 作为执行器接入 |
|
||||||
|
| `browser-plugin` | 浏览器能力对齐域名白名单与审批边界 |
|
||||||
|
|
||||||
|
**B. 宿主注入(照旧要做,但工作量变小)**:
|
||||||
|
|
||||||
|
| 要递什么 | 干什么用 | 工作量 |
|
||||||
|
|---|---|---|
|
||||||
|
| `setHostBridge` 4 个方法 | 弹窗、DPI 坐标换算、读 AI 配置 | 小。对面也是 Electron(`code/frontend/apps/desktop`),参考实现照抄本仓库 `src/main/index.ts` 第 28-50 行 |
|
||||||
|
| `setSubflowSource` 2 个方法 | 子流程的存与取 | 小。对面有自己的流程存储,包一层即可 |
|
||||||
|
| **内置浏览器**(`BrowserLike` 接口) | "浏览器打开网址/点按钮/填表单"这组能力需要宿主提供浏览器 | **不确定,是最大变数**。对面的浏览器方案还没定型(browser-plugin 只是契约);需与对面团队评审后再估工作量 |
|
||||||
|
| AI 能力配置 | `ai-caps.ts` 需要 baseUrl/apiKey 等 | 小。对面有模型网关(model-gateway),接它的配置面 |
|
||||||
|
|
||||||
|
**不搬的东西**(留在 demo / 由 Agent-Employee 重新做):
|
||||||
|
界面渲染层、悬浮框/取色器、本地视觉(onnxruntime 那套)、流程存储、内置浏览器编排。
|
||||||
|
完整清单见包 README 的「Electron 相关能力留守清单」。
|
||||||
|
|
||||||
|
### 第 3 步:验收(搬家算不算成功,看这 7 条)
|
||||||
|
|
||||||
|
1. `bun test` 包内纯逻辑测试全绿;
|
||||||
|
2. `npm run rpa-cli` 的等价物在 Bun 下跑通 5 个代表能力;
|
||||||
|
3. 能力注册数对得上(包内 719 个);
|
||||||
|
4. `check:deps` 通过(没人在搬家过程中偷偷加回 Electron);
|
||||||
|
5. 真实操作一遍:弹一次人机对话框、高分屏上拖一次窗口、调一次 AI——验证宿主桥三条路径通了;
|
||||||
|
6. 契约对齐有书面结论:capability-gateway(能力执行进网关)、workflow-engine(状态模型)、computer-control(driver 抽象)三份对照清单各评审通过;
|
||||||
|
7. demo 侧(如果还保留)测试全绿。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 三、以后要支持 Mac 和 Linux 怎么办?
|
||||||
|
|
||||||
|
好消息:当初设计时已经把口子留好了——能力层只对"驱动接口"说话,不关心底下是 Windows 还是 Mac。
|
||||||
|
所以三端化不用推翻任何东西,分两批做:
|
||||||
|
|
||||||
|
### 搬家之前(demo 侧,5 件小事)
|
||||||
|
|
||||||
|
| # | 事项 | 说明 |
|
||||||
|
|---|---|---|
|
||||||
|
| 1 | **平台标记统一机制**(三端优先 win) | 平台归属收敛为一处权威数据:`register-all.ts` 分组表 `PLATFORM_GROUPS`。三个平台类型都能填,但**数据只填 win32**,darwin/linux 仅类型占位;`meta.platforms`(运行时)与 catalog `platformTag`(构建期)都从这张表派生,`build-catalog.mjs` 删掉自带的 `WIN32_GROUPS` 硬编码;加一致性守卫测试,两处不一致 CI 直接红 |
|
||||||
|
| 2 | 驱动改成按系统自动选 | 现在写死 `new WindowsDriver()`;改成看 `process.platform` 自动选,Mac/Linux 上没实现就明确报"该平台暂不支持"(加个 `platform_unsupported` 错误码) |
|
||||||
|
| 3 | 检查能力分组 | Excel COM、Word COM 这些 Windows 专属能力,确保归在能整组过滤的分组里——将来 Mac 版就不注册这组 |
|
||||||
|
| 4 | 测试加平台守卫 | 现在只有 4 个测试文件写了"非 Windows 就跳过",键盘/鼠标/窗口冒烟测试没写。补齐后,Mac/Linux 的 CI 才能跑(跑纯逻辑部分,跳过真桌面操作) |
|
||||||
|
| 5 | 开发规范加一条 | 禁止在能力文件里随手写 `process.platform` 判断(现在 utility-extended 里已有 2 处,收拢到固定位置),防止平台差异到处扩散 |
|
||||||
|
|
||||||
|
### 搬家之后(Agent-Employee 侧,按平台写驱动)
|
||||||
|
|
||||||
|
0. **先对齐对面的 `computer-control` 契约**:对面已声明"跨平台观察、定位、动作抽象"的契约(还是空壳)。
|
||||||
|
写 MacDriver / LinuxDriver 之前,先让五域 driver 接口与这份契约对表,避免两边各长各的。
|
||||||
|
1. **写 MacDriver / LinuxDriver**(接口现成,按五个域补实现):
|
||||||
|
- Linux:鼠标键盘用 xdotool(X11);Wayland 支持受限,是三端里最难的坑;界面元素识别走 AT-SPI;
|
||||||
|
- Mac:鼠标键盘用 cliclick 或 CGEvent;窗口操作走 AppleScript;界面元素走 AX 辅助功能 API。
|
||||||
|
2. **先评估 macOS 权限问题**:Mac 上 RPA 软件必须引导用户开"辅助功能/屏幕录制"授权(系统 TCC 机制),
|
||||||
|
这是 Mac 端最大的部署摩擦,做之前先摸清。
|
||||||
|
3. **Office 替代方案**:Excel/Word 的 COM 自动化只有 Windows 有。Mac 可走 AppleScript + Office,
|
||||||
|
Linux 可走 LibreOffice;或者第一版直接标注"仅 Windows"。
|
||||||
|
4. **浏览器路径**:外部浏览器启动路径现在写死了 Windows 的(`C:\Program Files\...`),要补 Mac/Linux 的路径表。
|
||||||
|
5. **剪贴板等小件**:PowerShell 换成 `pbpaste/pbcopy`(Mac)、`xclip`(Linux)。
|
||||||
|
6. **CI 三个平台各跑一遍**;产品界面按能力标注的平台支持情况置灰不支持的项。
|
||||||
|
|
||||||
|
### 一条原则
|
||||||
|
|
||||||
|
**平台标记必须统一,这是硬约束(三端优先 win):**
|
||||||
|
|
||||||
|
- 平台归属只有**一处权威数据**:包内注册分组表 `PLATFORM_GROUPS`(33 个能力组 × 平台数组)。
|
||||||
|
- 运行时的 `meta.platforms` 和构建期的 catalog `platformTag` 都**从这张表自动派生**,任何代码不得自行声明第二份平台数据。
|
||||||
|
- **数据里今天只有 `win32`**;`darwin`/`linux` 只存在于类型枚举里占位,不填任何数据。
|
||||||
|
- 将来支持新平台 = 新增能力分组 + 表里加一行,仅此而已;守卫测试保证两处派生永远一致,谁绕过就红。
|
||||||
|
|
||||||
|
平台差异**只允许**出现在两个地方:`driver/<平台>/` 目录下的驱动实现,以及极少数白名单能力文件。
|
||||||
|
引擎、注册表、能力接口、宿主桥,永远保持平台无关。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 附:当时的 10 个关键决策(搬家时不要再讨论一遍)
|
||||||
|
|
||||||
|
| # | 决策 |
|
||||||
|
|---|---|
|
||||||
|
| 1 | 代码物理搬移(git mv),不是留着引用 |
|
||||||
|
| 2 | 保留全局注册表;进程级状态收进 `createEngine()` 工厂 |
|
||||||
|
| 3 | 集中突击改造,不搞长期双轨 |
|
||||||
|
| 4 | 宿主桥只留 4 个方法(弹窗 / DPI×2 / AI 配置),能不加就不加 |
|
||||||
|
| 5 | 浏览器:外部浏览器(CDP)进包,内置浏览器留宿主 |
|
||||||
|
| 6 | 驱动接口五域划分(鼠标/键盘/窗口/屏幕/元素),Windows 实现原样包装、行为不变 |
|
||||||
|
| 7 | 能力目录只填平台标记数据,界面置灰以后再说 |
|
||||||
|
| 8 | 能力元数据(meta)老能力可选、新能力强制 |
|
||||||
|
| 9 | 注册双入口过渡(副作用导入 + registerAll 显式注册) |
|
||||||
|
| 10 | Bun 冒烟测试延期到搬家时做(本文第二节第 0 步) |
|
||||||
116
docs/rpa-core-refactor-tasks.md
Normal file
116
docs/rpa-core-refactor-tasks.md
Normal file
@ -0,0 +1,116 @@
|
|||||||
|
# rpa-core 能力层抽离改造任务清单
|
||||||
|
|
||||||
|
> 目标:把 `src/main/capabilities/` + `src/main/engine/` 抽成零 Electron 依赖的独立包,
|
||||||
|
> 先在当前 demo 验证,最终以包为单位迁入 `D:\code\Agent-Employee`。
|
||||||
|
> 风险评估见对话记录(2026-11-04),任务编号 R1~R10 对应风险清单。
|
||||||
|
|
||||||
|
## 总体原则
|
||||||
|
|
||||||
|
1. **单一源**:包建成后 demo 与未来宿主都通过 workspace 引用,任何时候禁止复制代码(R3)。
|
||||||
|
2. **行为不变**:抽离期间能力行为保持不变,靠现有 vitest 套件做安全网。
|
||||||
|
3. **先断依赖再抽象**:Electron 依赖清零优先于 driver 接口美化。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 阶段 0 — 基线与护栏
|
||||||
|
|
||||||
|
- [ ] **0.1 确认基线全绿**
|
||||||
|
`npm run test` + `npm run typecheck` 全部通过,记录通过数作为回归基线。
|
||||||
|
- [ ] **0.2 确定包位置与工作区结构**
|
||||||
|
建 `packages/rpa-core/`,demo 的 `package.json` 加 workspaces 配置,源码从 `src/main` 移入后以 `@rpa/core` 引用回主进程。
|
||||||
|
- [ ] **0.3 生成能力目录快照**
|
||||||
|
用 `npm run catalog` 的产物做一份快照(能力 id / 参数 / 描述清单),抽离完成后 diff 必须为零,作为行为回归基准。
|
||||||
|
|
||||||
|
## 阶段 1 — 建包与纯 Node 能力搬运
|
||||||
|
|
||||||
|
- [ ] **1.1 包骨架**
|
||||||
|
`packages/rpa-core`:`package.json`(deps: exceljs / pdf-lib / nodemailer / basic-ftp / adm-zip / pngjs)、`tsconfig.json`、独立 vitest 配置(R6:包内保留 vitest,不迁 bun test)。
|
||||||
|
- [ ] **1.2 领域类型解耦(R2)**
|
||||||
|
搬 `engine/`(registry / types / runtime / expression),将 `shared/types` 依赖改为包内自有领域类型(FlowDefinition / FlowNode / CatalogComponent / RunError)。宿主后续做模型映射,不把画布模型塞进包。
|
||||||
|
- [ ] **1.3 全局状态收拢为工厂(R4)**
|
||||||
|
新增 `createEngine()` 工厂:excel-store 的模块级 `Map` 缓存、session 的 `globalElementWaitMs` 等进程级状态挂到引擎实例上;registry 注册目标从全局单例改为引擎实例。
|
||||||
|
- [ ] **1.4 搬运纯 Node 能力**
|
||||||
|
files / data / data-extended / http / ftp / mail / mssql / mysql / pg / excel(exceljs)/ word / pdf / office-file / time / crypto / data-table 等不依赖 Electron 的能力文件。
|
||||||
|
- [ ] **1.5 搬运对应测试**
|
||||||
|
上述能力的 `*.test.ts` 一并移入包内,路径修正后全量跑绿。
|
||||||
|
- [ ] **1.6 demo 主进程改为引用包**
|
||||||
|
`src/main` 里被搬走的部分改 import `@rpa/core`,`npm run test` 全绿、catalog 快照 diff 为零。
|
||||||
|
|
||||||
|
## 阶段 2 — 断 Electron 依赖
|
||||||
|
|
||||||
|
- [ ] **2.1 定义 HostBridge 接口**
|
||||||
|
包内声明 `HostBridge`:clipboard / dialog / screen / userAgentDataPath 等宿主能力;demo 的 Electron 主进程提供实现并注入。
|
||||||
|
- [ ] **2.2 改造直接依赖 Electron 的能力(3 处)**
|
||||||
|
`capabilities/human-dialog.ts`(BrowserWindow)、`capabilities/window-extended.ts`(screen)、`capabilities/ai-service.ts`(settings)→ 走 HostBridge 注入。
|
||||||
|
- [ ] **2.3 浏览器子系统切分(R1)**
|
||||||
|
`browser/external.ts` + `transport.ts` 的外部浏览器 CDP 路径搬进包;`session.ts` 中依赖内置 `BrowserWindow` 的部分留在宿主适配层,包内通过 `BrowserLike` 接口(types.ts 已有)注入。
|
||||||
|
- [ ] **2.4 防回归护栏**
|
||||||
|
在包内加依赖检查脚本(或 ESLint no-restricted-imports 规则):禁止 `import "electron"` 及对 `src/main` 的相对引用,CI 里跑。
|
||||||
|
- [ ] **2.5 Electron 相关能力留守清单**
|
||||||
|
overlay / desktop-pick / vision-lab / fixture server / human-dialog 的 UI 部分:明确标记"宿主适配层",从包中排除,记录在包 README。
|
||||||
|
|
||||||
|
## 阶段 3 — Driver 抽象(Windows 自动化能力)
|
||||||
|
|
||||||
|
- [ ] **3.1 拆出 powershell 助手(R5)**
|
||||||
|
`powershell()` / `psString()` / `USER32` 从 `capabilities/desktop.ts` 移到 `driver/windows/powershell.ts`;修正 11 个引用文件的 import;测试全绿。
|
||||||
|
- [ ] **3.2 定义 AutomationDriver 接口**
|
||||||
|
按域拆分:`mouse` / `keyboard` / `window` / `element`(UIA)/ `screen`。接口签名以现有 PowerShell 能力的输入输出为准,不重新发明。
|
||||||
|
- [ ] **3.3 现有实现包进 WindowsDriver**
|
||||||
|
desktop / desktop-uia / window-extended / remaining-core / system-extended / common-final / message / utility-extended 中的 user32/UIA 调用改为面向接口;`WindowsDriver` 内部仍用 PowerShell 实现(行为不变)。
|
||||||
|
- [ ] **3.4 driver 注入与平台标记**
|
||||||
|
引擎创建时注入 driver;能力目录(catalog)加 `platforms: ["win32"]` 字段,渲染层据此置灰。
|
||||||
|
- [ ] **3.5(可选,迁移前不阻塞)PowerShell 常驻 worker**
|
||||||
|
单进程 JSON 行协议替代每次 spawn;`powershell()` 接口签名不变,内部切换。性能优化项,放在功能等价验收之后。
|
||||||
|
|
||||||
|
## 阶段 4 — Tool 契约(为 Agent-Employee 做)
|
||||||
|
|
||||||
|
- [ ] **4.1 能力 schema 化**
|
||||||
|
每个能力导出结构化定义:参数类型 / 返回值 / 超时 / 错误码 / 是否需要截图回传。新能力一律按此格式写。
|
||||||
|
- [ ] **4.2 注册机制显式化**
|
||||||
|
取消 import 副作用注册,改为显式 `registerAll(engine)` 导出,宿主可按子集注册。
|
||||||
|
- [ ] **4.3 错误契约规范化(R9)**
|
||||||
|
RuntimeError 错误码枚举固化,中文 message 保留,增加机器可读字段,供 Agent 侧消费。
|
||||||
|
|
||||||
|
## 阶段 5 — 独立验证
|
||||||
|
|
||||||
|
- [ ] **5.1 CLI host**
|
||||||
|
写 `scripts/rpa-cli.ts`:Node 直接拉起 `createEngine()` + `WindowsDriver`,跑通 5 个代表能力(1 个桌面、1 个 Excel、1 个 HTTP、1 个文件、1 个窗口)。
|
||||||
|
- [ ] **5.2 Bun 冒烟(R6/R7)**
|
||||||
|
在 Agent-Employee 环境用 `bun test` 跑包内纯逻辑测试,验证 exceljs / pdf-lib 等依赖在 Bun 下正常;onnxruntime 相关暂不进包,另行验证。
|
||||||
|
- [ ] **5.3 demo 全量回归**
|
||||||
|
demo 切到 workspace 引用后:test + typecheck + catalog 快照 diff 三项全绿。
|
||||||
|
|
||||||
|
## 阶段 6 — 迁入 Agent-Employee
|
||||||
|
|
||||||
|
- [ ] **6.1 ADR 评审**
|
||||||
|
按 Agent-Employee 流程提交架构决策记录:包边界 / HostBridge 契约 / driver 生命周期。
|
||||||
|
- [ ] **6.2 包迁入 `packages/`**
|
||||||
|
整体搬入,改为 bun workspace 成员;demo 侧保留只读引用或解除引用。
|
||||||
|
- [ ] **6.3 宿主适配层**
|
||||||
|
按 Agent-Employee 的 product-shell 契约写 host 适配(clipboard / dialog / screen / BrowserLike 实现)。
|
||||||
|
- [ ] **6.4 权限与治理对接(R10)**
|
||||||
|
`System_ExecuteCommand` 等危险能力接入其权限模型;能力目录与插件体系对齐。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 风险 → 任务映射
|
||||||
|
|
||||||
|
| 风险 | 缓解任务 |
|
||||||
|
|---|---|
|
||||||
|
| R1 浏览器子系统耦合 Electron | 2.3 |
|
||||||
|
| R2 领域模型冲突 | 1.2、阶段 4 |
|
||||||
|
| R3 代码漂移 | 0.2、总原则 1 |
|
||||||
|
| R4 模块级全局状态 | 1.3 |
|
||||||
|
| R5 powershell 助手位置 | 3.1 |
|
||||||
|
| R6 测试运行器不兼容 | 1.1、5.2 |
|
||||||
|
| R7 原生依赖 Bun 兼容 | 5.2(能力层已确认为纯 JS) |
|
||||||
|
| R8 注册副作用 | 4.2 |
|
||||||
|
| R9 错误契约 | 4.3 |
|
||||||
|
| R10 权限治理 | 6.4 |
|
||||||
|
|
||||||
|
## 明确不迁入包的部分
|
||||||
|
|
||||||
|
- `src/renderer/`(React 渲染层,Agent-Employee 用 SolidJS,不迁移)
|
||||||
|
- `overlay.ts` / `desktop-pick.ts` / `vision-lab.ts` / `fixture/`(Electron 窗口强耦合,留宿主)
|
||||||
|
- `local-vision/`(onnxruntime-node 原生依赖,Bun 兼容性单独立项验证)
|
||||||
|
- `flows.ts` / `run/manager.ts` / `settings.ts`(应用级编排与持久化,属宿主职责)
|
||||||
@ -1,99 +0,0 @@
|
|||||||
# 图像与OCR 能力问题检测报告
|
|
||||||
|
|
||||||
- **检测时间**:2026-10-30 10:00–10:10
|
|
||||||
- **检测人**:刘爱家(AI 辅助静态核对 + 真实运行验证)
|
|
||||||
- **检测范围**:图像与OCR 分组已接入的全部 10 项能力(Image 7 项 + OcrOffline 3 项)
|
|
||||||
- **检测方式**:
|
|
||||||
1. 静态核对——组件目录(catalog.json)字段定义 vs 实现代码(vision.ts / ocr-caps.ts)逐键比对
|
|
||||||
2. 真实运行——GDI 生成含中英文的测试图、真实截屏、模板匹配直测、OCR 引擎实识别(飞桨 @gutenye/ocr-node)
|
|
||||||
- **说明**:本次仅检测未修改任何代码,问题留给同事修复。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 一、结论概览
|
|
||||||
|
|
||||||
| 组件 | 名称 | 可用性 | 核心问题 |
|
|
||||||
|---|---|---|---|
|
|
||||||
| Image_Screenshot | 屏幕截图 | ⚠️ 基本可用 | 截图范围/图像来源字段无效,仅支持全屏 |
|
|
||||||
| Image_Find | 查找图像 | ✅ 可用 | accuracy2 字段无效 |
|
|
||||||
| Image_Find_V1 | 查找图像 | ⚠️ 有隐患 | window_obj 必填但被忽略,实际全屏查找 |
|
|
||||||
| Image_Click | 点击图像 | ⚠️ 有缺陷 | position(点击位置)被忽略,永远点中心 |
|
|
||||||
| Image_Click_V1 | 点击图像 | ⚠️ 有缺陷 | 同上 |
|
|
||||||
| Image_Click_V2 | 点击图像 | ⚠️ 有缺陷 | 同上 |
|
|
||||||
| Image_Click_V3 | 鼠标点击图像 | ❌ 功能缺失 | click_key(左/右键)被忽略,右键点击无法实现;窗口限定无效 |
|
|
||||||
| OcrOffline_GetOcrTextNew | 图像OCR识别 | ❌ 不可用 | **按目录配置无法传入图片路径,任何填写方式都报「图片路径为空」** |
|
|
||||||
| OcrOffline_GetOcrTextPosition | 图像OCR识别(含坐标) | ✅ 可用 | 无阻塞性问题 |
|
|
||||||
| OcrOffline_GetOcrTable | 获取表格数据(OCR) | ⚠️ 部分可用 | 保存至数据表/Excel 的 4 个字段全部无效;返回扁平文本非表格结构 |
|
|
||||||
|
|
||||||
**正常项(已实测确认)**:截屏产物正确;模板匹配精度高(自测得分 0.9993);OCR 中文/数字识别准确、坐标与置信度返回正常;找图失败时报错提示合理。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 二、问题清单(按严重度)
|
|
||||||
|
|
||||||
### P0-1 OcrOffline_GetOcrTextNew:目录字段与实现完全脱节,组件不可用
|
|
||||||
|
|
||||||
- **现象**:该组件在目录里唯一的图片相关字段是 `uiSelector`(目标,非必填),而实现读取图片路径的键列表为 `image_path/imagePath/file_path/filePath/path/image/pictureSource`——**不包含 uiSelector**。
|
|
||||||
- **后果**:用户按目录界面填参数,无论怎么填都无法把图片传进实现,必然报「图片路径为空」。已实测复现:`{ uiSelector: "D:/.../ocr_input.png" }` → `RuntimeError: 图片路径为空`。
|
|
||||||
- **修复建议**:实现读取键中加入 `uiSelector`,或目录为该组件增加 `image_path` 必填字段(与 GetOcrTextPosition 保持一致)。
|
|
||||||
|
|
||||||
### P1-1 Image_Click 全 4 版本:忽略「点击位置」字段
|
|
||||||
|
|
||||||
- **现象**:目录字段 `position`(点击位置,默认 center),实现拿到匹配区域后**固定点击中心点**(`vision.ts` 中 `x + width/2, y + height/2`),position 值从不读取。
|
|
||||||
- **后果**:用户选择「左上角/右上角/自定义偏移」等位置时静默失效,点击落点与预期不符且无任何提示。
|
|
||||||
|
|
||||||
### P1-2 Image_Click_V3:忽略「鼠标按键」字段,右键点击无法实现
|
|
||||||
|
|
||||||
- **现象**:目录字段 `click_key`(鼠标按键,默认 left),实现写死左键单击(`mouse_event(2,...)`按下 + `mouse_event(4,...)`弹起)。
|
|
||||||
- **后果**:右键菜单类自动化场景完全无法实现,且不报错(静默按左键执行)。
|
|
||||||
|
|
||||||
### P1-3 OcrOffline_GetOcrTable:输出相关 4 个字段全部无效
|
|
||||||
|
|
||||||
- **现象**:目录字段 `showTable`(保存至数据表)、`isExcel`(保存至Excel)、`excelName`、`excelPath`,实现全部不读取;仅返回按 y 坐标排序的**扁平文本行数组**。
|
|
||||||
- **后果**:「将结果保存至数据表/Excel」的承诺静默不执行;且行内多列文本不拆分(如「鼠标 20」作为一整行文本返回),拿不到结构化表格。
|
|
||||||
|
|
||||||
### P2-1 Image_Find_V1 / Image_Click_V3:「窗口对象」必填但被忽略
|
|
||||||
|
|
||||||
- **现象**:目录要求 `window_obj` 必填、`find_range`(图像查找范围)可选,实现两者都不读取,永远全屏截图查找。
|
|
||||||
- **后果**:用户以为限定了窗口范围,实际全屏找图——找到错误目标(其他窗口的同款图标)或找不到时无从排查。且 window_obj 必填迫使用户必须配置一个无意义的参数。
|
|
||||||
|
|
||||||
### P2-2 Image_Screenshot:「截图范围」「图像来源」字段无效
|
|
||||||
|
|
||||||
- **现象**:目录字段 `pictureSource`(图像来源)、`shotRange`(截图范围),实现不读取,固定全屏(`PrimaryScreen`)。目录里还有 `action/position/position_img` 三个疑似遗留占位字段。
|
|
||||||
- **后果**:只支持全屏截图;另外多显示器环境只截主屏,副屏内容丢失。
|
|
||||||
|
|
||||||
### P3-1 「特征相似度」accuracy2 字段无效
|
|
||||||
|
|
||||||
- Image_Find / Image_Click / Image_Click_V1 的 `accuracy2`(特征相似度)字段,实现不读取。当前实现是纯像素差模板匹配,没有特征匹配层,该字段为冗余配置。
|
|
||||||
|
|
||||||
### P3-2 「展示鼠标移动轨迹」show_track 字段无效
|
|
||||||
|
|
||||||
- Image_Click 全版本的 `show_track` 被忽略。当前实现 SetCursorPos 直接跳转,无轨迹动画。低优先级。
|
|
||||||
|
|
||||||
### P3-3 其他小问题
|
|
||||||
|
|
||||||
- Image_Screenshot 当 `image_path` 为空时输出文件落盘到进程工作目录(不可预期);因该字段必填,实际风险低,建议加显式校验。
|
|
||||||
- OcrOffline_GetOcrTable 的行排序仅按 y 坐标,同一行内多个文本块不合并、不按 x 排序,多列场景顺序不稳定。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 三、实测验证记录(正常部分证据)
|
|
||||||
|
|
||||||
| 测试 | 结果 |
|
|
||||||
|---|---|
|
|
||||||
| GDI 生成测试图(600×200,含「RPA流程测试ABC123」「名称 数量」「鼠标 20」) | 成功 |
|
|
||||||
| Image_Screenshot 截屏逻辑(同款 PowerShell 脚本) | 成功,产物 246KB PNG |
|
|
||||||
| findImage 模板匹配(app_logo_tpl 模板 vs 实时截屏) | 命中 (93,27),得分 **0.9993** |
|
|
||||||
| OcrOffline_GetOcrTextNew(传入 image_path 时) | 识别结果 `RPA流程测试ABC123\n名称 数量\n鼠标 20` 完全正确 |
|
|
||||||
| OcrOffline_GetOcrTextPosition | 3 行文本坐标 + 置信度(0.999/0.949/0.899)返回正常 |
|
|
||||||
| 找图失败路径 | 正确抛「屏幕中未找到目标图像」并附建议(注:vitest 环境内 PowerShell 受 PATH 限制报 ENOENT,真实应用主进程不受影响) |
|
|
||||||
|
|
||||||
**说明**:OCR 引擎(飞桨)与模板匹配算法本身质量良好,问题集中在**目录字段与实现的键名/功能脱节**——与此前 Office、数据处理组发现的拼接列、单元格格式问题是同一类「接入只对了一半」的缺陷。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 四、给同事的修复优先级建议
|
|
||||||
|
|
||||||
1. **先修 P0-1**(GetOcrTextNew 传图键)——1 行改动,组件从不可用变可用
|
|
||||||
2. **再修 P1 三项**(Click 的 position/click_key、GetOcrTable 的输出字段)——都是读键 + 少量分支逻辑
|
|
||||||
3. **P2/P3 可排后**——窗口限定、截图范围属于功能增强,当前行为可接受但建议在 UI 上隐藏无效字段避免误导
|
|
||||||
@ -18,17 +18,25 @@ export default {
|
|||||||
},
|
},
|
||||||
files: [
|
files: [
|
||||||
"out/**/*",
|
"out/**/*",
|
||||||
"package.json"
|
"package.json",
|
||||||
|
// ws 已从主进程 bundle 外部化(运行时 require),必须随包发布;
|
||||||
|
// ws 为零依赖纯 JS 库,asar 内即可解析
|
||||||
|
"node_modules/ws/**/*"
|
||||||
],
|
],
|
||||||
extraResources: [
|
extraResources: [
|
||||||
{ from: path.join(root, "resources", "catalog.json"), to: "catalog.json" },
|
{ from: path.join(root, "resources", "catalog.json"), to: "catalog.json" },
|
||||||
|
{ from: path.join(root, "resources", "excluded.json"), to: "excluded.json" },
|
||||||
|
{ from: path.join(root, "resources", "capability-duplicates.json"), to: "capability-duplicates.json" },
|
||||||
|
{ from: path.join(root, "resources", "desktop-picker.ps1"), to: "desktop-picker.ps1" },
|
||||||
|
{ from: path.join(root, "resources", "models", "ui-element-detect"), to: "models/ui-element-detect" },
|
||||||
{ from: path.join(root, "resources", "fixture"), to: "fixture" }
|
{ from: path.join(root, "resources", "fixture"), to: "fixture" }
|
||||||
],
|
],
|
||||||
asar: true,
|
asar: true,
|
||||||
asarUnpack: [
|
asarUnpack: [
|
||||||
"node_modules/onnxruntime-node/**",
|
"node_modules/onnxruntime-node/**",
|
||||||
"node_modules/sharp/**",
|
"node_modules/sharp/**",
|
||||||
"node_modules/@gutenye/**"
|
"node_modules/@gutenye/**",
|
||||||
|
"node_modules/@napi-rs/**"
|
||||||
],
|
],
|
||||||
compression: "maximum",
|
compression: "maximum",
|
||||||
npmRebuild: false,
|
npmRebuild: false,
|
||||||
|
|||||||
@ -4,15 +4,20 @@ import react from "@vitejs/plugin-react";
|
|||||||
|
|
||||||
export default defineConfig({
|
export default defineConfig({
|
||||||
main: {
|
main: {
|
||||||
plugins: [externalizeDepsPlugin()],
|
// ws 必须外部化运行时 require:打包进 bundle 时其可选原生依赖 bufferutil 的解析产物
|
||||||
|
// 会破坏 mask fallback(客户端发帧静默丢字节 / 报 mask is not a function)
|
||||||
|
plugins: [externalizeDepsPlugin({ include: ["ws"] })],
|
||||||
resolve: {
|
resolve: {
|
||||||
alias: {
|
alias: [
|
||||||
"@shared": resolve("src/shared")
|
{ find: "@shared", replacement: resolve("src/shared") },
|
||||||
}
|
// rpa-core 包以源码形式打进主进程产物(不作为外部依赖运行时 require)
|
||||||
|
{ find: /^@rpa\/core$/, replacement: resolve("packages/rpa-core/src/index.ts") },
|
||||||
|
{ find: /^@rpa\/core\/capabilities$/, replacement: resolve("packages/rpa-core/src/capabilities/index.ts") }
|
||||||
|
]
|
||||||
},
|
},
|
||||||
build: {
|
build: {
|
||||||
rollupOptions: {
|
rollupOptions: {
|
||||||
input: { index: resolve("src/main/index.ts") }
|
input: { index: resolve("src/main/index.ts"), "local-vision/ui-detect-worker": resolve("src/main/local-vision/ui-detect-worker.ts") }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|||||||
2085
package-lock.json
generated
2085
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
23
package.json
23
package.json
@ -4,6 +4,9 @@
|
|||||||
"private": true,
|
"private": true,
|
||||||
"description": "Canvas-based RPA desktop application",
|
"description": "Canvas-based RPA desktop application",
|
||||||
"main": "./out/main/index.js",
|
"main": "./out/main/index.js",
|
||||||
|
"workspaces": [
|
||||||
|
"packages/*"
|
||||||
|
],
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"dev": "electron-vite dev",
|
"dev": "electron-vite dev",
|
||||||
"dev:local": "node scripts/dev-safe.mjs",
|
"dev:local": "node scripts/dev-safe.mjs",
|
||||||
@ -11,42 +14,56 @@
|
|||||||
"build": "electron-vite build",
|
"build": "electron-vite build",
|
||||||
"preview": "electron-vite preview",
|
"preview": "electron-vite preview",
|
||||||
"typecheck": "tsc --noEmit -p tsconfig.json",
|
"typecheck": "tsc --noEmit -p tsconfig.json",
|
||||||
"catalog": "node scripts/build-catalog.mjs",
|
"check:deps": "node packages/rpa-core/scripts/check-deps.mjs",
|
||||||
|
"rpa-cli": "tsx scripts/rpa-cli.ts",
|
||||||
|
"catalog": "tsx scripts/build-catalog.mjs",
|
||||||
|
"snapshot": "vitest run scripts/capability-snapshot.test.ts",
|
||||||
"package": "node scripts/package.mjs",
|
"package": "node scripts/package.mjs",
|
||||||
"test": "vitest run",
|
"test": "vitest run",
|
||||||
"test:e2e": "node scripts/demo-e2e.mjs"
|
"test:e2e": "node scripts/demo-e2e.mjs"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
|
"@cantoo/pdf-lib": "^2.11.1",
|
||||||
"@gutenye/ocr-models": "^1.4.2",
|
"@gutenye/ocr-models": "^1.4.2",
|
||||||
"@gutenye/ocr-node": "^1.4.9",
|
"@gutenye/ocr-node": "^1.4.9",
|
||||||
|
"@napi-rs/canvas": "^1.1.0",
|
||||||
"@xyflow/react": "^12.12.0",
|
"@xyflow/react": "^12.12.0",
|
||||||
"adm-zip": "^0.5.16",
|
"adm-zip": "^0.5.16",
|
||||||
"basic-ftp": "^6.2.3",
|
"basic-ftp": "^6.2.3",
|
||||||
"exceljs": "^4.4.0",
|
"exceljs": "^4.4.0",
|
||||||
|
"imapflow": "^2.10.0",
|
||||||
"lucide-react": "^0.468.0",
|
"lucide-react": "^0.468.0",
|
||||||
|
"mailparser": "^3.14.1",
|
||||||
|
"mssql": "^12.7.13",
|
||||||
|
"mysql2": "^3.24.6",
|
||||||
"nodemailer": "^6.9.16",
|
"nodemailer": "^6.9.16",
|
||||||
|
"onnxruntime-node": "^1.31.1",
|
||||||
"pdf-lib": "^1.17.1",
|
"pdf-lib": "^1.17.1",
|
||||||
"pdfjs-dist": "^4.10.38",
|
"pdfjs-dist": "^4.10.38",
|
||||||
|
"pg": "^8.23.1",
|
||||||
"pngjs": "^7.0.0",
|
"pngjs": "^7.0.0",
|
||||||
"react": "^18.3.1",
|
"react": "^18.3.1",
|
||||||
"react-dom": "^18.3.1",
|
"react-dom": "^18.3.1",
|
||||||
|
"sm-crypto": "^0.5.8",
|
||||||
"tesseract.js": "^7.0.0",
|
"tesseract.js": "^7.0.0",
|
||||||
"ws": "^8.22.0",
|
"ws": "^8.18.0",
|
||||||
"zustand": "^5.0.2"
|
"zustand": "^5.0.2"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/adm-zip": "^0.5.7",
|
"@types/adm-zip": "^0.5.7",
|
||||||
|
"@types/mssql": "^12.3.0",
|
||||||
"@types/node": "^22.10.2",
|
"@types/node": "^22.10.2",
|
||||||
"@types/nodemailer": "^6.4.17",
|
"@types/nodemailer": "^6.4.17",
|
||||||
|
"@types/pg": "^8.23.1",
|
||||||
"@types/pngjs": "^6.0.5",
|
"@types/pngjs": "^6.0.5",
|
||||||
"@types/react": "^18.3.12",
|
"@types/react": "^18.3.12",
|
||||||
"@types/react-dom": "^18.3.1",
|
"@types/react-dom": "^18.3.1",
|
||||||
"@types/ws": "^8.18.2",
|
|
||||||
"@vitejs/plugin-react": "^4.7.0",
|
"@vitejs/plugin-react": "^4.7.0",
|
||||||
"electron": "31.7.7",
|
"electron": "31.7.7",
|
||||||
"electron-builder": "^26.15.7",
|
"electron-builder": "^26.15.7",
|
||||||
"electron-vite": "^6.0.0",
|
"electron-vite": "^6.0.0",
|
||||||
"typescript": "^5.7.2",
|
"typescript": "^5.7.2",
|
||||||
|
"tsx": "^4.19.2",
|
||||||
"vite": "^7.1.0",
|
"vite": "^7.1.0",
|
||||||
"vitest": "^2.1.8"
|
"vitest": "^2.1.8"
|
||||||
},
|
},
|
||||||
|
|||||||
61
packages/rpa-core/README.md
Normal file
61
packages/rpa-core/README.md
Normal file
@ -0,0 +1,61 @@
|
|||||||
|
# @rpa/core
|
||||||
|
|
||||||
|
RPA 能力层包:零 Electron 依赖的流程引擎、能力注册表与能力实现。
|
||||||
|
抽离路线见 `docs/rpa-core-refactor-tasks.md`,开发规范见 `docs/dev-conventions.md`。
|
||||||
|
|
||||||
|
## 包边界
|
||||||
|
|
||||||
|
| 模块 | 说明 |
|
||||||
|
|---|---|
|
||||||
|
| `src/engine/` | 注册表、运行时、表达式、`createEngine()` 工厂、`host-bridge.ts` 宿主桥 |
|
||||||
|
| `src/driver/` | `types.ts` 五域 AutomationDriver 接口(mouse/keyboard/window/screen/element)+ `windows/powershell.ts` 进程助手 + `windows/windows-driver.ts` PowerShell 实现 |
|
||||||
|
| `src/capabilities/` | 能力实现(副作用注册,经 `@rpa/core/capabilities` 拉起) |
|
||||||
|
| `src/browser/` | 外部浏览器 CDP:`external.ts`(Edge/Chrome 进程管理)+ `transport.ts`(CdpTransport 接口、ExternalTransport) |
|
||||||
|
|
||||||
|
## HostBridge 契约(决策点 4 最小面)
|
||||||
|
|
||||||
|
宿主(Electron 主进程 / 未来 Agent-Employee product-shell)启动时注入:
|
||||||
|
|
||||||
|
```ts
|
||||||
|
import { setHostBridge, setSubflowSource } from "@rpa/core";
|
||||||
|
|
||||||
|
setHostBridge({
|
||||||
|
createFormWindow({ title }) { /* 返回 HostDialogWindow 适配(BrowserWindow) */ },
|
||||||
|
dipToScreenPoint(p) { /* screen.dipToScreenPoint */ },
|
||||||
|
screenToDipPoint(p) { /* screen.screenToDipPoint */ },
|
||||||
|
loadAiConfig() { /* settings.aiConfig */ }
|
||||||
|
});
|
||||||
|
setSubflowSource({ listFlows, getFlow }); // 子流程仓库
|
||||||
|
```
|
||||||
|
|
||||||
|
未注入时的行为:window-extended 的 DPI 换算回退恒等(与旧 vitest 行为一致);
|
||||||
|
ai-service 返回空配置(requireAiConfig 统一抛 `ai_not_configured`);
|
||||||
|
human-dialog 抛 `host_unavailable`。
|
||||||
|
|
||||||
|
## Electron 相关能力留守清单(宿主适配层,不入包)
|
||||||
|
|
||||||
|
- `src/main/browser/session.ts`:BrowserSession 编排 + 内置运行浏览器(依赖 `BrowserWindow` 与应用生命周期)
|
||||||
|
- `src/main/browser/transport.ts`:`EmbeddedTransport`(包装 `webContents.debugger`)
|
||||||
|
- `src/main/capabilities/browser*.ts` / `web-batch2.ts` / `web-crawl.ts`:经 BrowserSession 操作浏览器窗口
|
||||||
|
- `src/main/capabilities/ai-caps.ts`:AI 编排能力(依赖宿主注入的 chatCompletion 配置面,可随后续阶段评估迁入)
|
||||||
|
- `src/main/overlay.ts` / `desktop-pick.ts` / `vision-lab.ts` / `fixture/`:Electron 窗口强耦合
|
||||||
|
- `src/main/local-vision/`:onnxruntime-node 原生依赖,Bun 兼容性单独立项验证
|
||||||
|
- `src/main/flows.ts` / `run/` / `settings.ts`:应用级编排与持久化
|
||||||
|
|
||||||
|
## 依赖护栏
|
||||||
|
|
||||||
|
`npm run check:deps`(根目录亦可)扫描 `src/**`:禁止 `import "electron"`、
|
||||||
|
相对引用逃出包 `src/`、引用宿主 `src/main`。阶段 2 起为提交前必跑项。
|
||||||
|
|
||||||
|
## Driver(阶段 3)
|
||||||
|
|
||||||
|
- 五域接口 `AutomationDriver`(决策点 6):`RpaEngine.driver` 默认 `new WindowsDriver()`,
|
||||||
|
`createEngine()` 可注入自定义实现(CLI/Linux driver 等未来形态)。
|
||||||
|
- 驱动只承载**原始注入原语**(单次 SetCursorPos / SendKeys / 窗口操作);拟真抖动、
|
||||||
|
DIP 坐标换算、多步组合等策略留在能力文件,保证机械替换不改时序行为。
|
||||||
|
- element 域实现承载在 `capabilities/desktop-uia.ts`(经 `setElementDriverImplementation`
|
||||||
|
注册到 WindowsDriver),handler 层面向 driver 的收口在阶段 5/6 按需进行。
|
||||||
|
- `powershell()` 助手统一位于 `driver/windows/powershell.ts`,包内能力禁止自行
|
||||||
|
`execFile("powershell.exe")`(check:deps 与 review 双重把关)。
|
||||||
|
- 平台标记:`npm run catalog` 构建时按组填充 `platformTag: "win32"`(desktop/app/
|
||||||
|
office/vision 组),渲染层置灰延后(决策点 7)。
|
||||||
46
packages/rpa-core/package.json
Normal file
46
packages/rpa-core/package.json
Normal file
@ -0,0 +1,46 @@
|
|||||||
|
{
|
||||||
|
"name": "@rpa/core",
|
||||||
|
"version": "0.1.0",
|
||||||
|
"private": true,
|
||||||
|
"description": "RPA 能力层:零 Electron 依赖的流程引擎、能力注册表与纯 Node 能力",
|
||||||
|
"main": "./src/index.ts",
|
||||||
|
"types": "./src/index.ts",
|
||||||
|
"exports": {
|
||||||
|
".": "./src/index.ts",
|
||||||
|
"./capabilities": "./src/capabilities/index.ts"
|
||||||
|
},
|
||||||
|
"scripts": {
|
||||||
|
"test": "vitest run",
|
||||||
|
"typecheck": "tsc --noEmit -p tsconfig.json",
|
||||||
|
"check:deps": "node scripts/check-deps.mjs"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"@cantoo/pdf-lib": "^2.11.1",
|
||||||
|
"@gutenye/ocr-node": "^1.4.9",
|
||||||
|
"@napi-rs/canvas": "^1.1.0",
|
||||||
|
"adm-zip": "^0.5.16",
|
||||||
|
"basic-ftp": "^6.2.3",
|
||||||
|
"exceljs": "^4.4.0",
|
||||||
|
"imapflow": "^2.10.0",
|
||||||
|
"mailparser": "^3.14.1",
|
||||||
|
"mssql": "^12.7.13",
|
||||||
|
"mysql2": "^3.24.6",
|
||||||
|
"nodemailer": "^6.9.16",
|
||||||
|
"pdf-lib": "^1.17.1",
|
||||||
|
"pdfjs-dist": "^4.10.38",
|
||||||
|
"pg": "^8.23.1",
|
||||||
|
"pngjs": "^7.0.0",
|
||||||
|
"sm-crypto": "^0.5.8",
|
||||||
|
"ws": "^8.22.0"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/adm-zip": "^0.5.7",
|
||||||
|
"@types/mssql": "^12.3.0",
|
||||||
|
"@types/nodemailer": "^6.4.17",
|
||||||
|
"@types/pg": "^8.23.1",
|
||||||
|
"@types/pngjs": "^6.0.5",
|
||||||
|
"@types/ws": "^8.18.2",
|
||||||
|
"typescript": "^5.7.2",
|
||||||
|
"vitest": "^2.1.8"
|
||||||
|
}
|
||||||
|
}
|
||||||
78
packages/rpa-core/scripts/check-deps.mjs
Normal file
78
packages/rpa-core/scripts/check-deps.mjs
Normal file
@ -0,0 +1,78 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
/**
|
||||||
|
* rpa-core 依赖护栏(任务 2.4,规范 v1 第二节的机器强制版)。
|
||||||
|
*
|
||||||
|
* 禁止:
|
||||||
|
* 1. import "electron"(含 "electron/..." 子路径);
|
||||||
|
* 2. 相对 import 逃出包 src/(如指向宿主 src/main 的 ../../../src/main/...);
|
||||||
|
* 3. 任何 import 路径中出现 src/main(宿主模块横向依赖)。
|
||||||
|
*
|
||||||
|
* 用法:node scripts/check-deps.mjs(退出码非 0 表示违规)
|
||||||
|
*/
|
||||||
|
import { readdir, readFile } from "node:fs/promises";
|
||||||
|
import path from "node:path";
|
||||||
|
import process from "node:process";
|
||||||
|
import { fileURLToPath } from "node:url";
|
||||||
|
|
||||||
|
const pkgRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||||
|
const srcRoot = path.join(pkgRoot, "src");
|
||||||
|
|
||||||
|
const IMPORT_RE = /(?:import|export)\s+(?:[\s\S]*?from\s*)?["']([^"']+)["']/g;
|
||||||
|
// 动态形式:import("x") / require("x") / createRequire 后的 require 调用。
|
||||||
|
// 旧正则要求 import 后跟空白,`import("electron")` 完全不匹配(实测 0 命中,
|
||||||
|
// vision/cv-vision 曾靠这个盲区逃过检查——迁移阶段1-A9 修复)。
|
||||||
|
const DYNAMIC_IMPORT_RE = /(?:\bimport\s*\(|\brequire\s*\()\s*["']([^"']+)["']/g;
|
||||||
|
|
||||||
|
async function walk(dir) {
|
||||||
|
const entries = await readdir(dir, { withFileTypes: true });
|
||||||
|
const files = [];
|
||||||
|
for (const entry of entries) {
|
||||||
|
const full = path.join(dir, entry.name);
|
||||||
|
if (entry.isDirectory()) files.push(...(await walk(full)));
|
||||||
|
else if (/\.tsx?$/.test(entry.name)) files.push(full);
|
||||||
|
}
|
||||||
|
return files;
|
||||||
|
}
|
||||||
|
|
||||||
|
const offenders = [];
|
||||||
|
for (const file of await walk(srcRoot)) {
|
||||||
|
const raw = await readFile(file, "utf8");
|
||||||
|
// 剥掉注释,避免文档示例里的 "electron" 字样误报
|
||||||
|
const source = raw.replace(/\/\*[\s\S]*?\*\//g, "").replace(/^\s*\/\/.*$/gm, "");
|
||||||
|
for (const match of source.matchAll(IMPORT_RE)) {
|
||||||
|
const spec = match[1];
|
||||||
|
const rel = path.relative(pkgRoot, file).replaceAll("\\", "/");
|
||||||
|
if (spec === "electron" || spec.startsWith("electron/")) {
|
||||||
|
offenders.push(`${rel}: 禁止 import "${spec}"(宿主能力走 HostBridge 注入)`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (spec.includes("src/main")) {
|
||||||
|
offenders.push(`${rel}: 禁止引用宿主模块 "${spec}"`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (spec.startsWith(".")) {
|
||||||
|
const resolved = path.resolve(path.dirname(file), spec);
|
||||||
|
if (!resolved.startsWith(srcRoot + path.sep) && resolved !== srcRoot) {
|
||||||
|
offenders.push(`${rel}: 相对引用逃出包 src/ → "${spec}"`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (const match of source.matchAll(DYNAMIC_IMPORT_RE)) {
|
||||||
|
const spec = match[1];
|
||||||
|
const rel = path.relative(pkgRoot, file).replaceAll("\\", "/");
|
||||||
|
if (spec === "electron" || spec.startsWith("electron/")) {
|
||||||
|
offenders.push(`${rel}: 禁止动态 import("${spec}")(宿主能力走 HostBridge 注入)`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (spec.includes("src/main")) {
|
||||||
|
offenders.push(`${rel}: 禁止动态引用宿主模块 "${spec}"`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (offenders.length) {
|
||||||
|
console.error("rpa-core 依赖护栏检查失败:");
|
||||||
|
for (const line of offenders) console.error(" - " + line);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
console.log(`rpa-core 依赖护栏检查通过(src/ 下 ${offenders.length === 0 ? "无" : ""}违规 import)`);
|
||||||
@ -1,10 +1,14 @@
|
|||||||
import { execFile, spawn, type ChildProcess } from "node:child_process";
|
import { execFile, spawn, type ChildProcess } from "node:child_process";
|
||||||
|
import { promisify } from "node:util";
|
||||||
import { promises as fs } from "node:fs";
|
import { promises as fs } from "node:fs";
|
||||||
|
import { randomUUID } from "node:crypto";
|
||||||
import os from "node:os";
|
import os from "node:os";
|
||||||
import path from "node:path";
|
import path from "node:path";
|
||||||
import { RuntimeError } from "../engine/types";
|
import { RuntimeError } from "../engine/types";
|
||||||
import { ExternalTransport } from "./transport";
|
import { ExternalTransport } from "./transport";
|
||||||
|
|
||||||
|
const execAsync = promisify(execFile);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 外部浏览器(Edge / Chrome)进程管理与 CDP 连接。
|
* 外部浏览器(Edge / Chrome)进程管理与 CDP 连接。
|
||||||
*
|
*
|
||||||
@ -24,6 +28,8 @@ export interface ExternalLaunchOptions {
|
|||||||
extraArgs?: string[];
|
extraArgs?: string[];
|
||||||
/** 独立用户数据目录;缺省使用系统临时目录下的独立 profile */
|
/** 独立用户数据目录;缺省使用系统临时目录下的独立 profile */
|
||||||
userDataDir?: string;
|
userDataDir?: string;
|
||||||
|
/** 首次启动时直接打开的 URL,避免先出现 about:blank */
|
||||||
|
initialUrl?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ExternalRemoteOptions {
|
export interface ExternalRemoteOptions {
|
||||||
@ -207,6 +213,31 @@ export async function connectRemoteBrowser(options: ExternalRemoteOptions): Prom
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** 清理占用专用 profile 的残留浏览器进程。
|
||||||
|
* 同一 user-data-dir 已有实例时,再次启动的 Chrome 会把请求转交旧实例并立即退出(代码 0),
|
||||||
|
* 导致「启动失败(进程已退出,代码 0)」——这些实例都是本模块之前拉起的,清理是安全的。
|
||||||
|
* 匹配按完整 user-data-dir 路径字面量过滤(非通配),只会命中本模块自己拉起的实例。 */
|
||||||
|
async function killStaleBrowserProcesses(browserType: ExternalBrowserType, userDataDir: string): Promise<void> {
|
||||||
|
try {
|
||||||
|
if (process.platform === "win32") {
|
||||||
|
const names = browserType === "edge" ? "msedge.exe" : "chrome.exe";
|
||||||
|
// 单引号转义防注入;-like 用字面量路径(路径内无通配符语义字符时的精确匹配)
|
||||||
|
const marker = userDataDir.replace(/'/g, "''").replace(/`/g, "``").replace(/\[/g, "``[");
|
||||||
|
const script =
|
||||||
|
`Get-CimInstance Win32_Process -Filter "Name='${names}'" | ` +
|
||||||
|
`Where-Object { $_.CommandLine -like '*${marker}*' } | ` +
|
||||||
|
"ForEach-Object { Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue }";
|
||||||
|
await execAsync("powershell.exe", ["-NoProfile", "-NonInteractive", "-Command", script], { timeout: 8000, windowsHide: true });
|
||||||
|
} else {
|
||||||
|
// pkill -f 用完整 profile 路径字面量(含 rpa-studio-external 前缀),不扩大清理范围
|
||||||
|
await execAsync("pkill", ["-f", userDataDir], { timeout: 5000 });
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
// 清理失败不阻塞启动:可能本来就没有残留,或权限受限;记 warning 供审计(整改:不再静默吞)
|
||||||
|
console.warn(`[browser] 清理残留 ${browserType} 进程失败(不影响启动):`, (error as Error).message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/** 拉起本机 Edge/Chrome(带独立 profile + 调试端口),并接管其页面 */
|
/** 拉起本机 Edge/Chrome(带独立 profile + 调试端口),并接管其页面 */
|
||||||
export async function launchExternalBrowser(options: ExternalLaunchOptions): Promise<ExternalConnection> {
|
export async function launchExternalBrowser(options: ExternalLaunchOptions): Promise<ExternalConnection> {
|
||||||
const browserType = options.browserType === "edge" ? "edge" : "chrome";
|
const browserType = options.browserType === "edge" ? "edge" : "chrome";
|
||||||
@ -221,20 +252,27 @@ export async function launchExternalBrowser(options: ExternalLaunchOptions): Pro
|
|||||||
}
|
}
|
||||||
|
|
||||||
const userDataDir =
|
const userDataDir =
|
||||||
options.userDataDir?.trim() || path.join(os.tmpdir(), `rpa-studio-external-${browserType}`);
|
options.userDataDir?.trim() ||
|
||||||
|
// 并发/多会话隔离(审查整改):默认 profile 加入随机后缀,进程级唯一,
|
||||||
|
// 避免多个运行共用同一 profile 互相挤掉登录态与调试实例。
|
||||||
|
path.join(os.tmpdir(), `rpa-studio-external-${browserType}-${randomUUID().slice(0, 8)}`);
|
||||||
await fs.mkdir(userDataDir, { recursive: true });
|
await fs.mkdir(userDataDir, { recursive: true });
|
||||||
|
|
||||||
const extraArgs = (options.extraArgs ?? []).filter(Boolean);
|
const extraArgs = (options.extraArgs ?? []).filter(Boolean);
|
||||||
|
const startUrl = options.initialUrl?.trim() || "about:blank";
|
||||||
const args = [
|
const args = [
|
||||||
`--remote-debugging-port=${port}`,
|
`--remote-debugging-port=${port}`,
|
||||||
`--user-data-dir=${userDataDir}`,
|
`--user-data-dir=${userDataDir}`,
|
||||||
"--no-first-run",
|
"--no-first-run",
|
||||||
"--no-default-browser-check",
|
"--no-default-browser-check",
|
||||||
"about:blank",
|
// 固定 profile 曾被强杀过时,Chrome 会弹「恢复页面?」气泡挡住页面拾取
|
||||||
|
"--hide-crash-restore-bubble",
|
||||||
|
startUrl,
|
||||||
...extraArgs
|
...extraArgs
|
||||||
];
|
];
|
||||||
|
|
||||||
const child = spawn(browserPath, args, { detached: false, stdio: "ignore", windowsHide: true });
|
const spawnBrowser = (): ChildProcess => spawn(browserPath, args, { detached: false, stdio: "ignore", windowsHide: true });
|
||||||
|
let child = spawnBrowser();
|
||||||
child.on("error", () => {
|
child.on("error", () => {
|
||||||
// spawn 失败由下方就绪轮询超时统一报错
|
// spawn 失败由下方就绪轮询超时统一报错
|
||||||
});
|
});
|
||||||
@ -242,11 +280,24 @@ export async function launchExternalBrowser(options: ExternalLaunchOptions): Pro
|
|||||||
// 就绪轮询:等 /json/version 可访问(最长 15s)
|
// 就绪轮询:等 /json/version 可访问(最长 15s)
|
||||||
const deadline = Date.now() + 15000;
|
const deadline = Date.now() + 15000;
|
||||||
let ready = false;
|
let ready = false;
|
||||||
|
let retriedStale = false;
|
||||||
while (Date.now() < deadline) {
|
while (Date.now() < deadline) {
|
||||||
if (child.exitCode !== null) {
|
if (child.exitCode !== null) {
|
||||||
|
if (!retriedStale) {
|
||||||
|
// 立即退出(常见代码 0):专用 profile 被残留实例占用,Chrome 转交后自杀;
|
||||||
|
// 清理残留后重试一次
|
||||||
|
retriedStale = true;
|
||||||
|
await killStaleBrowserProcesses(browserType, userDataDir);
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 300));
|
||||||
|
child = spawnBrowser();
|
||||||
|
child.on("error", () => {
|
||||||
|
// spawn 失败由下方就绪轮询超时统一报错
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
throw new RuntimeError("browser_error", `${label} 启动失败(进程已退出,代码 ${child.exitCode})`, {
|
throw new RuntimeError("browser_error", `${label} 启动失败(进程已退出,代码 ${child.exitCode})`, {
|
||||||
detail: browserPath,
|
detail: browserPath,
|
||||||
recommendation: "检查浏览器安装是否完整,或参数是否合法"
|
recommendation: "已尝试清理残留实例仍失败:检查浏览器安装是否完整,或重启电脑后重试"
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
if (await isDebugPortReady(port)) {
|
if (await isDebugPortReady(port)) {
|
||||||
@ -266,7 +317,10 @@ export async function launchExternalBrowser(options: ExternalLaunchOptions): Pro
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 就绪之后的接管链(/json/list → transport.connect):任一步失败都必须回收自己拉起的
|
||||||
|
// 子进程(审查整改:此前失败路径会遗留浏览器进程)。成功路径 killOnClose=true 交由关闭逻辑。
|
||||||
const base = `http://127.0.0.1:${port}`;
|
const base = `http://127.0.0.1:${port}`;
|
||||||
|
try {
|
||||||
const targets = await fetchJson<CdpTargetInfo[]>(`${base}/json/list`, 5000);
|
const targets = await fetchJson<CdpTargetInfo[]>(`${base}/json/list`, 5000);
|
||||||
const target = pickPageTarget(targets, "about:blank");
|
const target = pickPageTarget(targets, "about:blank");
|
||||||
const transport = await ExternalTransport.connect(target.webSocketDebuggerUrl as string, `127.0.0.1:${port}`);
|
const transport = await ExternalTransport.connect(target.webSocketDebuggerUrl as string, `127.0.0.1:${port}`);
|
||||||
@ -279,6 +333,14 @@ export async function launchExternalBrowser(options: ExternalLaunchOptions): Pro
|
|||||||
child,
|
child,
|
||||||
killOnClose: true
|
killOnClose: true
|
||||||
};
|
};
|
||||||
|
} catch (error) {
|
||||||
|
try {
|
||||||
|
child.kill();
|
||||||
|
} catch {
|
||||||
|
// 进程可能已自行退出
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 列出调试端点上的全部页面级 target(多标签枚举用);排除 chrome://、edge:// 内部页 */
|
/** 列出调试端点上的全部页面级 target(多标签枚举用);排除 chrome://、edge:// 内部页 */
|
||||||
@ -299,7 +361,7 @@ export async function openNewPageTarget(httpBase: string, url = "about:blank"):
|
|||||||
}
|
}
|
||||||
return {
|
return {
|
||||||
id: String(target.id ?? ""),
|
id: String(target.id ?? ""),
|
||||||
webSocketDebuggerUrl: target.webSocketDebuggerUrl,
|
webSocketDebuggerUrl: String(target.webSocketDebuggerUrl),
|
||||||
url: String(target.url ?? url)
|
url: String(target.url ?? url)
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
220
packages/rpa-core/src/browser/transport.ts
Normal file
220
packages/rpa-core/src/browser/transport.ts
Normal file
@ -0,0 +1,220 @@
|
|||||||
|
import WebSocket from "ws";
|
||||||
|
import { RuntimeError } from "../engine/types";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* CDP 传输层抽象。
|
||||||
|
*
|
||||||
|
* 上层 BrowserSession(宿主适配层)的全部 CDP 通信都经由本接口:
|
||||||
|
* - EmbeddedTransport:内置运行浏览器(Electron BrowserWindow.webContents.debugger),留宿主
|
||||||
|
* - ExternalTransport:外部 Edge/Chrome 浏览器(WebSocket 直连调试端口),在本包内
|
||||||
|
*
|
||||||
|
* 两者只在传输方式上不同,CDP 协议域(DOM/Input/Page/Network/Runtime)完全一致。
|
||||||
|
*/
|
||||||
|
export type CdpEventHandler = (params: Record<string, unknown>) => void;
|
||||||
|
|
||||||
|
export interface CdpTransport {
|
||||||
|
readonly kind: "embedded" | "external";
|
||||||
|
/** 连接是否已失效(窗口销毁 / ws 断开) */
|
||||||
|
readonly closed: boolean;
|
||||||
|
send<T = unknown>(method: string, params?: Record<string, unknown>): Promise<T>;
|
||||||
|
/** 订阅 CDP 事件(如 Page.loadEventFired、Network.responseReceived),handler 只收到对应 method 的 params */
|
||||||
|
on(method: string, handler: CdpEventHandler): void;
|
||||||
|
off(method: string, handler: CdpEventHandler): void;
|
||||||
|
close(): Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function mapCdpError(error: unknown, kind: "embedded" | "external"): RuntimeError {
|
||||||
|
const message = (error as Error)?.message || String(error);
|
||||||
|
if (/destroyed|Target closed|not attached|connection.*(closed|lost)|websocket/i.test(message)) {
|
||||||
|
if (kind === "external") {
|
||||||
|
return new RuntimeError("browser_disconnected", "外部浏览器连接已断开", {
|
||||||
|
detail: message,
|
||||||
|
recommendation: "确认浏览器仍处于打开状态;若被手动关闭请重新运行「打开网页」"
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return new RuntimeError("tab_unavailable", "浏览器连接已失效", { detail: message });
|
||||||
|
}
|
||||||
|
return new RuntimeError("browser_error", message, { recommendation: "重试或检查页面状态" });
|
||||||
|
}
|
||||||
|
|
||||||
|
interface CdpMessage {
|
||||||
|
id?: number;
|
||||||
|
method?: string;
|
||||||
|
params?: Record<string, unknown>;
|
||||||
|
result?: unknown;
|
||||||
|
error?: { message?: string; data?: unknown };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 外部 Edge/Chrome:WebSocket 直连页面级 CDP 端点(webSocketDebuggerUrl) */
|
||||||
|
export class ExternalTransport implements CdpTransport {
|
||||||
|
readonly kind = "external" as const;
|
||||||
|
private ws: WebSocket | null = null;
|
||||||
|
private nextId = 1;
|
||||||
|
private pending = new Map<number, { resolve: (value: unknown) => void; reject: (error: Error) => void }>();
|
||||||
|
private handlers = new Map<string, Set<CdpEventHandler>>();
|
||||||
|
private closedFlag = false;
|
||||||
|
private readonly openPromise: Promise<void>;
|
||||||
|
// 单命令超时:CDP 对端不回包时(页面冻结/debugger 忙/eval 挂起),send 必须在有限时间内失败,
|
||||||
|
// 否则挂起会占住节点直到引擎级超时,且 pending 表项泄漏。
|
||||||
|
private readonly commandTimeoutMs: number;
|
||||||
|
|
||||||
|
private constructor(
|
||||||
|
wsUrl: string,
|
||||||
|
readonly description: string,
|
||||||
|
commandTimeoutMs = 30_000
|
||||||
|
) {
|
||||||
|
this.commandTimeoutMs = commandTimeoutMs;
|
||||||
|
this.openPromise = new Promise<void>((resolve, reject) => {
|
||||||
|
// perMessageDeflate 必须显式关闭:ws 客户端默认开启压缩协商后,若 ws 被打包进
|
||||||
|
// 主进程 bundle,其可选依赖 bufferutil 的解析产物会破坏 mask fallback,
|
||||||
|
// 压缩发送路径的错误又被静默吞掉 —— 表现为帧零字节写出且无任何报错。
|
||||||
|
// (ws 已在 electron.vite.config 外部化运行时 require,此项为双保险)
|
||||||
|
const ws = new WebSocket(wsUrl, { maxPayload: 256 * 1024 * 1024, perMessageDeflate: false });
|
||||||
|
this.ws = ws;
|
||||||
|
ws.on("open", () => resolve());
|
||||||
|
ws.on("message", (data: unknown) => this.handleMessage(String(data)));
|
||||||
|
ws.on("error", (error: Error) => {
|
||||||
|
this.closedFlag = true;
|
||||||
|
this.failAllPending(new Error(`外部浏览器连接异常:${error.message}`));
|
||||||
|
reject(error);
|
||||||
|
});
|
||||||
|
ws.on("close", () => {
|
||||||
|
this.closedFlag = true;
|
||||||
|
this.failAllPending(new Error("外部浏览器连接已关闭"));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
static async connect(
|
||||||
|
wsUrl: string,
|
||||||
|
description: string,
|
||||||
|
timeoutMs = 10000,
|
||||||
|
commandTimeoutMs = 30_000
|
||||||
|
): Promise<ExternalTransport> {
|
||||||
|
const transport = new ExternalTransport(wsUrl, description, commandTimeoutMs);
|
||||||
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||||
|
const timeout = new Promise<never>((_, reject) => {
|
||||||
|
timer = setTimeout(() => reject(new RuntimeError("browser_disconnected", `连接外部浏览器超时:${description}`)), timeoutMs);
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
await Promise.race([transport.openPromise, timeout]);
|
||||||
|
return transport;
|
||||||
|
} catch (error) {
|
||||||
|
// 超时/失败路径必须关闭底层 WebSocket(审查整改:此前失败会泄漏挂起的 ws)
|
||||||
|
await transport.close().catch(() => {});
|
||||||
|
throw error;
|
||||||
|
} finally {
|
||||||
|
if (timer !== undefined) clearTimeout(timer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
get closed(): boolean {
|
||||||
|
return this.closedFlag || !this.ws || this.ws.readyState !== this.ws.OPEN;
|
||||||
|
}
|
||||||
|
|
||||||
|
async send<T = unknown>(method: string, params: Record<string, unknown> = {}): Promise<T> {
|
||||||
|
if (this.closed) {
|
||||||
|
throw new RuntimeError("browser_disconnected", "外部浏览器连接已断开", {
|
||||||
|
recommendation: "确认浏览器未被手动关闭,然后重新运行「打开网页」"
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const id = this.nextId++;
|
||||||
|
const ws = this.ws as WebSocket;
|
||||||
|
let timeoutTimer: ReturnType<typeof setTimeout> | undefined;
|
||||||
|
const reply = new Promise<unknown>((resolve, reject) => {
|
||||||
|
this.pending.set(id, { resolve, reject });
|
||||||
|
try {
|
||||||
|
ws.send(JSON.stringify({ id, method, params }));
|
||||||
|
} catch (error) {
|
||||||
|
this.pending.delete(id);
|
||||||
|
reject(error as Error);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// 单命令超时:以稳定错误码失败并清理 pending,避免对端不回包时永久挂起
|
||||||
|
timeoutTimer = setTimeout(() => {
|
||||||
|
if (this.pending.delete(id)) {
|
||||||
|
reject(
|
||||||
|
new RuntimeError("timeout", `CDP 命令超时:${method}`, {
|
||||||
|
detail: `单命令超时上限 ${this.commandTimeoutMs}ms,对端未返回响应`,
|
||||||
|
recommendation: "检查目标页面是否冻结或弹出了模态对话框;必要时调大节点超时时间"
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}, this.commandTimeoutMs);
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
return (await reply) as T;
|
||||||
|
} catch (error) {
|
||||||
|
// RuntimeError(如上面的单命令超时)原样透传,不再经 mapCdpError 改写错误码
|
||||||
|
if (error instanceof RuntimeError) throw error;
|
||||||
|
throw mapCdpError(error, "external");
|
||||||
|
} finally {
|
||||||
|
if (timeoutTimer !== undefined) clearTimeout(timeoutTimer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
on(method: string, handler: CdpEventHandler): void {
|
||||||
|
if (!this.handlers.has(method)) this.handlers.set(method, new Set());
|
||||||
|
this.handlers.get(method)?.add(handler);
|
||||||
|
}
|
||||||
|
|
||||||
|
off(method: string, handler: CdpEventHandler): void {
|
||||||
|
this.handlers.get(method)?.delete(handler);
|
||||||
|
}
|
||||||
|
|
||||||
|
async close(): Promise<void> {
|
||||||
|
const ws = this.ws;
|
||||||
|
this.closedFlag = true;
|
||||||
|
this.failAllPending(new Error("外部浏览器连接已主动关闭"));
|
||||||
|
if (!ws) return;
|
||||||
|
await new Promise<void>((resolve) => {
|
||||||
|
if (ws.readyState === ws.CLOSED) return resolve();
|
||||||
|
ws.once("close", () => resolve());
|
||||||
|
try {
|
||||||
|
ws.close();
|
||||||
|
} catch {
|
||||||
|
resolve();
|
||||||
|
}
|
||||||
|
setTimeout(() => resolve(), 1500);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private handleMessage(raw: string): void {
|
||||||
|
let message: CdpMessage;
|
||||||
|
try {
|
||||||
|
message = JSON.parse(raw) as CdpMessage;
|
||||||
|
} catch {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (typeof message.id === "number") {
|
||||||
|
const entry = this.pending.get(message.id);
|
||||||
|
if (!entry) return;
|
||||||
|
this.pending.delete(message.id);
|
||||||
|
if (message.error) {
|
||||||
|
entry.reject(new Error(message.error.message || "CDP 调用失败"));
|
||||||
|
} else {
|
||||||
|
entry.resolve(message.result);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (message.method) {
|
||||||
|
const handlers = this.handlers.get(message.method);
|
||||||
|
if (!handlers) return;
|
||||||
|
const params = message.params ?? {};
|
||||||
|
for (const handler of handlers) {
|
||||||
|
try {
|
||||||
|
handler(params);
|
||||||
|
} catch {
|
||||||
|
// 单个监听器异常不影响其他监听器
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private failAllPending(error: Error): void {
|
||||||
|
for (const [id, entry] of this.pending) {
|
||||||
|
entry.reject(error);
|
||||||
|
this.pending.delete(id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
106
packages/rpa-core/src/capabilities/ai-service.ts
Normal file
106
packages/rpa-core/src/capabilities/ai-service.ts
Normal file
@ -0,0 +1,106 @@
|
|||||||
|
import { getHostBridge } from "../engine/host-bridge";
|
||||||
|
import { RuntimeError } from "../engine/types";
|
||||||
|
|
||||||
|
export type ChatContentPart =
|
||||||
|
| { type: "text"; text: string }
|
||||||
|
| { type: "image_url"; image_url: { url: string } };
|
||||||
|
|
||||||
|
export interface ChatMessage {
|
||||||
|
role: "system" | "user" | "assistant";
|
||||||
|
content: string | ChatContentPart[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ResolvedAiConfig {
|
||||||
|
baseUrl: string;
|
||||||
|
apiKey: string;
|
||||||
|
model: string;
|
||||||
|
visionModel: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getAiConfig(): ResolvedAiConfig {
|
||||||
|
const bridge = getHostBridge();
|
||||||
|
// 宿主桥未注入(vitest / CLI 未配置)时返回空配置,requireAiConfig 统一抛 ai_not_configured
|
||||||
|
const config = bridge ? bridge.loadAiConfig() : { baseUrl: "", apiKey: "", model: "", visionModel: "" };
|
||||||
|
return { ...config, visionModel: config.visionModel || config.model };
|
||||||
|
}
|
||||||
|
|
||||||
|
export function requireAiConfig(options: { vision?: boolean } = {}): ResolvedAiConfig {
|
||||||
|
const config = getAiConfig();
|
||||||
|
if (!config.baseUrl || !config.model || (options.vision && !config.visionModel)) {
|
||||||
|
throw new RuntimeError("ai_not_configured", "AI 服务未配置", {
|
||||||
|
recommendation: options.vision
|
||||||
|
? "到 设置 → AI 服务 填写 Base URL 与多模态视觉模型(兼容 OpenAI 协议)"
|
||||||
|
: "到 设置 → AI 服务 填写 Base URL 与模型名(兼容 OpenAI 协议)"
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return config;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ChatCompletionOptions {
|
||||||
|
temperature?: number;
|
||||||
|
vision?: boolean;
|
||||||
|
/** 交互式请求(用户正在等待,如 AI 拾取):默认短超时 45s、最多 2 次,超时不重试 */
|
||||||
|
interactive?: boolean;
|
||||||
|
/** 单次请求超时毫秒数;显式指定后覆盖交互式/后台默认值 */
|
||||||
|
timeoutMs?: number;
|
||||||
|
/** 最大尝试次数;显式指定后覆盖交互式/后台默认值 */
|
||||||
|
maxAttempts?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function chatCompletion(messages: ChatMessage[], options: ChatCompletionOptions = {}): Promise<string> {
|
||||||
|
const config = requireAiConfig({ vision: options.vision });
|
||||||
|
const base = config.baseUrl.replace(/\/+$/, "");
|
||||||
|
const interactive = options.interactive === true;
|
||||||
|
const timeoutMs = Math.max(1000, Math.round(options.timeoutMs ?? (interactive ? 45000 : 170000)));
|
||||||
|
const maxAttempts = Math.max(1, Math.round(options.maxAttempts ?? (interactive ? 2 : 3)));
|
||||||
|
|
||||||
|
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||||
|
let response: Response;
|
||||||
|
try {
|
||||||
|
response = await fetch(base + "/chat/completions", {
|
||||||
|
signal: AbortSignal.timeout(timeoutMs),
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json", Authorization: "Bearer " + config.apiKey },
|
||||||
|
body: JSON.stringify({
|
||||||
|
model: options.vision ? config.visionModel : config.model,
|
||||||
|
messages,
|
||||||
|
temperature: options.temperature ?? 0.3
|
||||||
|
})
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
// 超时(AbortSignal.timeout)不重试:重发只会成倍拉长等待
|
||||||
|
if ((error as { name?: string }).name === "TimeoutError") {
|
||||||
|
throw new RuntimeError("ai_request_timeout", `AI 请求超时(${timeoutMs}ms),请检查网络或稍后重试`);
|
||||||
|
}
|
||||||
|
if (attempt === maxAttempts) throw error;
|
||||||
|
await new Promise(resolve => setTimeout(resolve, 1200 * attempt));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (response.ok) {
|
||||||
|
const data = (await response.json()) as { choices?: Array<{ message?: { content?: string } }> };
|
||||||
|
return data.choices?.[0]?.message?.content ?? "";
|
||||||
|
}
|
||||||
|
|
||||||
|
const detail = (await response.text()).slice(0, 300);
|
||||||
|
const retryable = response.status === 408 || response.status === 429 || response.status >= 500;
|
||||||
|
if (retryable && attempt < maxAttempts) {
|
||||||
|
const retryAfter = Number(response.headers.get("retry-after"));
|
||||||
|
const fallbackDelay = 1400 * attempt;
|
||||||
|
const delay = Number.isFinite(retryAfter) && retryAfter >= 0 && retryAfter <= 15
|
||||||
|
? retryAfter * 1000
|
||||||
|
: fallbackDelay;
|
||||||
|
await new Promise(resolve => setTimeout(resolve, delay));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (response.status === 429) {
|
||||||
|
const suffix = detail ? " 详情:" + detail.slice(0, 180) : "";
|
||||||
|
throw new RuntimeError("ai_rate_limited", "AI 服务限流或配额不足(HTTP 429),请稍后重试或更换可用视觉模型。" + suffix);
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new RuntimeError("ai_request_failed", "AI 请求失败(HTTP " + response.status + "):" + detail);
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new RuntimeError("ai_request_failed", "AI 请求失败:已重试仍未成功");
|
||||||
|
}
|
||||||
50
packages/rpa-core/src/capabilities/basic-encryption.ts
Normal file
50
packages/rpa-core/src/capabilities/basic-encryption.ts
Normal file
@ -0,0 +1,50 @@
|
|||||||
|
import { Buffer } from "node:buffer";
|
||||||
|
import { register } from "../engine/registry";
|
||||||
|
import { RuntimeError, type CapabilityHandler } from "../engine/types";
|
||||||
|
|
||||||
|
const SM4_KEY = "INPUT_PARAM_KEY4";
|
||||||
|
const SM4_KEY_HEX = Buffer.from(SM4_KEY, "utf8").toString("hex");
|
||||||
|
|
||||||
|
function looksLikeBase64(value: string): boolean {
|
||||||
|
const text = value.trim().replace(/\s+/g, "");
|
||||||
|
if (!text || text.length % 4 !== 0 || !/^[A-Za-z0-9+/]+={0,2}$/.test(text)) return false;
|
||||||
|
return Buffer.from(text, "base64").toString("base64").replace(/=+$/, "") === text.replace(/=+$/, "");
|
||||||
|
}
|
||||||
|
|
||||||
|
function readableUtf8(value: string): boolean {
|
||||||
|
if (!value) return false;
|
||||||
|
let controls = 0;
|
||||||
|
for (const char of value) {
|
||||||
|
const code = char.codePointAt(0)!;
|
||||||
|
if (code < 32 && !["\n", "\r", "\t"].includes(char)) controls += 1;
|
||||||
|
if (code === 0xfffd) controls += 10;
|
||||||
|
}
|
||||||
|
return controls === 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function sm4Decrypt(value: string): Promise<string> {
|
||||||
|
const { sm4 } = await import("sm-crypto");
|
||||||
|
const errors: string[] = [];
|
||||||
|
for (const padding of ["pkcs#7", "none"] as const) {
|
||||||
|
try {
|
||||||
|
const output = sm4.decrypt(value.toLowerCase(), SM4_KEY_HEX, { padding });
|
||||||
|
if (output && readableUtf8(output)) return output;
|
||||||
|
errors.push(`${padding}: empty or non-text output`);
|
||||||
|
} catch (error) {
|
||||||
|
errors.push(`${padding}: ${error instanceof Error ? error.message : String(error)}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw new RuntimeError("decryption_failed", "SM4 密文解密失败", { detail: errors.join("; ") });
|
||||||
|
}
|
||||||
|
|
||||||
|
const dodecryption: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
|
const words = String(args.words ?? args.value ?? args.text ?? "");
|
||||||
|
if (!words.trim()) throw new RuntimeError("validation_error", "密文内容为空", { nodeId: node.id, componentId: node.componentId });
|
||||||
|
if (looksLikeBase64(words)) {
|
||||||
|
const decoded = Buffer.from(words, "base64").toString("utf8");
|
||||||
|
if (readableUtf8(decoded)) return decoded;
|
||||||
|
}
|
||||||
|
return sm4Decrypt(words.trim());
|
||||||
|
};
|
||||||
|
|
||||||
|
register("Basic_Dodecryption", dodecryption);
|
||||||
82
packages/rpa-core/src/capabilities/captcha-caps.ts
Normal file
82
packages/rpa-core/src/capabilities/captcha-caps.ts
Normal file
@ -0,0 +1,82 @@
|
|||||||
|
/**
|
||||||
|
* 验证码类 OCR 专门实现。
|
||||||
|
*
|
||||||
|
* 此前 OCR_CalculationCode / OCR_CommonEnglishNumbersCode 与其他 30 个 OCR
|
||||||
|
* 组件共用通用实现(remaining-core 的 ocrImage),只返回整图 OCR 原文:
|
||||||
|
* - 算术验证码返回的是 "3+5=?" 这样的算式原文,没有计算结果,流程无法直接填入答案;
|
||||||
|
* - 真实验证码的干扰线/噪点会让原文混入杂字符,符号常被认成 x/X/*、全角等。
|
||||||
|
*
|
||||||
|
* 这里提供专门处理:OCR 原文 -> 字符归一化 -> 算式解析 -> 计算结果。
|
||||||
|
* 解析失败时返回清洗后的原文并记 warning,不让节点直接失败(调用方可自行判断)。
|
||||||
|
*/
|
||||||
|
import { getHandler, register } from "../engine/registry";
|
||||||
|
import { RuntimeError, type RunContext } from "../engine/types";
|
||||||
|
import type { CatalogComponent, FlowNode } from "../types";
|
||||||
|
|
||||||
|
interface CalculationParse {
|
||||||
|
expression: string;
|
||||||
|
result: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 从 OCR 原文中解析算术算式。
|
||||||
|
* 归一化范围:全角数字/符号、常见运算符误认(x/X/* -> 乘,全角+-×÷,长破折号 -> 减)。
|
||||||
|
*/
|
||||||
|
export function parseCalculation(raw: string): CalculationParse | null {
|
||||||
|
const s = String(raw ?? "")
|
||||||
|
.replace(/[0-9]/g, (c) => String.fromCharCode(c.charCodeAt(0) - 0xfee0))
|
||||||
|
.replace(/+/g, "+")
|
||||||
|
.replace(/[-―—–−]/g, "-")
|
||||||
|
.replace(/[×✕✖xX*X*]/g, "*")
|
||||||
|
.replace(/[÷/]/g, "/")
|
||||||
|
.replace(/[==??。,,、\s]/g, "");
|
||||||
|
const match = s.match(/(\d+)([+\-*/])(\d+)/);
|
||||||
|
if (!match) return null;
|
||||||
|
const a = Number(match[1]);
|
||||||
|
const b = Number(match[3]);
|
||||||
|
const op = match[2];
|
||||||
|
let result: number;
|
||||||
|
if (op === "+") result = a + b;
|
||||||
|
else if (op === "-") result = a - b;
|
||||||
|
else if (op === "*") result = a * b;
|
||||||
|
else {
|
||||||
|
if (b === 0) return null;
|
||||||
|
result = a / b;
|
||||||
|
}
|
||||||
|
if (!Number.isFinite(result)) return null;
|
||||||
|
result = Math.round(result * 100) / 100;
|
||||||
|
const prettyOp = op === "*" ? "×" : op === "/" ? "÷" : op;
|
||||||
|
return { expression: `${a}${prettyOp}${b}`, result };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 普通字符验证码:去除空白与常见杂字符,保留字母数字 */
|
||||||
|
export function cleanCodeText(raw: string): string {
|
||||||
|
return String(raw ?? "")
|
||||||
|
.replace(/\s+/g, "")
|
||||||
|
.replace(/[==??.,,、;;:"'()()\[\]{}]/g, "");
|
||||||
|
}
|
||||||
|
|
||||||
|
async function ocrRaw(args: Record<string, unknown>, node: FlowNode, ctx: RunContext): Promise<string> {
|
||||||
|
// 复用通用 OCR(remaining-core 已注册的 OCR_GeneralTextV2)拿原文,避免重复加载模型
|
||||||
|
const base = getHandler("OCR_GeneralTextV2");
|
||||||
|
if (!base) throw new RuntimeError("capability_not_implemented", "通用 OCR 能力不可用", { nodeId: node.id, componentId: node.componentId });
|
||||||
|
return String(await base(ctx, node, args, {} as CatalogComponent));
|
||||||
|
}
|
||||||
|
|
||||||
|
register("OCR_CalculationCode", async (ctx, node, args) => {
|
||||||
|
const raw = await ocrRaw(args, node, ctx);
|
||||||
|
const parsed = parseCalculation(raw);
|
||||||
|
if (!parsed) {
|
||||||
|
await ctx.log(`算术验证码未能解析出算式(原文:${raw}),已返回清洗原文`, "warn", node.id);
|
||||||
|
return raw.replace(/\s+/g, "");
|
||||||
|
}
|
||||||
|
await ctx.log(`算术验证码:${parsed.expression} = ${parsed.result}(OCR 原文:${raw})`, "info", node.id);
|
||||||
|
return String(parsed.result);
|
||||||
|
});
|
||||||
|
|
||||||
|
register("OCR_CommonEnglishNumbersCode", async (ctx, node, args) => {
|
||||||
|
const raw = await ocrRaw(args, node, ctx);
|
||||||
|
const cleaned = cleanCodeText(raw);
|
||||||
|
await ctx.log(`字符验证码:${cleaned}(OCR 原文:${raw})`, "info", node.id);
|
||||||
|
return cleaned;
|
||||||
|
});
|
||||||
@ -4,7 +4,11 @@ import path from "node:path";
|
|||||||
import { PDFDocument, StandardFonts, degrees, rgb } from "pdf-lib";
|
import { PDFDocument, StandardFonts, degrees, rgb } from "pdf-lib";
|
||||||
import { getHandler, register } from "../engine/registry";
|
import { getHandler, register } from "../engine/registry";
|
||||||
import { RuntimeError, type CapabilityHandler } from "../engine/types";
|
import { RuntimeError, type CapabilityHandler } from "../engine/types";
|
||||||
import { powershell, psString } from "./desktop";
|
import { powershell, psString } from "../driver/windows/powershell";
|
||||||
|
import { defaultEngine } from "../engine/engine";
|
||||||
|
|
||||||
|
// 窗口/屏幕查询统一走驱动(任务 3.3)
|
||||||
|
const getDriver = () => defaultEngine.driver;
|
||||||
|
|
||||||
function text(args: Record<string, unknown>, keys: string[], fallback = ""): string {
|
function text(args: Record<string, unknown>, keys: string[], fallback = ""): string {
|
||||||
for (const key of keys) { const value = args[key]; if (value !== undefined && value !== null && value !== "") return String(value); }
|
for (const key of keys) { const value = args[key]; if (value !== undefined && value !== null && value !== "") return String(value); }
|
||||||
@ -20,25 +24,21 @@ register("Window_GetRect", delegate("Window_GetWindowRect"));
|
|||||||
register("Window_MaximumSize", (ctx, node, args, component) => delegate("Window_SetWindowState")(ctx, node, { ...args, WindowType: "maximize" }, component));
|
register("Window_MaximumSize", (ctx, node, args, component) => delegate("Window_SetWindowState")(ctx, node, { ...args, WindowType: "maximize" }, component));
|
||||||
register(["Keyboard_DriveSend", "Keyboard_DriveSend_V1", "Keyboard_DriveSend_V2"], delegate("Keyboard_Send_V1"));
|
register(["Keyboard_DriveSend", "Keyboard_DriveSend_V1", "Keyboard_DriveSend_V2"], delegate("Keyboard_Send_V1"));
|
||||||
|
|
||||||
register("Window_GetScreenSize", async () => {
|
register("Window_GetScreenSize", async () => getDriver().screen.size());
|
||||||
const output = await powershell("Add-Type -AssemblyName System.Windows.Forms; $b=[System.Windows.Forms.Screen]::PrimaryScreen.Bounds; @{width=$b.Width;height=$b.Height;x=$b.X;y=$b.Y} | ConvertTo-Json -Compress");
|
|
||||||
return JSON.parse(output);
|
|
||||||
});
|
|
||||||
register("Window_WindowList", async (_ctx, _node, args) => {
|
register("Window_WindowList", async (_ctx, _node, args) => {
|
||||||
const title = text(args, ["match_title", "select_title"]); const regex = String(args.is_regex ?? "0") === "1";
|
const title = text(args, ["match_title", "select_title"]); const regex = String(args.is_regex ?? "0") === "1";
|
||||||
const output = await powershell(`Get-Process | Where-Object { $_.MainWindowHandle -ne 0 ${title ? (regex ? `-and $_.MainWindowTitle -match ${psString(title)}` : `-and $_.MainWindowTitle -like ${psString(`*${title}*`)}`) : ""} } | ForEach-Object { @{pid=$_.Id;title=$_.MainWindowTitle;hwnd=$_.MainWindowHandle.ToInt64()} } | ConvertTo-Json -Compress`);
|
return getDriver().window.list({ title, regex });
|
||||||
if (!output) return []; const parsed = JSON.parse(output); return Array.isArray(parsed) ? parsed : [parsed];
|
|
||||||
});
|
});
|
||||||
register("Window_SetWindowShow", async (_ctx, node, args) => {
|
register("Window_SetWindowShow", async (_ctx, node, args) => {
|
||||||
const title = text(args, ["窗口对象", "window_obj", "title"]); if (!title || title === "窗口对象") throw new RuntimeError("validation_error", "窗口标题为空", { nodeId: node.id, componentId: node.componentId });
|
const title = text(args, ["窗口对象", "window_obj", "title"]); if (!title || title === "窗口对象") throw new RuntimeError("validation_error", "窗口标题为空", { nodeId: node.id, componentId: node.componentId });
|
||||||
const show = !text(args, ["WindowType"], "show").toLowerCase().includes("hide");
|
const show = !text(args, ["WindowType"], "show").toLowerCase().includes("hide");
|
||||||
await powershell(`$p=Get-Process|Where-Object{$_.MainWindowTitle -like ${psString(`*${title}*`)}}|Select-Object -First 1;if(!$p){throw '找不到窗口'};Add-Type @'\nusing System;using System.Runtime.InteropServices;public static class W{[DllImport("user32.dll")]public static extern bool ShowWindow(IntPtr h,int n);}\n'@;[W]::ShowWindow($p.MainWindowHandle,${show ? 5 : 0})|Out-Null`);
|
await getDriver().window.setState(title, show ? 5 : 0);
|
||||||
return show;
|
return show;
|
||||||
});
|
});
|
||||||
register("Window_WaitWindow", async (_ctx, node, args) => {
|
register("Window_WaitWindow", async (_ctx, node, args) => {
|
||||||
const title = text(args, ["match_title", "select_title", "window_obj", "target_window"]); if (!title) throw new RuntimeError("validation_error", "窗口标题为空", { nodeId: node.id, componentId: node.componentId });
|
const title = text(args, ["match_title", "select_title", "window_obj", "target_window"]); if (!title) throw new RuntimeError("validation_error", "窗口标题为空", { nodeId: node.id, componentId: node.componentId });
|
||||||
const expected = text(args, ["switchType"], "0") !== "1"; const deadline = Date.now() + Math.max(0, Number(args.second ?? 5)) * 1000;
|
const expected = text(args, ["switchType"], "0") !== "1"; const deadline = Date.now() + Math.max(0, Number(args.second ?? 5)) * 1000;
|
||||||
do { const exists = (await powershell(`$p=Get-Process|Where-Object{$_.MainWindowTitle -like ${psString(`*${title}*`)}}|Select-Object -First 1;[bool]$p`)).toLowerCase() === "true"; if (exists === expected) return true; await new Promise((resolve) => setTimeout(resolve, 200)); } while (Date.now() <= deadline);
|
do { const exists = await getDriver().window.exists(title); if (exists === expected) return true; await new Promise((resolve) => setTimeout(resolve, 200)); } while (Date.now() <= deadline);
|
||||||
return false;
|
return false;
|
||||||
});
|
});
|
||||||
|
|
||||||
265
packages/rpa-core/src/capabilities/cv-vision.ts
Normal file
265
packages/rpa-core/src/capabilities/cv-vision.ts
Normal file
@ -0,0 +1,265 @@
|
|||||||
|
import { register } from "../engine/registry";
|
||||||
|
import { RuntimeError, type CapabilityHandler } from "../engine/types";
|
||||||
|
import { requireScreenService } from "../engine/host-bridge";
|
||||||
|
import { powershell } from "../driver/windows/powershell";
|
||||||
|
import fs from "node:fs";
|
||||||
|
import os from "node:os";
|
||||||
|
import path from "node:path";
|
||||||
|
|
||||||
|
interface CvTarget { text: string; cx: number; cy: number }
|
||||||
|
let paddleOcr: { detect: (image: string) => Promise<{ texts: Array<{ text: string; box?: number[][]; mean?: number }> }> } | null = null;
|
||||||
|
|
||||||
|
async function getOcr() {
|
||||||
|
if (paddleOcr) return paddleOcr;
|
||||||
|
const mod = await import("@gutenye/ocr-node");
|
||||||
|
paddleOcr = await mod.default.create();
|
||||||
|
return paddleOcr as { detect: (image: string) => Promise<{ texts: Array<{ text: string; box?: number[][]; mean?: number }> }> };
|
||||||
|
}
|
||||||
|
|
||||||
|
function cvTarget(args: Record<string, unknown>, nodeId: string): string {
|
||||||
|
for (const key of ["cvSelector", "target", "text"]) {
|
||||||
|
const v = String(args[key] ?? "").trim();
|
||||||
|
if (v) return v.replace(/^text:/, "").replace(/^图片:/, "");
|
||||||
|
}
|
||||||
|
throw new RuntimeError("validation_error", "CV 目标为空", { nodeId, componentId: nodeId, recommendation: "填写要在屏幕上查找的目标文本" });
|
||||||
|
}
|
||||||
|
|
||||||
|
async function gdiGrabScreen(imagePath: string): Promise<void> {
|
||||||
|
const psScript = [
|
||||||
|
// 与点击脚本一致: 先声明 DPI 感知, 否则缩放非 100% 时 Bounds 是逻辑像素, 截屏只覆盖屏幕左上角
|
||||||
|
"Add-Type -MemberDefinition '[DllImport(\"user32.dll\")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value);' -Name RpaCvDpi -Namespace RpaCv | Out-Null;",
|
||||||
|
"try { [RpaCv.RpaCvDpi]::SetProcessDpiAwarenessContext([IntPtr](-4)) | Out-Null } catch {}",
|
||||||
|
"Add-Type -AssemblyName System.Drawing | Out-Null;",
|
||||||
|
"Add-Type -AssemblyName System.Windows.Forms | Out-Null;",
|
||||||
|
"$b = [System.Windows.Forms.SystemInformation]::VirtualScreen;",
|
||||||
|
"$bmp = New-Object System.Drawing.Bitmap($b.Width, $b.Height);",
|
||||||
|
"$g = [System.Drawing.Graphics]::FromImage($bmp);",
|
||||||
|
"$g.CopyFromScreen($b.X, $b.Y, 0, 0, $b.Size);",
|
||||||
|
"$g.Dispose();",
|
||||||
|
"$bmp.Save(" + JSON.stringify(imagePath) + ");",
|
||||||
|
"$bmp.Dispose();"
|
||||||
|
].join(" ");
|
||||||
|
await powershell(psScript, 30000);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 一次屏幕截图: 图片左上角在虚拟桌面上的绝对 DIP 坐标 + 物理/DIP 缩放比 */
|
||||||
|
interface ScreenShot { imagePath: string; dipX: number; dipY: number; scale: number }
|
||||||
|
|
||||||
|
/** 宿主屏幕服务逐显示器截屏(每屏原生分辨率), 不可用时回退 GDI 全虚拟桌面单图 */
|
||||||
|
async function grabScreenAll(baseDir: string): Promise<ScreenShot[]> {
|
||||||
|
fs.mkdirSync(baseDir, { recursive: true });
|
||||||
|
try {
|
||||||
|
const screenHost = requireScreenService();
|
||||||
|
const shots: ScreenShot[] = [];
|
||||||
|
for (const display of screenHost.listDisplays()) {
|
||||||
|
const scale = display.scaleFactor || 1;
|
||||||
|
try {
|
||||||
|
const pngBytes = await screenHost.captureDisplay(
|
||||||
|
display.id,
|
||||||
|
Math.round(display.bounds.width * scale),
|
||||||
|
Math.round(display.bounds.height * scale)
|
||||||
|
);
|
||||||
|
if (!pngBytes?.length) continue;
|
||||||
|
const imagePath = path.join(baseDir, `rpa-cv-screen-${display.id}-${Date.now()}.png`);
|
||||||
|
fs.writeFileSync(imagePath, Buffer.from(pngBytes));
|
||||||
|
shots.push({ imagePath, dipX: display.bounds.x, dipY: display.bounds.y, scale });
|
||||||
|
} catch {}
|
||||||
|
}
|
||||||
|
if (shots.length) return shots;
|
||||||
|
} catch {}
|
||||||
|
const imagePath = path.join(baseDir, `rpa-cv-screen-${Date.now()}.png`);
|
||||||
|
await gdiGrabScreen(imagePath);
|
||||||
|
return [{ imagePath, dipX: 0, dipY: 0, scale: 1 }];
|
||||||
|
}
|
||||||
|
|
||||||
|
interface OcrLine { text: string; box?: number[][]; mean?: number }
|
||||||
|
|
||||||
|
/** 搜索区域: 框选生成时含 imagePath(区域截图)+x/y(屏幕绝对 DIP 坐标)+scale(该屏缩放); 也可只给坐标范围 */
|
||||||
|
interface CvRegion { imagePath?: string; x?: number; y?: number; width?: number; height?: number; scale?: number }
|
||||||
|
|
||||||
|
function parseRegionArg(value: unknown): CvRegion | null {
|
||||||
|
if (!value) return null;
|
||||||
|
if (typeof value === "object") return value as CvRegion;
|
||||||
|
const s = String(value).trim();
|
||||||
|
if (!s) return null;
|
||||||
|
if (s.startsWith("{")) {
|
||||||
|
try { return JSON.parse(s) as CvRegion; } catch { return null; }
|
||||||
|
}
|
||||||
|
if (/\.(png|jpe?g|bmp)$/i.test(s) || fs.existsSync(s)) return { imagePath: s };
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function boxCenter(box: number[][]): { cx: number; cy: number } {
|
||||||
|
const xs = box.map(p => p[0]);
|
||||||
|
const ys = box.map(p => p[1]);
|
||||||
|
return { cx: (Math.min(...xs) + Math.max(...xs)) / 2, cy: (Math.min(...ys) + Math.max(...ys)) / 2 };
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function locateTextOnScreen(
|
||||||
|
target: string,
|
||||||
|
nodeId: string,
|
||||||
|
region?: CvRegion | null
|
||||||
|
): Promise<{ target: string; text: string; cx: number; cy: number }> {
|
||||||
|
const ocr = await getOcr();
|
||||||
|
const keyword = target.trim();
|
||||||
|
// 命中中心换算为屏幕绝对 DIP 坐标
|
||||||
|
let found: { text: string; cx: number; cy: number } | null = null;
|
||||||
|
const cleanups: string[] = [];
|
||||||
|
|
||||||
|
if (region?.imagePath && fs.existsSync(region.imagePath)) {
|
||||||
|
// 框选截图: OCR 只识别该图; 图内物理坐标 ÷ 该屏缩放 + 框选绝对 DIP 原点
|
||||||
|
const scale = Number(region.scale ?? 1) || 1;
|
||||||
|
const result = await ocr.detect(region.imagePath);
|
||||||
|
const hit = (result.texts ?? []).find(t => t.text && t.text.includes(keyword) && t.box && t.box.length >= 4);
|
||||||
|
if (hit?.box) {
|
||||||
|
const c = boxCenter(hit.box);
|
||||||
|
found = {
|
||||||
|
text: hit.text,
|
||||||
|
cx: (Number(region.x ?? 0) || 0) + c.cx / scale,
|
||||||
|
cy: (Number(region.y ?? 0) || 0) + c.cy / scale
|
||||||
|
};
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// 全屏查找: 逐显示器截屏识别, 覆盖副屏(含负坐标屏); 命中即停
|
||||||
|
const shots = await grabScreenAll(path.join(os.tmpdir(), "rpa-cv-shots"));
|
||||||
|
for (const shot of shots) {
|
||||||
|
const result = await ocr.detect(shot.imagePath);
|
||||||
|
const candidates = (result.texts ?? []).filter(t => t.text && t.text.includes(keyword) && t.box && t.box.length >= 4);
|
||||||
|
let hit: OcrLine | undefined = candidates[0];
|
||||||
|
// 给了坐标范围(绝对 DIP)时, 过滤中心落在区域内的命中
|
||||||
|
if (hit && region?.width && region?.height) {
|
||||||
|
const rx = Number(region.x ?? 0) || 0;
|
||||||
|
const ry = Number(region.y ?? 0) || 0;
|
||||||
|
hit = candidates.find(t => {
|
||||||
|
const c = boxCenter(t.box!);
|
||||||
|
const dx = shot.dipX + c.cx / shot.scale;
|
||||||
|
const dy = shot.dipY + c.cy / shot.scale;
|
||||||
|
return dx >= rx && dx <= rx + region.width! && dy >= ry && dy <= ry + region.height!;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (hit?.box) {
|
||||||
|
const c = boxCenter(hit.box);
|
||||||
|
found = { text: hit.text, cx: shot.dipX + c.cx / shot.scale, cy: shot.dipY + c.cy / shot.scale };
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
cleanups.push(shot.imagePath);
|
||||||
|
}
|
||||||
|
// 成功命中时清理全部截图, 未命中保留全部以便排查
|
||||||
|
if (found) for (const p of shots.map(s => s.imagePath)) { try { fs.unlinkSync(p); } catch {} }
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!found) {
|
||||||
|
throw new RuntimeError("cv_not_found", "屏幕上未找到目标文本:" + keyword + (region?.imagePath ? "(搜索范围: 框选区域)" : ""), {
|
||||||
|
nodeId,
|
||||||
|
componentId: nodeId,
|
||||||
|
recommendation: "确认目标文本在" + (region?.imagePath ? "框选区域内" : "屏幕上(含副屏)") + "可见且清晰"
|
||||||
|
});
|
||||||
|
}
|
||||||
|
for (const p of cleanups) { try { fs.unlinkSync(p); } catch {} }
|
||||||
|
return { target: keyword, text: found.text, cx: found.cx, cy: found.cy };
|
||||||
|
}
|
||||||
|
function screenUser32(): string {
|
||||||
|
return [
|
||||||
|
"Add-Type -MemberDefinition '[DllImport(\"user32.dll\")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value); [DllImport(\"user32.dll\")] public static extern bool SetCursorPos(int X, int Y); [DllImport(\"user32.dll\")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra);' -Name RpaCvInput -Namespace RpaCv | Out-Null;",
|
||||||
|
"try { [RpaCv.RpaCvInput]::SetProcessDpiAwarenessContext([IntPtr](-4)) | Out-Null } catch {}"
|
||||||
|
].join("\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
function coordinateScript(x: number, y: number, action: "hover" | "click" | "doubleClick" | "paste", value = ""): string {
|
||||||
|
const lines = [
|
||||||
|
screenUser32(),
|
||||||
|
"if (-not [RpaCv.RpaCvInput]::SetCursorPos(" + x + ", " + y + ")) { throw 'cursor_move_failed' }",
|
||||||
|
"Start-Sleep -Milliseconds 100"
|
||||||
|
];
|
||||||
|
if (action !== "hover") {
|
||||||
|
lines.push(
|
||||||
|
"[RpaCv.RpaCvInput]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)",
|
||||||
|
"Start-Sleep -Milliseconds 40",
|
||||||
|
"[RpaCv.RpaCvInput]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (action === "doubleClick") {
|
||||||
|
lines.push(
|
||||||
|
"Start-Sleep -Milliseconds 70",
|
||||||
|
"[RpaCv.RpaCvInput]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)",
|
||||||
|
"Start-Sleep -Milliseconds 40",
|
||||||
|
"[RpaCv.RpaCvInput]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (action === "paste") {
|
||||||
|
lines.push(
|
||||||
|
"Add-Type -AssemblyName System.Windows.Forms",
|
||||||
|
"Start-Sleep -Milliseconds 220",
|
||||||
|
"$saved = $null",
|
||||||
|
"try { $saved = [System.Windows.Forms.Clipboard]::GetText() } catch {}",
|
||||||
|
"try { [System.Windows.Forms.Clipboard]::SetText(" + JSON.stringify(value) + ") } catch { throw 'clipboard_failed' }",
|
||||||
|
"Start-Sleep -Milliseconds 80",
|
||||||
|
"[System.Windows.Forms.SendKeys]::SendWait('^a')",
|
||||||
|
"Start-Sleep -Milliseconds 60",
|
||||||
|
"[System.Windows.Forms.SendKeys]::SendWait('^v')",
|
||||||
|
"Start-Sleep -Milliseconds 280",
|
||||||
|
"try { if ($saved) { [System.Windows.Forms.Clipboard]::SetText($saved) } else { [System.Windows.Forms.Clipboard]::Clear() } } catch {}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
lines.push("Write-Output (@{ ok = $true; x = " + x + "; y = " + y + " } | ConvertTo-Json -Compress)");
|
||||||
|
return lines.join("\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
/** DIP → 物理像素(SetCursorPos 在 DPI 感知进程里用物理坐标); 无宿主屏幕服务时按原值(缩放 100% 等价) */
|
||||||
|
async function toPhysical(x: number, y: number): Promise<{ x: number; y: number }> {
|
||||||
|
try {
|
||||||
|
const point = requireScreenService().dipToScreenPoint({ x: Math.round(x), y: Math.round(y) });
|
||||||
|
return { x: point.x, y: point.y };
|
||||||
|
} catch {}
|
||||||
|
return { x: Math.round(x), y: Math.round(y) };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function locateFor(args: Record<string, unknown>, node: { id: string; componentId: string }) {
|
||||||
|
const target = cvTarget(args, node.id);
|
||||||
|
// 可选搜索区域: 框选生成 {x,y,width,height,imagePath,scale}, 限定 OCR 查找范围
|
||||||
|
const region = parseRegionArg(args.searchRegion ?? args.region);
|
||||||
|
const located = await locateTextOnScreen(target, node.id, region);
|
||||||
|
const offsetX = Number(args.offsetX ?? args.offset_x ?? 0) || 0;
|
||||||
|
const offsetY = Number(args.offsetY ?? args.offset_y ?? 0) || 0;
|
||||||
|
return { located, x: Math.round(located.cx + offsetX), y: Math.round(located.cy + offsetY) };
|
||||||
|
}
|
||||||
|
|
||||||
|
const getTextGround: CapabilityHandler = async (ctx, node, args) => {
|
||||||
|
const target = cvTarget(args, node.id);
|
||||||
|
const result = await locateTextOnScreen(target, node.id);
|
||||||
|
ctx.log("CV_GetText found: " + result.text, "info", node.id);
|
||||||
|
return result.text;
|
||||||
|
};
|
||||||
|
|
||||||
|
const setTextGround: CapabilityHandler = async (ctx, node, args) => {
|
||||||
|
const value = String(args.value ?? args.text ?? "");
|
||||||
|
if (!value) throw new RuntimeError("validation_error", "CV 文本输入内容为空", { nodeId: node.id, componentId: node.componentId });
|
||||||
|
const { x, y } = await locateFor(args, node);
|
||||||
|
const phys = await toPhysical(x, y);
|
||||||
|
await powershell(coordinateScript(phys.x, phys.y, "paste", value), 30000);
|
||||||
|
ctx.log("CV_SetText input at " + phys.x + "," + phys.y, "info", node.id);
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
|
||||||
|
const mouseHoverGround: CapabilityHandler = async (ctx, node, args) => {
|
||||||
|
const { x, y } = await locateFor(args, node);
|
||||||
|
const phys = await toPhysical(x, y);
|
||||||
|
await powershell(coordinateScript(phys.x, phys.y, "hover"), 15000);
|
||||||
|
ctx.log("CV_MouseHover hover at " + phys.x + "," + phys.y, "info", node.id);
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
|
||||||
|
const mouseClickGround: CapabilityHandler = async (ctx, node, args) => {
|
||||||
|
const { x, y } = await locateFor(args, node);
|
||||||
|
const phys = await toPhysical(x, y);
|
||||||
|
const clickway = String(args.clickway ?? "0");
|
||||||
|
const action = clickway === "1" || /double/i.test(clickway) ? "doubleClick" as const : "click" as const;
|
||||||
|
await powershell(coordinateScript(phys.x, phys.y, action), 15000);
|
||||||
|
ctx.log("CV_MouseClick " + action + " at " + phys.x + "," + phys.y, "info", node.id);
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
|
||||||
|
register("CV_GetText", getTextGround);
|
||||||
|
register("CV_SetText", setTextGround);
|
||||||
|
register("CV_MouseHover", mouseHoverGround);
|
||||||
|
register("CV_MouseClick", mouseClickGround);
|
||||||
@ -152,8 +152,24 @@ register("Bivartable_PivotDf", (_ctx, _node, args) => {
|
|||||||
async function exportExcel(source: Table, filePath: string): Promise<string> {
|
async function exportExcel(source: Table, filePath: string): Promise<string> {
|
||||||
const workbook = new ExcelJS.Workbook(); const sheet = workbook.addWorksheet("Sheet1"); sheet.addRows(source as ExcelJS.CellValue[][]); await fs.mkdir(path.dirname(filePath), { recursive: true }); await workbook.xlsx.writeFile(filePath); return filePath;
|
const workbook = new ExcelJS.Workbook(); const sheet = workbook.addWorksheet("Sheet1"); sheet.addRows(source as ExcelJS.CellValue[][]); await fs.mkdir(path.dirname(filePath), { recursive: true }); await workbook.xlsx.writeFile(filePath); return filePath;
|
||||||
}
|
}
|
||||||
register(["Bivartable_DfWriteToFile", "Bivartable_DoSave"], async (_ctx, node, args) => {
|
register(["Bivartable_DfWriteToFile", "Bivartable_DoSave"], async (ctx, node, args) => {
|
||||||
const filePath = path.resolve(required(args.filePath ?? args.path ?? args.savePath, "Excel路径", node.id, node.componentId)); return exportExcel(table(args.df), filePath);
|
const filePath = path.resolve(required(args.filePath ?? args.path ?? args.savePath, "Excel路径", node.id, node.componentId));
|
||||||
|
try {
|
||||||
|
return await exportExcel(table(args.df), filePath);
|
||||||
|
} catch (error) {
|
||||||
|
// 目标文件被占用(正被 WPS/Excel 打开等)时,自动另存为带时间戳的文件,不让流程失败
|
||||||
|
const code = (error as NodeJS.ErrnoException | undefined)?.code;
|
||||||
|
const message = error instanceof Error ? error.message : String(error);
|
||||||
|
if (code === "EBUSY" || code === "EPERM" || /EBUSY|resource busy|being used by another/i.test(message)) {
|
||||||
|
const info = path.parse(filePath);
|
||||||
|
const stamp = new Date().toISOString().slice(0, 19).replace(/[-:T]/g, "").replace(/^(\d{8})(\d{6})$/, "$1_$2");
|
||||||
|
const fallback = path.join(info.dir, `${info.name}_${stamp}${info.ext}`);
|
||||||
|
await exportExcel(table(args.df), fallback);
|
||||||
|
ctx.log(`目标文件被占用(可能正被打开),已另存为:${fallback}`, "warn", node.id);
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
});
|
});
|
||||||
register("Bivartable_DoOpenExcel", async (_ctx, node, args) => {
|
register("Bivartable_DoOpenExcel", async (_ctx, node, args) => {
|
||||||
const filePath = path.resolve(required(args.filePath, "Excel路径", node.id, node.componentId)); const workbook = new ExcelJS.Workbook(); await workbook.xlsx.readFile(filePath); const sheet = workbook.getWorksheet(String(args.sheetName || "Sheet1")) || workbook.worksheets[0]; return sheet.getSheetValues().slice(1).map((row) => Array.isArray(row) ? row.slice(1).map((cell) => cell ?? "") : []);
|
const filePath = path.resolve(required(args.filePath, "Excel路径", node.id, node.componentId)); const workbook = new ExcelJS.Workbook(); await workbook.xlsx.readFile(filePath); const sheet = workbook.getWorksheet(String(args.sheetName || "Sheet1")) || workbook.worksheets[0]; return sheet.getSheetValues().slice(1).map((row) => Array.isArray(row) ? row.slice(1).map((cell) => cell ?? "") : []);
|
||||||
@ -111,18 +111,39 @@ register("StrUtil_InStr", (_ctx, _node, args) => {
|
|||||||
});
|
});
|
||||||
register("StrUtil_Replace", (_ctx, _node, args) => {
|
register("StrUtil_Replace", (_ctx, _node, args) => {
|
||||||
const text = str(first(args, ["string"]));
|
const text = str(first(args, ["string"]));
|
||||||
const search = str(first(args, ["search", "Config"]));
|
// catalog 契约:search 为下拉枚举("0"内容/"1"数字/"2"手机号/"3"邮箱/"4"身份证/"5"自定义正则),查找目标在 Config
|
||||||
|
// 旧实现直接把枚举值 "0" 当字面文本分割,导致按内容替换永远不生效(W26 系枚举映射 bug 同族)
|
||||||
|
const modeRaw = first(args, ["search"]);
|
||||||
|
const mode = str(modeRaw === undefined ? "0" : modeRaw);
|
||||||
|
const config = str(first(args, ["Config"]));
|
||||||
const replacement = str(first(args, ["replace"]));
|
const replacement = str(first(args, ["replace"]));
|
||||||
return text.split(search).join(replacement);
|
const presets: Record<string, RegExp> = {
|
||||||
|
"1": /\d+/g,
|
||||||
|
"2": /1[3-9]\d{9}/g,
|
||||||
|
"3": /[\w.+-]+@[\w-]+(?:\.[\w-]+)+/g,
|
||||||
|
"4": /\d{17}[\dXx]/g
|
||||||
|
};
|
||||||
|
if (presets[mode]) return text.replace(presets[mode], replacement);
|
||||||
|
if (mode === "5") {
|
||||||
|
try {
|
||||||
|
return text.replace(new RegExp(config, "g"), replacement);
|
||||||
|
} catch {
|
||||||
|
return text;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// "0" = 指定内容;兼容旧契约:Config 为空时退回把 search 当字面文本
|
||||||
|
const target = config || mode;
|
||||||
|
return text.split(target).join(replacement);
|
||||||
});
|
});
|
||||||
register(["StrUtil_LowerCase"], (_ctx, _node, args) => str(first(args, ["string"])).toLowerCase());
|
register(["StrUtil_LowerCase"], (_ctx, _node, args) => str(first(args, ["string"])).toLowerCase());
|
||||||
register(["StrUtil_UpperCase"], (_ctx, _node, args) => str(first(args, ["string"])).toUpperCase());
|
register(["StrUtil_UpperCase"], (_ctx, _node, args) => str(first(args, ["string"])).toUpperCase());
|
||||||
register("StrUtil_CaseChange", (_ctx, _node, args) => {
|
register("StrUtil_CaseChange", (_ctx, _node, args) => {
|
||||||
const text = str(first(args, ["string"]));
|
const text = str(first(args, ["string"]));
|
||||||
const mode = str(first(args, ["case"]));
|
const mode = str(first(args, ["case"]));
|
||||||
if (mode.includes("小写")) return text.toLowerCase();
|
// 目录选项值为 "1"(大写)/"2"(小写),文字模式兼容 大写/小写/首字母大写
|
||||||
if (mode.includes("大写")) return text.toUpperCase();
|
|
||||||
if (mode.includes("首")) return text ? text[0].toUpperCase() + text.slice(1) : "";
|
if (mode.includes("首")) return text ? text[0].toUpperCase() + text.slice(1) : "";
|
||||||
|
if (mode === "1" || mode.includes("大写")) return text.toUpperCase();
|
||||||
|
if (mode === "2" || mode.includes("小写")) return text.toLowerCase();
|
||||||
return text;
|
return text;
|
||||||
});
|
});
|
||||||
register("StrUtil_Trim", (_ctx, _node, args) => str(first(args, ["string"])).trim());
|
register("StrUtil_Trim", (_ctx, _node, args) => str(first(args, ["string"])).trim());
|
||||||
@ -463,25 +484,30 @@ function toCsv(rows: unknown[][]): string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
register(["CSV_Read", "CSV_ReadNew"], async (_ctx, node, args) => {
|
register(["CSV_Read", "CSV_ReadNew"], async (_ctx, node, args) => {
|
||||||
const filePath = path.resolve(str(first(args, ["path"])));
|
const filePath = str(first(args, ["path"])).trim();
|
||||||
if (!filePath) throw new RuntimeError("validation_error", "CSV 路径为空", { nodeId: node.id, componentId: node.componentId });
|
if (!filePath) throw new RuntimeError("validation_error", "CSV 路径为空", { nodeId: node.id, componentId: node.componentId });
|
||||||
const text = await fs.readFile(filePath, "utf8");
|
const text = await fs.readFile(path.resolve(filePath), "utf8");
|
||||||
return parseCsv(text);
|
return parseCsv(text);
|
||||||
});
|
});
|
||||||
register(["CSV_Write", "CSV_WriteNew"], async (_ctx, node, args) => {
|
register(["CSV_Write", "CSV_WriteNew"], async (_ctx, node, args) => {
|
||||||
const filePath = path.resolve(str(first(args, ["path"])));
|
const filePath = str(first(args, ["path"])).trim();
|
||||||
if (!filePath) throw new RuntimeError("validation_error", "CSV 路径为空", { nodeId: node.id, componentId: node.componentId });
|
if (!filePath) throw new RuntimeError("validation_error", "CSV 路径为空", { nodeId: node.id, componentId: node.componentId });
|
||||||
const table = asArray(first(args, ["table"])).map((row) => asArray(row));
|
const table = asArray(first(args, ["table"])).map((row) => asArray(row));
|
||||||
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
const resolved = path.resolve(filePath);
|
||||||
await fs.writeFile(filePath, toCsv(table), "utf8");
|
await fs.mkdir(path.dirname(resolved), { recursive: true });
|
||||||
return filePath;
|
await fs.writeFile(resolved, toCsv(table), "utf8");
|
||||||
|
return resolved;
|
||||||
});
|
});
|
||||||
|
|
||||||
/* ---------------- Basic ---------------- */
|
/* ---------------- Basic ---------------- */
|
||||||
register(["Basic_ConvertNumber", "Basic_StringToNum"], (_ctx, _node, args) => num(first(args, ["val"])));
|
register(["Basic_ConvertNumber", "Basic_StringToNum"], (_ctx, _node, args) => num(first(args, ["val"])));
|
||||||
register("Basic_ConvertString", (_ctx, _node, args) => str(first(args, ["val"])));
|
register("Basic_ConvertString", (_ctx, _node, args) => str(first(args, ["val"])));
|
||||||
register("Basic_ConvertInt", (_ctx, _node, args) => Math.trunc(num(first(args, ["val"]))));
|
register("Basic_ConvertInt", (_ctx, _node, args) => Math.trunc(num(first(args, ["val"]))));
|
||||||
register("Basic_ConvertBool", (_ctx, _node, args) => bool(first(args, ["val"])));
|
register("Basic_ConvertBool", (_ctx, _node, args) => {
|
||||||
|
const value = first(args, ["val"]);
|
||||||
|
if (typeof value === "number") return value !== 0;
|
||||||
|
return bool(value);
|
||||||
|
});
|
||||||
register("Basic_IsNull", (_ctx, _node, args) => {
|
register("Basic_IsNull", (_ctx, _node, args) => {
|
||||||
const value = first(args, ["val"]);
|
const value = first(args, ["val"]);
|
||||||
return value === null || value === undefined || value === "";
|
return value === null || value === undefined || value === "";
|
||||||
1069
packages/rpa-core/src/capabilities/desktop-uia.ts
Normal file
1069
packages/rpa-core/src/capabilities/desktop-uia.ts
Normal file
File diff suppressed because it is too large
Load Diff
287
packages/rpa-core/src/capabilities/desktop.ts
Normal file
287
packages/rpa-core/src/capabilities/desktop.ts
Normal file
@ -0,0 +1,287 @@
|
|||||||
|
import { register } from "../engine/registry";
|
||||||
|
import { RuntimeError, type CapabilityHandler } from "../engine/types";
|
||||||
|
import { type ManualOperationSettings, defaultEngine } from "../engine/engine";
|
||||||
|
import { powershell, psString } from "../driver/windows/powershell";
|
||||||
|
|
||||||
|
// 人为拟真设置本体挂在引擎实例上(任务 1.3),此处委托 defaultEngine。
|
||||||
|
const manualOperation: ManualOperationSettings = defaultEngine.manualOperation;
|
||||||
|
// 桌面自动化驱动(任务 3.3):user32 注入原语统一走 WindowsDriver
|
||||||
|
// 惰性获取驱动(迁移阶段1-A1):非 win32 上 import 本模块不再抛错,
|
||||||
|
// 调用桌面能力时才触发 platform_unsupported;也使注入 driver 的引擎可被测试/宿主使用
|
||||||
|
const getDriver = () => defaultEngine.driver;
|
||||||
|
|
||||||
|
function jitter(value: number, amount = 3): number {
|
||||||
|
return Math.max(0, Math.round(value + (Math.random() * amount * 2 - amount)));
|
||||||
|
}
|
||||||
|
|
||||||
|
function manualJitter(point: [number, number], move: boolean, click: boolean): [number, number] {
|
||||||
|
if (!manualOperation.enabled) return point;
|
||||||
|
const amount = (click && manualOperation.randomClick) || (move && manualOperation.randomMove) ? 3 : 0;
|
||||||
|
return [jitter(point[0], amount), jitter(point[1], amount)];
|
||||||
|
}
|
||||||
|
|
||||||
|
async function manualDelay(): Promise<void> {
|
||||||
|
if (!manualOperation.enabled || !manualOperation.randomSleep) return;
|
||||||
|
const low = Math.max(0, manualOperation.minMs);
|
||||||
|
const high = Math.max(low, manualOperation.maxMs);
|
||||||
|
const timeout = low + Math.random() * (high - low);
|
||||||
|
if (timeout > 0) await new Promise((resolve) => setTimeout(resolve, timeout));
|
||||||
|
}
|
||||||
|
function text(args: Record<string, unknown>, keys: string[], fallback = ""): string {
|
||||||
|
for (const key of keys) {
|
||||||
|
if (args[key] !== undefined && args[key] !== null && args[key] !== "") return String(args[key]);
|
||||||
|
}
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
function bool(value: unknown): boolean {
|
||||||
|
return [true, 1, "1", "true", "True", "是", "开启"].includes(value as never);
|
||||||
|
}
|
||||||
|
|
||||||
|
function number(args: Record<string, unknown>, keys: string[], fallback = 0): number {
|
||||||
|
const value = Number(text(args, keys, String(fallback)));
|
||||||
|
return Number.isFinite(value) ? value : fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
function point(value: unknown): [number, number] {
|
||||||
|
if (Array.isArray(value)) return [Number(value[0]) || 0, Number(value[1]) || 0];
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(String(value || "[0,0]"));
|
||||||
|
return point(parsed);
|
||||||
|
} catch {
|
||||||
|
const values = String(value || "").split(/[ ,,]+/).map(Number);
|
||||||
|
return [values[0] || 0, values[1] || 0];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const keyboardSend: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
|
// 字段: words=输入内容(键盘输入), keys/key=按键组合(按键系列), interval=输入间隔(V1毫秒/基础版秒), fuctype=输入方式(0后台接口/1人工模拟/2剪贴板粘贴), window_title=输入目标窗口(留空=当前焦点窗口)
|
||||||
|
const keys = text(args, ["words", "keys", "key", "text", "value"]);
|
||||||
|
if (!keys) throw new RuntimeError("validation_error", "键盘输入为空", { nodeId: node.id, componentId: node.componentId });
|
||||||
|
await manualDelay();
|
||||||
|
const rawInterval = number(args, ["interval"], 0);
|
||||||
|
const intervalMs = node.componentId === "Keyboard_Send" ? Math.max(0, Math.round(rawInterval * 1000)) : Math.max(0, Math.round(rawInterval));
|
||||||
|
const mode = text(args, ["fuctype"], "0");
|
||||||
|
const isTextSend = node.componentId === "Keyboard_Send" || node.componentId === "Keyboard_Send_V1";
|
||||||
|
// 指定目标窗口时先激活再输入,内容才会进到目标窗口而不是运行时的焦点窗口
|
||||||
|
const targetTitle = text(args, ["window_title", "target_window", "window_obj", "match_title", "select_title"]);
|
||||||
|
await getDriver().keyboard.send({ keys, intervalMs, mode: mode === "2" ? "clipboard" : mode === "1" ? "manual" : "api", textMode: isTextSend, activateTitle: targetTitle || undefined });
|
||||||
|
await manualDelay();
|
||||||
|
return keys;
|
||||||
|
};
|
||||||
|
|
||||||
|
const enableManualOperation: CapabilityHandler = async (_ctx, _node, args) => {
|
||||||
|
const min = number(args, ["min_time", "minTime"], 1) * 1000;
|
||||||
|
const max = number(args, ["max_time", "maxTime"], 3) * 1000;
|
||||||
|
manualOperation.enabled = true;
|
||||||
|
manualOperation.randomMove = bool(args.random_move ?? true);
|
||||||
|
manualOperation.randomClick = bool(args.random_click ?? true);
|
||||||
|
manualOperation.randomSleep = bool(args.random_sleep ?? true);
|
||||||
|
manualOperation.minMs = min;
|
||||||
|
manualOperation.maxMs = Math.max(min, max);
|
||||||
|
return { enabled: true, randomMove: manualOperation.randomMove, randomClick: manualOperation.randomClick, randomSleep: manualOperation.randomSleep, minMs: manualOperation.minMs, maxMs: manualOperation.maxMs };
|
||||||
|
};
|
||||||
|
|
||||||
|
const disableManualOperation: CapabilityHandler = async () => {
|
||||||
|
manualOperation.enabled = false;
|
||||||
|
return { enabled: false };
|
||||||
|
};
|
||||||
|
|
||||||
|
const mouseClick: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
|
const [x, y] = manualJitter(point(args.position ?? args.point ?? args.location ?? "[0,0]"), false, true);
|
||||||
|
const button = text(args, ["side", "button"], "0").toLowerCase();
|
||||||
|
await getDriver().mouse.click({ x, y }, { button: button.includes("right") || button === "1" ? "right" : "left" });
|
||||||
|
return { x, y };
|
||||||
|
};
|
||||||
|
|
||||||
|
const mouseMove: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
|
const [x, y] = manualJitter(point(args.position ?? args.point ?? args.location ?? "[0,0]"), true, false);
|
||||||
|
await getDriver().mouse.moveTo({ x, y });
|
||||||
|
return { x, y };
|
||||||
|
};
|
||||||
|
|
||||||
|
const mouseDrag: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
|
const [sx, sy] = point(args.Start ?? args.start ?? "[0,0]");
|
||||||
|
const [ex, ey] = point(args.End ?? args.end ?? "[0,0]");
|
||||||
|
await getDriver().mouse.drag({ x: sx, y: sy }, { x: ex, y: ey }, { holdMs: 80, releaseDelayMs: 80 });
|
||||||
|
return { start: [sx, sy], end: [ex, ey] };
|
||||||
|
};
|
||||||
|
|
||||||
|
const mouseRoll: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
|
const amount = number(args, ["RollNum", "rollNum", "amount"], 1) * 120;
|
||||||
|
await getDriver().mouse.scroll(amount);
|
||||||
|
return amount;
|
||||||
|
};
|
||||||
|
|
||||||
|
const setActiveWindow: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
|
const selector = text(args, ["window_obj", "match_title", "select_title", "title", "target_window", "window_title"]);
|
||||||
|
if (!selector) throw new RuntimeError("validation_error", "窗口标题为空", { nodeId: node.id, componentId: node.componentId });
|
||||||
|
return getDriver().window.focus(selector);
|
||||||
|
};
|
||||||
|
|
||||||
|
const getActiveWindow: CapabilityHandler = async () => getDriver().window.foregroundTitle();
|
||||||
|
|
||||||
|
const windowExists: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
|
const selector = text(args, ["window_obj", "match_title", "select_title", "title", "target_window", "window_title"]);
|
||||||
|
if (!selector) throw new RuntimeError("validation_error", "窗口标题为空", { nodeId: node.id, componentId: node.componentId });
|
||||||
|
return getDriver().window.exists(selector);
|
||||||
|
};
|
||||||
|
|
||||||
|
const closeWindow: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
|
const selector = text(args, ["window_obj", "match_title", "select_title", "title", "target_window", "window_title"]);
|
||||||
|
if (!selector) throw new RuntimeError("validation_error", "窗口标题为空", { nodeId: node.id, componentId: node.componentId });
|
||||||
|
return getDriver().window.close(selector);
|
||||||
|
};
|
||||||
|
|
||||||
|
const runApp: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
|
const executable = text(args, ["executeable", "executable", "path"]);
|
||||||
|
if (!executable) throw new RuntimeError("validation_error", "应用路径为空", { nodeId: node.id, componentId: node.componentId });
|
||||||
|
const params = text(args, ["params", "arguments"]);
|
||||||
|
const cwd = text(args, ["cwd"]);
|
||||||
|
const argPart = params ? ` -ArgumentList ${psString(params)}` : "";
|
||||||
|
const result = await powershell(`$p = Start-Process -FilePath ${psString(executable)}${argPart} ${cwd ? `-WorkingDirectory ${psString(cwd)}` : ""} -PassThru; $p.Id`);
|
||||||
|
return Number(result) || result;
|
||||||
|
};
|
||||||
|
|
||||||
|
const clipboardGet: CapabilityHandler = async () => powershell("Get-Clipboard -Raw");
|
||||||
|
const clipboardSet: CapabilityHandler = async (_ctx, _node, args) => {
|
||||||
|
const content = text(args, ["text", "value"]);
|
||||||
|
await powershell(`Set-Clipboard -Value ${psString(content)}`);
|
||||||
|
return content;
|
||||||
|
};
|
||||||
|
|
||||||
|
const notice: CapabilityHandler = async (_ctx, _node, args) => {
|
||||||
|
const title = text(args, ["title"], "提醒");
|
||||||
|
const content = text(args, ["text", "message"]);
|
||||||
|
// MessageBox 阻塞等用户点确定,同样不能吃 powershell() 默认 30s 超时;且 MB_TOPMOST 保证弹窗不被主窗口盖住
|
||||||
|
const waitSeconds = number(args, ["wait_time", "waitTime", "timeout"], 600);
|
||||||
|
const timeoutMs = Math.max(10000, Math.round(waitSeconds * 1000));
|
||||||
|
// 置顶方案: MB_TOPMOST/262144 是 Win32 老接口标志, .NET MessageBoxOptions 枚举里没有(只有 RightAlign/RtlReading/ServiceNotification 等),
|
||||||
|
// 传 262144 会被 PS 枚举转换拒绝。正确做法是传一个 TopMost 的隐形属主窗体;
|
||||||
|
// 属主窗体万一失败则 catch 降级为普通 Show, 保证弹窗一定出现不阻塞流程。
|
||||||
|
await powershell(`Add-Type -AssemblyName System.Windows.Forms; $f = New-Object System.Windows.Forms.Form; $f.TopMost = $true; try { [System.Windows.Forms.MessageBox]::Show($f,${psString(content)},${psString(title)},'OK','Information') | Out-Null } catch { [System.Windows.Forms.MessageBox]::Show(${psString(content)},${psString(title)},'OK','Information') | Out-Null }; $f.Dispose()`, timeoutMs);
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
|
||||||
|
const executeCommand: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
|
const command = text(args, ["action", "command"]);
|
||||||
|
if (!command) throw new RuntimeError("validation_error", "命令行为空", { nodeId: node.id, componentId: node.componentId });
|
||||||
|
return powershell(command, Math.max(1000, number(args, ["timeoutMs"], 120000)));
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
register(["Keyboard_Send_V1", "Keyboard_Send", "Keyboard_PressNew", "Keyboard_Press"], keyboardSend);
|
||||||
|
register("Keyboard_EnableManualOperation", enableManualOperation);
|
||||||
|
register("Keyboard_DisableManualOperation", disableManualOperation);
|
||||||
|
register(["Mouse_Click", "Mouse_ClickNew"], mouseClick);
|
||||||
|
register(["Mouse_MoveTo", "Mouse_Move"], mouseMove);
|
||||||
|
register(["Mouse_DragTo", "Mouse_Drag"], mouseDrag);
|
||||||
|
register(["Mouse_RollTo", "Mouse_Scroll"], mouseRoll);
|
||||||
|
register(["Window_SetActive", "Window_Activate"], setActiveWindow);
|
||||||
|
register(["Window_GetActive", "Window_GetActiveWindow"], getActiveWindow);
|
||||||
|
register(["Window_WindowExists", "Window_Exists"], windowExists);
|
||||||
|
register(["Window_CloseWindow", "Window_Close"], closeWindow);
|
||||||
|
register(["App_Run", "App_Start"], runApp);
|
||||||
|
register("Clipboard_GetText", clipboardGet);
|
||||||
|
register("Clipboard_SetText", clipboardSet);
|
||||||
|
register("Dialog_Notice_V1", async (_ctx, node, args) => {
|
||||||
|
const msg = text(args, ["text", "message"]);
|
||||||
|
const title = text(args, ["title"], "通知");
|
||||||
|
const timeout = Math.max(1, number(args, ["timeout", "time_out"], 10));
|
||||||
|
return powershell("(New-Object -ComObject WScript.Shell).Popup(" + psString(msg) + ", " + timeout + ", " + psString(title) + ", 0) | Out-Null; $true");
|
||||||
|
});
|
||||||
|
|
||||||
|
register("Dialog_Notice_V2", async (ctx, node, args) => {
|
||||||
|
const msg = text(args, ["text", "message"]);
|
||||||
|
const title = text(args, ["title"], "通知");
|
||||||
|
const timeout = Math.max(1, number(args, ["time_out", "timeout"], 10));
|
||||||
|
return powershell("(New-Object -ComObject WScript.Shell).Popup(" + psString(msg) + ", " + timeout + ", " + psString(title) + ", 0) | Out-Null; $true");
|
||||||
|
});
|
||||||
|
|
||||||
|
register("Dialog_CsMsgBox", async (_ctx, _node, args) => {
|
||||||
|
const msg = text(args, ["text", "message"]);
|
||||||
|
const title = text(args, ["title"], "提示");
|
||||||
|
const buttons = Number(args.msg_button ?? 0);
|
||||||
|
const autoClose = Number(args.autoClose ?? 0);
|
||||||
|
const time = Math.max(1, Number(args.time ?? 10));
|
||||||
|
const popup = "(New-Object -ComObject WScript.Shell).Popup(" + psString(msg) + ", " + (autoClose ? time : -1) + ", " + psString(title) + ", " + buttons + ")";
|
||||||
|
const choice = await powershell(popup + " | Out-Null; if (" + (autoClose ? "$true" : "$false") + ") {}");
|
||||||
|
return powershell(popup);
|
||||||
|
});
|
||||||
|
|
||||||
|
register("Dialog_OpenFile", async (_ctx, _node, args) => {
|
||||||
|
const title = text(args, ["title"], "选择文件");
|
||||||
|
const wildcard = text(args, ["wildcard"], "所有文件|*.*");
|
||||||
|
// ShowDialog 阻塞等用户选择,等待上限给到 30 分钟(文件挑选通常耗时更久)
|
||||||
|
return powershell("Add-Type -AssemblyName System.Windows.Forms | Out-Null; $dlg = New-Object System.Windows.Forms.OpenFileDialog; $dlg.Filter = " + psString(wildcard) + "; $dlg.Title = " + psString(title) + "; if ($dlg.ShowDialog() -eq 'OK') { $dlg.FileName } else { '' }", 1800000);
|
||||||
|
});
|
||||||
|
|
||||||
|
register("Dialog_OpenFiles", async (_ctx, _node, args) => {
|
||||||
|
const title = text(args, ["title"], "选择文件");
|
||||||
|
const wildcard = text(args, ["wildcard"], "所有文件|*.*");
|
||||||
|
return powershell("Add-Type -AssemblyName System.Windows.Forms | Out-Null; $dlg = New-Object System.Windows.Forms.OpenFileDialog; $dlg.Filter = " + psString(wildcard) + "; $dlg.Title = " + psString(title) + "; $dlg.Multiselect = $true; if ($dlg.ShowDialog() -eq 'OK') { $dlg.FileNames -join ';' } else { '' }", 1800000);
|
||||||
|
});
|
||||||
|
|
||||||
|
register("Dialog_SaveFile", async (_ctx, _node, args) => {
|
||||||
|
const title = text(args, ["title"], "保存文件");
|
||||||
|
const wildcard = text(args, ["wildcard"], "所有文件|*.*");
|
||||||
|
return powershell("Add-Type -AssemblyName System.Windows.Forms | Out-Null; $dlg = New-Object System.Windows.Forms.SaveFileDialog; $dlg.Filter = " + psString(wildcard) + "; $dlg.Title = " + psString(title) + "; if ($dlg.ShowDialog() -eq 'OK') { $dlg.FileName } else { '' }", 1800000);
|
||||||
|
});
|
||||||
|
|
||||||
|
register(["Dialog_InputBox", "Dialog_CsInputBox"], async (_ctx, _node, args) => {
|
||||||
|
const title = text(args, ["title", "input_title"], "请输入");
|
||||||
|
const description = text(args, ["description", "txt_content"], "");
|
||||||
|
const defaultValue = text(args, ["text"], "");
|
||||||
|
// 交互对话框会一直阻塞等用户输入,必须给 powershell() 传足够长的超时(默认 30s 会把还在等人的对话框进程杀掉,报 Command failed)。
|
||||||
|
// 默认 10 分钟,可用 wait_time(秒)自定义;仍超时则报「用户未及时输入」而非笼统的 Command failed。
|
||||||
|
const waitSeconds = number(args, ["wait_time", "waitTime", "timeout"], 600);
|
||||||
|
const timeoutMs = Math.max(10000, Math.round(waitSeconds * 1000));
|
||||||
|
// VB InputBox 不置顶也不在任务栏显示,容易被主窗口盖住导致"看不见弹窗→干等超时"。
|
||||||
|
// 改用 WinForms 窗口:Topmost 强制置顶 + ShowInTaskbar + StartPosition 屏幕中央,保证用户一定看得见。
|
||||||
|
const script = [
|
||||||
|
"Add-Type -AssemblyName System.Windows.Forms | Out-Null",
|
||||||
|
"Add-Type -AssemblyName System.Drawing | Out-Null",
|
||||||
|
"$form = New-Object System.Windows.Forms.Form",
|
||||||
|
"$form.Text = " + psString(title),
|
||||||
|
"$form.Size = New-Object System.Drawing.Size(460, 240)",
|
||||||
|
"$form.StartPosition = 'CenterScreen'",
|
||||||
|
"$form.TopMost = $true",
|
||||||
|
"$form.ShowInTaskbar = $true",
|
||||||
|
"$form.FormBorderStyle = 'FixedDialog'",
|
||||||
|
"$form.MaximizeBox = $false",
|
||||||
|
"$label = New-Object System.Windows.Forms.Label",
|
||||||
|
"$label.Text = " + psString(description),
|
||||||
|
"$label.Location = New-Object System.Drawing.Point(14, 12)",
|
||||||
|
"$label.Size = New-Object System.Drawing.Size(416, 60)",
|
||||||
|
"$box = New-Object System.Windows.Forms.TextBox",
|
||||||
|
"$box.Text = " + psString(defaultValue),
|
||||||
|
"$box.Location = New-Object System.Drawing.Point(14, 78)",
|
||||||
|
"$box.Size = New-Object System.Drawing.Size(416, 26)",
|
||||||
|
"$ok = New-Object System.Windows.Forms.Button",
|
||||||
|
"$ok.Text = '确定'",
|
||||||
|
"$ok.DialogResult = [System.Windows.Forms.DialogResult]::OK",
|
||||||
|
"$ok.Location = New-Object System.Drawing.Point(256, 150)",
|
||||||
|
"$ok.Size = New-Object System.Drawing.Size(84, 30)",
|
||||||
|
"$form.AcceptButton = $ok",
|
||||||
|
"$cancel = New-Object System.Windows.Forms.Button",
|
||||||
|
"$cancel.Text = '取消'",
|
||||||
|
"$cancel.DialogResult = [System.Windows.Forms.DialogResult]::Cancel",
|
||||||
|
"$cancel.Location = New-Object System.Drawing.Point(346, 150)",
|
||||||
|
"$cancel.Size = New-Object System.Drawing.Size(84, 30)",
|
||||||
|
"$form.Controls.AddRange(@($label, $box, $ok, $cancel))",
|
||||||
|
"$form.Add_Shown({ $form.Activate() })",
|
||||||
|
"$result = $form.ShowDialog()",
|
||||||
|
"if ($result -eq [System.Windows.Forms.DialogResult]::OK) { $box.Text } else { '' }"
|
||||||
|
].join("; ");
|
||||||
|
return powershell(script, timeoutMs).then((value) => ({ value, canceled: value === "" })).catch((error) => {
|
||||||
|
if (error instanceof RuntimeError && /Command failed/.test(error.message) && !error.message.includes("RPA_ERROR")) {
|
||||||
|
throw new RuntimeError("dialog_timeout", `输入对话框等待超时(${Math.round(timeoutMs / 1000)} 秒)用户未完成输入`, { nodeId: _node.id, componentId: _node.componentId, recommendation: "置顶输入框应显示在屏幕最前;若仍超时请调大节点的超时时间或 wait_time 参数" });
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
register("Dialog_Notice", notice);
|
||||||
|
register("Dialog_MsgBox", notice);
|
||||||
|
register("System_ExecuteCommand", executeCommand);
|
||||||
|
|
||||||
288
packages/rpa-core/src/capabilities/excel-com.ts
Normal file
288
packages/rpa-core/src/capabilities/excel-com.ts
Normal file
@ -0,0 +1,288 @@
|
|||||||
|
import fs from "node:fs";
|
||||||
|
import path from "node:path";
|
||||||
|
import ExcelJS from "exceljs";
|
||||||
|
import { register } from "../engine/registry";
|
||||||
|
import { RuntimeError, type CapabilityHandler } from "../engine/types";
|
||||||
|
import { powershell, psString } from "../driver/windows/powershell";
|
||||||
|
import { releaseWorkbook } from "./excel-store";
|
||||||
|
|
||||||
|
function text(args: Record<string, unknown>, keys: string[], fallback = ""): string {
|
||||||
|
for (const key of keys) {
|
||||||
|
const value = args[key];
|
||||||
|
if (value !== undefined && value !== null && value !== "") return String(value);
|
||||||
|
}
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
function bool(value: unknown, fallback = false): boolean {
|
||||||
|
if (value === undefined || value === null || value === "") return fallback;
|
||||||
|
return [true, 1, "1", "true", "True", "TRUE", "yes", "是", "开启"].includes(value as never);
|
||||||
|
}
|
||||||
|
|
||||||
|
function number(value: unknown, fallback = 0): number { const result = Number(value); return Number.isFinite(result) ? result : fallback; }
|
||||||
|
|
||||||
|
function workbookPath(args: Record<string, unknown>, nodeId: string, componentId: string, keys = ["workbook", "objExcel", "objExcelWorkBook", "destWorkbook"]): string {
|
||||||
|
for (const key of keys) {
|
||||||
|
const raw = args[key];
|
||||||
|
const candidate = raw && typeof raw === "object" ? (raw as { path?: unknown }).path : raw;
|
||||||
|
const value = String(candidate ?? "").trim();
|
||||||
|
if (value && value !== "Excel文件" && value !== "objExcelWorkBook") return path.resolve(value);
|
||||||
|
}
|
||||||
|
throw new RuntimeError("validation_error", "Excel文件路径为空", { nodeId, componentId });
|
||||||
|
}
|
||||||
|
|
||||||
|
function sheetExpression(args: Record<string, unknown>): string {
|
||||||
|
const sheet = text(args, ["sheet", "sheetName"]);
|
||||||
|
return sheet ? `$workbook.Worksheets.Item(${psString(sheet)})` : "$workbook.ActiveSheet";
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runExcelCom(filePath: string, body: string, save = false, readonly = false, timeout = 120000): Promise<unknown> {
|
||||||
|
const openMode = readonly ? "$true" : "$false";
|
||||||
|
const saveMode = save ? "$true" : "$false";
|
||||||
|
const script = [
|
||||||
|
"$excel=$null;$workbook=$null;$rpaOutput=$null",
|
||||||
|
"try {",
|
||||||
|
" $excel=New-Object -ComObject Excel.Application;$excel.Visible=$false;$excel.DisplayAlerts=$false",
|
||||||
|
" $workbook=$excel.Workbooks.Open(" + psString(filePath) + ",0," + openMode + ")",
|
||||||
|
" " + body,
|
||||||
|
" if (" + saveMode + ") { $workbook.Save() }",
|
||||||
|
" if ($null -ne $rpaOutput) { [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes(($rpaOutput | ConvertTo-Json -Compress -Depth 6))) | Write-Output }",
|
||||||
|
"} finally {",
|
||||||
|
" if($null -ne $workbook){try{$workbook.Close($false)}catch{};try{[void][Runtime.InteropServices.Marshal]::ReleaseComObject($workbook)}catch{}}",
|
||||||
|
" if($null -ne $excel){try{$excel.Quit()}catch{};try{[void][Runtime.InteropServices.Marshal]::ReleaseComObject($excel)}catch{}}",
|
||||||
|
" [GC]::Collect();[GC]::WaitForPendingFinalizers()",
|
||||||
|
"}"
|
||||||
|
].join("\n");
|
||||||
|
const output = (await powershell(script, timeout)).trim();
|
||||||
|
if (save) releaseWorkbook(filePath);
|
||||||
|
if (!output) return null;
|
||||||
|
try { return JSON.parse(Buffer.from(output, "base64").toString("utf8")); }
|
||||||
|
catch { throw new RuntimeError("excel_com_failed", "Excel COM 输出异常", { detail: output.slice(0, 300) }); }
|
||||||
|
}
|
||||||
|
|
||||||
|
function list(value: unknown): string[] {
|
||||||
|
if (Array.isArray(value)) return value.map((item) => String(item)).filter(Boolean);
|
||||||
|
const raw = String(value ?? "").trim();
|
||||||
|
if (!raw) return [];
|
||||||
|
try { const parsed = JSON.parse(raw); return Array.isArray(parsed) ? parsed.map(String) : [String(parsed)]; }
|
||||||
|
catch { return raw.split(/[,,;;\s]+/).filter(Boolean); }
|
||||||
|
}
|
||||||
|
const group: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
|
const filePath = workbookPath(args, node.id, node.componentId, ["workbook"]);
|
||||||
|
const ungroup = String(args.group ?? "0") === "1";
|
||||||
|
const columns = String(args.groupType ?? "0") === "1";
|
||||||
|
const start = columns ? text(args, ["startColumn"], "A") : String(number(args.startRow, 1));
|
||||||
|
const end = columns ? text(args, ["endColumn"], text(args, ["startColumn"], "A")) : String(number(args.endRow, number(args.startRow, 1)));
|
||||||
|
const method = ungroup ? "Ungroup" : "Group";
|
||||||
|
const body = `$sheet=${sheetExpression(args)};$start=${psString(start)};$end=${psString(end)};
|
||||||
|
$count=if(${columns ? "$true" : "$false"}){$sheet.UsedRange.Columns.Count}else{$sheet.UsedRange.Rows.Count};
|
||||||
|
foreach($name in @("start","end")){ $v=[int](Get-Variable $name -ValueOnly); if($v -lt 0){ Set-Variable $name ($count+$v+1) } };
|
||||||
|
$range=$sheet.Range("$($start):$($end)");$range.${method}|Out-Null`;
|
||||||
|
await runExcelCom(filePath, body, bool(args.save));
|
||||||
|
return { grouped: !ungroup, type: columns ? "columns" : "rows" };
|
||||||
|
};
|
||||||
|
|
||||||
|
const deletePicture: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
|
const filePath = workbookPath(args, node.id, node.componentId, ["workbook", "objExcel"]);
|
||||||
|
const name = text(args, ["name", "pictureName"]);
|
||||||
|
if (!name) throw new RuntimeError("validation_error", "图片名称为空", { nodeId: node.id, componentId: node.componentId });
|
||||||
|
const body = `$sheet=${sheetExpression(args)};$sheet.Shapes.Item(${psString(name)}).Delete()`;
|
||||||
|
await runExcelCom(filePath, body, bool(args.save));
|
||||||
|
return { deleted: name };
|
||||||
|
};
|
||||||
|
|
||||||
|
const insertPicture: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
|
const filePath = workbookPath(args, node.id, node.componentId, ["workbook", "objExcel"]);
|
||||||
|
const image = path.resolve(text(args, ["path", "pictureName"]));
|
||||||
|
if (!fs.existsSync(image)) throw new RuntimeError("validation_error", "图片文件不存在", { nodeId: node.id, componentId: node.componentId, detail: image });
|
||||||
|
const row = number(args.row, 1); const column = number(args.column, 1);
|
||||||
|
const left = number(args.Left ?? args.left, 20); const top = number(args.Top ?? args.top, 20);
|
||||||
|
const width = number(args.Width ?? args.width, -1); const height = number(args.Height ?? args.height, -1);
|
||||||
|
const scale = width > 0 && height > 0 ? "$true" : "$false";
|
||||||
|
const body = `$sheet=${sheetExpression(args)};$cell=$sheet.Cells(${row},${column});$left=${left}+if($left -gt 0){$cell.Left}else{0};$top=${top}+if($top -gt 0){$cell.Top}else{0};[void]$sheet.Shapes.AddPicture(${psString(image)},$false,${scale},$left,$top,${width},${height})`;
|
||||||
|
await runExcelCom(filePath, body, bool(args.save));
|
||||||
|
return { inserted: image };
|
||||||
|
};
|
||||||
|
|
||||||
|
const encryptExcel: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
|
const filePath = workbookPath(args, node.id, node.componentId, ["workbook"]);
|
||||||
|
const password = String(args.password1 ?? args.password ?? "");
|
||||||
|
const confirm = String(args.password2 ?? password);
|
||||||
|
if (!password) throw new RuntimeError("validation_error", "Excel密码为空", { nodeId: node.id, componentId: node.componentId });
|
||||||
|
if (password !== confirm) throw new RuntimeError("validation_error", "两次输入的Excel密码不一致", { nodeId: node.id, componentId: node.componentId });
|
||||||
|
const temporary = `${filePath}.${Date.now()}.enc.xlsx`;
|
||||||
|
try {
|
||||||
|
await runExcelCom(filePath, `$workbook.SaveAs(${psString(temporary)},51,${psString(password)},${psString(confirm)})`, false);
|
||||||
|
fs.rmSync(temporary, { force: true });
|
||||||
|
// COM SaveAs cannot replace the workbook while it is open, so save beside it and rename after Close.
|
||||||
|
} finally {
|
||||||
|
if (fs.existsSync(temporary)) {
|
||||||
|
fs.rmSync(filePath, { force: true });
|
||||||
|
fs.renameSync(temporary, filePath);
|
||||||
|
}
|
||||||
|
releaseWorkbook(filePath);
|
||||||
|
}
|
||||||
|
return filePath;
|
||||||
|
};
|
||||||
|
|
||||||
|
const moveSheet: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
|
const filePath = workbookPath(args, node.id, node.componentId, ["workbook"]);
|
||||||
|
const mode = text(args, ["mode"], "按序号"); const after = text(args, ["option"], "之前").includes("之后");
|
||||||
|
const targetName = text(args, ["name", "index"]); const position = text(args, ["position", "index"], "第一位");
|
||||||
|
const index = number(args.index, 1);
|
||||||
|
const afterMode = after ? "$true" : "$false";
|
||||||
|
const body = `$sheet=${sheetExpression(args)};$count=$workbook.Worksheets.Count;$targetIndex=0;
|
||||||
|
if(${psString(mode)} -eq "按名称"){ $targetName=${psString(targetName)}; for($i=1;$i -le $count;$i++){ if($workbook.Worksheets.Item($i).Name -eq $targetName){$targetIndex=$i;break} }; if($targetIndex -eq 0){throw "target_sheet_not_found"} }
|
||||||
|
elseif(${psString(mode)} -eq "按位置"){ $p=${psString(position)}; if($p -eq "第一位"){$targetIndex=1}elseif($p -eq "最后一位"){$targetIndex=$count}else{ $delta=-1; if($p -eq "向后移一位"){$delta=1}; $targetIndex=$sheet.Index+$delta } }
|
||||||
|
else { $targetIndex=${index}; if($targetIndex -lt 0){$targetIndex=$count+$targetIndex+1} };
|
||||||
|
$targetIndex=[Math]::Min([Math]::Max(1,$targetIndex),$count);$target=$workbook.Worksheets.Item($targetIndex);
|
||||||
|
if(${afterMode}){ if($target.Index -ne $sheet.Index){ $sheet.Move([Type]::Missing,$target)|Out-Null } }
|
||||||
|
else { if($target.Index -ne $sheet.Index){ $sheet.Move($target)|Out-Null } }`;
|
||||||
|
await runExcelCom(filePath, body, bool(args.save));
|
||||||
|
return { moved: true };
|
||||||
|
};
|
||||||
|
const mergeSheet: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
|
const filePath = workbookPath(args, node.id, node.componentId, ["workbook"]);
|
||||||
|
const destPath = workbookPath(args, node.id, node.componentId, ["destWorkbook", "workbook"]);
|
||||||
|
const horizontal = String(args.option ?? "0") === "1";
|
||||||
|
const horizontalMode = horizontal ? "$true" : "$false";
|
||||||
|
const saveDest = bool(args.save) ? "$true" : "$false";
|
||||||
|
const body = `$source=${sheetExpression(args)};$dest=$excel.Workbooks.Open(${psString(destPath)});$target=$dest.Worksheets.Item(${psString(text(args, ["destSheet"], "Sheet1"))});
|
||||||
|
if(${horizontalMode}){$offset=$target.UsedRange.Columns.Count+1}else{$offset=$target.UsedRange.Rows.Count+1};
|
||||||
|
for($r=1;$r -le $source.UsedRange.Rows.Count;$r++){for($c=1;$c -le $source.UsedRange.Columns.Count;$c++){ if(${horizontalMode}){$target.Cells($r,$offset+$c-1).Value2=$source.Cells($r,$c).Value2}else{$target.Cells($offset+$r-1,$c).Value2=$source.Cells($r,$c).Value2}}};
|
||||||
|
if(${saveDest}){$dest.Save()}`;
|
||||||
|
await runExcelCom(filePath, body, false);
|
||||||
|
return { merged: true, target: destPath };
|
||||||
|
};
|
||||||
|
|
||||||
|
const refreshAll: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
|
const filePath = workbookPath(args, node.id, node.componentId, ["workbook"]);
|
||||||
|
await runExcelCom(filePath, "$workbook.RefreshAll()", bool(args.save));
|
||||||
|
return { refreshed: true };
|
||||||
|
};
|
||||||
|
|
||||||
|
const refreshPivot: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
|
const filePath = workbookPath(args, node.id, node.componentId, ["workbook"]);
|
||||||
|
const name = text(args, ["pivotTable"]); const sheetExpr = sheetExpression(args);
|
||||||
|
const body = name ? `${sheetExpr}.PivotTables(${psString(name)}).RefreshTable()` : `${sheetExpr}.PivotTables() | ForEach-Object { $_.RefreshTable() }`;
|
||||||
|
await runExcelCom(filePath, body, bool(args.save));
|
||||||
|
return { refreshed: true, name: name || "all" };
|
||||||
|
};
|
||||||
|
|
||||||
|
const filterPivot: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
|
const filePath = workbookPath(args, node.id, node.componentId, ["workbook"]);
|
||||||
|
const field = text(args, ["field"]); const values = list(args.fieldValue);
|
||||||
|
if (!field) throw new RuntimeError("validation_error", "筛选字段为空", { nodeId: node.id, componentId: node.componentId });
|
||||||
|
const encoded = Buffer.from(JSON.stringify(values), "utf16le").toString("base64");
|
||||||
|
const body = `$pt=${sheetExpression(args)}.PivotTables(${psString(text(args, ["pivotTable"]))});$pf=$pt.PivotFields(${psString(field)});
|
||||||
|
try{$pf.ClearAllFilters()}catch{};
|
||||||
|
$wanted=[Text.Encoding]::Unicode.GetString([Convert]::FromBase64String("${encoded}"))|ConvertFrom-Json;
|
||||||
|
foreach($item in $pf.PivotItems()){ $selected=if($wanted.Count -eq 0){$true}else{$wanted -contains $item.Name}; $item.Visible=$selected }`;
|
||||||
|
await runExcelCom(filePath, body, bool(args.save));
|
||||||
|
return { field, values };
|
||||||
|
};
|
||||||
|
|
||||||
|
const createPivot: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
|
const filePath = workbookPath(args, node.id, node.componentId, ["workbook"]);
|
||||||
|
const chart = node.componentId.includes("PivotView");
|
||||||
|
let rule: Record<string, unknown> = {};
|
||||||
|
try { rule = JSON.parse(String(args.rule || "{}")) as Record<string, unknown>; } catch { /* sourceData can still be supplied separately. */ }
|
||||||
|
const source = text({ ...rule, ...args }, ["SourceData", "sourceData", "source"]);
|
||||||
|
if (!source) throw new RuntimeError("validation_error", "透视表数据源为空", { nodeId: node.id, componentId: node.componentId });
|
||||||
|
const name = text(rule, ["Name", "name"], chart ? "RPAPivotChart" : "RPAPivotTable");
|
||||||
|
const body = `$sheet=${sheetExpression(args)};$cache=$excel.ActiveWorkbook.PivotCaches().Create(1,$sheet.Range(${psString(source)}));
|
||||||
|
$dest=$sheet.Cells(1,20);$pt=$cache.CreatePivotTable($dest,${psString(name)});
|
||||||
|
if(${chart ? "$true" : "$false"}){[void]$sheet.Shapes.AddChart(-4109,20,20,420,260)}`;
|
||||||
|
await runExcelCom(filePath, body, bool(args.save));
|
||||||
|
return { name, chart };
|
||||||
|
};
|
||||||
|
const runMacro = (scriptKey: "script" | "code" | "macroName"): CapabilityHandler => async (_ctx, node, args) => {
|
||||||
|
const filePath = workbookPath(args, node.id, node.componentId, ["workbook", "objExcel"]);
|
||||||
|
const macro = text(args, [scriptKey, "script", "code", "macroName", "name"]);
|
||||||
|
if (!macro) throw new RuntimeError("validation_error", "宏名称为空", { nodeId: node.id, componentId: node.componentId });
|
||||||
|
const parameters = list(args.args ?? args.macroArgs).map((item) => psString(item)).join(",");
|
||||||
|
const body = `$result=$excel.Run(${psString(macro)}${parameters ? "," + parameters : ""});$rpaOutput=@{result=$result}`;
|
||||||
|
return await runExcelCom(filePath, body, bool(args.save), false, 180000);
|
||||||
|
};
|
||||||
|
|
||||||
|
const interop: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
|
const filePath = workbookPath(args, node.id, node.componentId, ["workbook", "objExcel"]);
|
||||||
|
const property = text(args, ["property"]);
|
||||||
|
if (!property) throw new RuntimeError("validation_error", "属性或方法名为空", { nodeId: node.id, componentId: node.componentId });
|
||||||
|
const objectKind = number(args.obj ?? args.object, 0);
|
||||||
|
const range = text(args, ["range"], "A1");
|
||||||
|
const target = objectKind === 0 ? "$excel" : objectKind === 1 ? "$workbook" : objectKind === 2 ? sheetExpression(args) : `${sheetExpression(args)}.Range(${psString(range)})`;
|
||||||
|
const value = args.value;
|
||||||
|
const empty = value === undefined || value === null || value === "";
|
||||||
|
const body = empty
|
||||||
|
? `$target=${target};$rpaOutput=@{value=$target.GetType().InvokeMember(${psString(property)},[Reflection.BindingFlags]::GetProperty,$null,$target,@())}`
|
||||||
|
: `$target=${target};[void]$target.GetType().InvokeMember(${psString(property)},[Reflection.BindingFlags]::SetProperty,$null,$target,@(${psString(String(value))}));$rpaOutput=@{set=$true}`;
|
||||||
|
return await runExcelCom(filePath, body, bool(args.save));
|
||||||
|
};
|
||||||
|
|
||||||
|
const exportPdf: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
|
const filePath = workbookPath(args, node.id, node.componentId, ["workbook"]);
|
||||||
|
const folder = text(args, ["path"]);
|
||||||
|
const name = text(args, ["name"], path.basename(filePath, path.extname(filePath)) + ".pdf");
|
||||||
|
const output = path.join(path.resolve(folder), /\.(pdf)$/i.test(name) ? name : `${name}.pdf`);
|
||||||
|
fs.mkdirSync(path.dirname(output), { recursive: true });
|
||||||
|
if (!bool(args.cover, true) && fs.existsSync(output)) throw new RuntimeError("validation_error", "PDF 文件已存在", { detail: output });
|
||||||
|
const activateOnly = text(args, ["option"], "全部") === "激活页";
|
||||||
|
const body = `if(${activateOnly ? "$true" : "$false"}){$workbook.ActiveSheet.ExportAsFixedFormat(0,${psString(output)})}else{$workbook.ExportAsFixedFormat(0,${psString(output)})}`;
|
||||||
|
await runExcelCom(filePath, body, false);
|
||||||
|
return output;
|
||||||
|
};
|
||||||
|
|
||||||
|
const exportRangePicture: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
|
const filePath = workbookPath(args, node.id, node.componentId, ["workbook"]);
|
||||||
|
const folder = text(args, ["path"]); const name = text(args, ["name"], `excel-range-${Date.now()}`);
|
||||||
|
const output = path.join(path.resolve(folder), /\.(png|jpg|jpeg)$/i.test(name) ? name : `${name}.png`);
|
||||||
|
fs.mkdirSync(path.dirname(output), { recursive: true });
|
||||||
|
const body = `$range=${sheetExpression(args)}.Range(${psString(text(args, ["range"], "A1"))});$range.CopyPicture(1,2)|Out-Null;$shape=${sheetExpression(args)}.ChartObjects().Add(0,0,$range.Width,$range.Height);$chart=$shape.Chart;$chart.Paste();[void]$chart.Export(${psString(output)},"PNG");$shape.Delete()`;
|
||||||
|
await runExcelCom(filePath, body, false);
|
||||||
|
return output;
|
||||||
|
};
|
||||||
|
|
||||||
|
const setSlicer: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
|
const filePath = workbookPath(args, node.id, node.componentId, ["workbook"]);
|
||||||
|
const name = text(args, ["slicerName"]); const values = list(args.slicerOptions);
|
||||||
|
if (!name) throw new RuntimeError("validation_error", "切片器名称为空", { nodeId: node.id, componentId: node.componentId });
|
||||||
|
const encoded = Buffer.from(JSON.stringify(values), "utf16le").toString("base64");
|
||||||
|
const body = `$cache=$workbook.SlicerCaches.Item(${psString(name)});
|
||||||
|
$wanted=[Text.Encoding]::Unicode.GetString([Convert]::FromBase64String("${encoded}"))|ConvertFrom-Json;
|
||||||
|
foreach($item in $cache.SlicerItems()){ $item.Selected=if($wanted.Count -eq 0){$true}else{$wanted -contains $item.Name} }`;
|
||||||
|
await runExcelCom(filePath, body, bool(args.save));
|
||||||
|
return { slicer: name, values };
|
||||||
|
};
|
||||||
|
|
||||||
|
const excelLoop: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
|
const filePath = workbookPath(args, node.id, node.componentId, ["objExcel", "workbook"]);
|
||||||
|
const workbook = new ExcelJS.Workbook();
|
||||||
|
await workbook.xlsx.readFile(filePath);
|
||||||
|
const rows: unknown[][] = [];
|
||||||
|
for (const sheet of workbook.worksheets) {
|
||||||
|
for (let row = 1; row <= sheet.rowCount; row += 1) {
|
||||||
|
const cells: unknown[] = [];
|
||||||
|
for (let column = 1; column <= sheet.columnCount; column += 1) cells.push(sheet.getCell(row, column).value ?? "");
|
||||||
|
if (cells.some((cell) => cell !== "")) rows.push(cells);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return rows;
|
||||||
|
};
|
||||||
|
|
||||||
|
register("Excel_Cs_Group", group);
|
||||||
|
register("Excel_Cs_Encrypt", encryptExcel);
|
||||||
|
register(["Excel_Cs_DeletePicture", "Excel_Cs_DeletePictureV2", "Excel_DeleteImage"], deletePicture);
|
||||||
|
register(["Excel_Cs_InsertPicture", "Excel_Cs_InsertPictureV2", "Excel_InsertImage"], insertPicture);
|
||||||
|
register("Excel_Cs_Move", moveSheet);
|
||||||
|
register("Excel_Cs_Merge", mergeSheet);
|
||||||
|
register(["Excel_Cs_RefreshAll", "Excel_Cs_RefreshPivotTable"], async (ctx, node, args, component) => node.componentId === "Excel_Cs_RefreshPivotTable" ? refreshPivot(ctx, node, args, component) : refreshAll(ctx, node, args, component));
|
||||||
|
register("Excel_Cs_FilterPivotTable", filterPivot);
|
||||||
|
register(["Excel_Cs_InsertPivotTable", "Excel_Cs_InsertPivotView"], createPivot);
|
||||||
|
register(["Excel_Cs_RunMacro", "Excel_Cs_RunMacroV2", "Excel_ExecuteMacro"], runMacro("script"));
|
||||||
|
register("Excel_Cs_Interop", interop);
|
||||||
|
register("Excel_Cs_Export", exportPdf);
|
||||||
|
register("Excel_Cs_ExcelAddin", runMacro("code"));
|
||||||
|
register("Excel_Cs_SetSlicer", setSlicer);
|
||||||
|
register("Excel_Loop", excelLoop);
|
||||||
|
register("Excel_ExportRangeAsPicture", exportRangePicture);
|
||||||
@ -1,10 +1,11 @@
|
|||||||
import ExcelJS from "exceljs";
|
import ExcelJS from "exceljs";
|
||||||
import path from "node:path";
|
import path from "node:path";
|
||||||
|
import { defaultEngine } from "../engine/engine";
|
||||||
|
|
||||||
// Excel 工作簿共享缓存:同一文件路径在整条流程中复用同一个 ExcelJS 实例。
|
// Excel 工作簿共享缓存:同一文件路径在整条流程中复用同一个 ExcelJS 实例。
|
||||||
// 此前 office.ts / office-file-extended.ts / excel-common.ts / extended.ts 各自持有
|
// 此前 office.ts / office-file-extended.ts / excel-common.ts / extended.ts 各自持有
|
||||||
// 工作簿实例并整簿写回磁盘,后保存者会把先保存者的修改整体覆盖掉。
|
// 工作簿实例并整簿写回磁盘,后保存者会把先保存者的修改整体覆盖掉。
|
||||||
const cache = new Map<string, ExcelJS.Workbook>();
|
// 缓存本体挂在引擎实例上(任务 1.3),此处委托 defaultEngine。
|
||||||
|
|
||||||
function keyOf(filePath: string): string {
|
function keyOf(filePath: string): string {
|
||||||
return path.resolve(filePath);
|
return path.resolve(filePath);
|
||||||
@ -13,22 +14,22 @@ function keyOf(filePath: string): string {
|
|||||||
/** 按路径取工作簿:命中缓存直接复用,未命中从磁盘读取后登记。 */
|
/** 按路径取工作簿:命中缓存直接复用,未命中从磁盘读取后登记。 */
|
||||||
export async function acquireWorkbook(filePath: string): Promise<ExcelJS.Workbook> {
|
export async function acquireWorkbook(filePath: string): Promise<ExcelJS.Workbook> {
|
||||||
const key = keyOf(filePath);
|
const key = keyOf(filePath);
|
||||||
let workbook = cache.get(key);
|
let workbook = defaultEngine.excelCache.get(key);
|
||||||
if (!workbook) {
|
if (!workbook) {
|
||||||
workbook = new ExcelJS.Workbook();
|
workbook = new ExcelJS.Workbook();
|
||||||
await workbook.xlsx.readFile(key);
|
await workbook.xlsx.readFile(key);
|
||||||
cache.set(key, workbook);
|
defaultEngine.excelCache.set(key, workbook);
|
||||||
}
|
}
|
||||||
return workbook;
|
return workbook;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 把一个新建/另建的工作簿实例登记为该路径的共享实例。 */
|
/** 把一个新建/另建的工作簿实例登记为该路径的共享实例。 */
|
||||||
export function publishWorkbook(filePath: string, workbook: ExcelJS.Workbook): ExcelJS.Workbook {
|
export function publishWorkbook(filePath: string, workbook: ExcelJS.Workbook): ExcelJS.Workbook {
|
||||||
cache.set(keyOf(filePath), workbook);
|
defaultEngine.excelCache.set(keyOf(filePath), workbook);
|
||||||
return workbook;
|
return workbook;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 释放路径对应的共享实例(关闭工作簿时调用)。 */
|
/** 释放路径对应的共享实例(关闭工作簿时调用)。 */
|
||||||
export function releaseWorkbook(filePath: string): void {
|
export function releaseWorkbook(filePath: string): void {
|
||||||
cache.delete(keyOf(filePath));
|
defaultEngine.excelCache.delete(keyOf(filePath));
|
||||||
}
|
}
|
||||||
@ -92,6 +92,16 @@ register(["File_File_Move", "File_Move"], async (_ctx, node, args) => {
|
|||||||
const newName = truthy(args.rename) ? pick(args, ["name"]) : "";
|
const newName = truthy(args.rename) ? pick(args, ["name"]) : "";
|
||||||
target = requirePath(newName ? path.join(destDir, newName) : destDir, "目的地路径", node.id, node.componentId);
|
target = requirePath(newName ? path.join(destDir, newName) : destDir, "目的地路径", node.id, node.componentId);
|
||||||
}
|
}
|
||||||
|
let targetStat: import("node:fs").Stats | null = null;
|
||||||
|
try {
|
||||||
|
targetStat = await fs.stat(target);
|
||||||
|
} catch {
|
||||||
|
// 目标不存在,按完整文件路径处理
|
||||||
|
}
|
||||||
|
if (targetStat?.isDirectory()) {
|
||||||
|
// 目标是已存在的目录:语义为“移入该目录”,自动拼上源文件名
|
||||||
|
target = path.join(target, path.basename(source));
|
||||||
|
}
|
||||||
await fs.mkdir(path.dirname(target), { recursive: true });
|
await fs.mkdir(path.dirname(target), { recursive: true });
|
||||||
try {
|
try {
|
||||||
await fs.rename(source, target);
|
await fs.rename(source, target);
|
||||||
@ -196,10 +206,45 @@ register("File_WaitFile", async (ctx, node, args) => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
function directoryOutputPath(outputPath: string, sourcePath: string): string {
|
||||||
|
const output = path.resolve(outputPath);
|
||||||
|
if (/[\\/]$/.test(outputPath) || path.extname(output).toLowerCase() !== ".zip") {
|
||||||
|
return path.join(output, path.basename(sourcePath) + ".zip");
|
||||||
|
}
|
||||||
|
return output;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function uniqueZipPath(targetPath: string): Promise<string> {
|
||||||
|
try {
|
||||||
|
await fs.access(targetPath);
|
||||||
|
} catch {
|
||||||
|
return targetPath;
|
||||||
|
}
|
||||||
|
const dir = path.dirname(targetPath);
|
||||||
|
const ext = path.extname(targetPath);
|
||||||
|
const base = path.basename(targetPath, ext);
|
||||||
|
for (let index = 1; ; index++) {
|
||||||
|
const candidate = path.join(dir, `${base} (${index})${ext}`);
|
||||||
|
try {
|
||||||
|
await fs.access(candidate);
|
||||||
|
} catch {
|
||||||
|
return candidate;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
register("File_CompressFile", async (_ctx, node, args) => {
|
register("File_CompressFile", async (_ctx, node, args) => {
|
||||||
// catalog 字段为 file_path(文件模式)/folder_path(文件夹模式)+ out_path,兼容旧键
|
// catalog 字段为 file_path(文件模式)/folder_path(文件夹模式)+ out_path,兼容旧键
|
||||||
const source = requirePath(pick(args, ["file_path", "pathSrc", "path", "folder_path"]), "源路径", node.id, node.componentId);
|
const contentType = String(args.file_type ?? "openFile");
|
||||||
const target = requirePath(pick(args, ["pathDest", "zip_path", "out_path"]), "压缩包路径", node.id, node.componentId);
|
const sourceKeys = contentType === "openDirectory"
|
||||||
|
? ["folder_path", "pathSrc", "path"]
|
||||||
|
: ["file_path", "pathSrc", "path"];
|
||||||
|
const source = requirePath(pick(args, sourceKeys), contentType === "openDirectory" ? "待压缩文件夹路径" : "待压缩文件路径", node.id, node.componentId);
|
||||||
|
const saveToSourceDir = String(args.direct_type ?? "1") !== "0";
|
||||||
|
const requestedTarget = saveToSourceDir
|
||||||
|
? path.join(path.dirname(source), path.basename(source) + ".zip")
|
||||||
|
: directoryOutputPath(requirePath(pick(args, ["pathDest", "zip_path", "out_path"]), "压缩包输出位置", node.id, node.componentId), source);
|
||||||
|
const target = await uniqueZipPath(requestedTarget);
|
||||||
const zip = new AdmZip();
|
const zip = new AdmZip();
|
||||||
const stat = await fs.stat(source);
|
const stat = await fs.stat(source);
|
||||||
if (stat.isDirectory()) zip.addLocalFolder(source);
|
if (stat.isDirectory()) zip.addLocalFolder(source);
|
||||||
@ -211,7 +256,14 @@ register("File_CompressFile", async (_ctx, node, args) => {
|
|||||||
|
|
||||||
register("File_DecompressFile", async (_ctx, node, args) => {
|
register("File_DecompressFile", async (_ctx, node, args) => {
|
||||||
const source = requirePath(pick(args, ["folder_path", "path", "zip_path"]), "压缩包路径", node.id, node.componentId);
|
const source = requirePath(pick(args, ["folder_path", "path", "zip_path"]), "压缩包路径", node.id, node.componentId);
|
||||||
const target = requirePath(pick(args, ["out_path", "pathDest", "target"]), "解压目录", node.id, node.componentId);
|
// adm-zip 对不存在的文件抛 "ADM-ZIP: Invalid filename",极具误导性;先显式检查文件存在性
|
||||||
|
await fs.access(source).catch(() => {
|
||||||
|
throw new RuntimeError("file_not_found", `压缩包不存在:${source}`, { nodeId: node.id, componentId: node.componentId, recommendation: "确认压缩包实际产物路径(压缩组件默认在原文件名后追加 .zip,如 a.txt 会压成 a.txt.zip),或用压缩节点的输出变量传入" });
|
||||||
|
});
|
||||||
|
const saveToSourceDir = String(args.direct_type ?? "1") !== "0";
|
||||||
|
const target = saveToSourceDir
|
||||||
|
? path.dirname(source)
|
||||||
|
: requirePath(pick(args, ["out_path", "pathDest", "target"]), "解压目标目录", node.id, node.componentId);
|
||||||
await fs.mkdir(target, { recursive: true });
|
await fs.mkdir(target, { recursive: true });
|
||||||
new AdmZip(source).extractAllTo(target, true);
|
new AdmZip(source).extractAllTo(target, true);
|
||||||
return target;
|
return target;
|
||||||
@ -3,11 +3,10 @@ import path from "node:path";
|
|||||||
import fs from "node:fs";
|
import fs from "node:fs";
|
||||||
import { register } from "../engine/registry";
|
import { register } from "../engine/registry";
|
||||||
import { RuntimeError } from "../engine/types";
|
import { RuntimeError } from "../engine/types";
|
||||||
|
import { defaultEngine } from "../engine/engine";
|
||||||
const flags = globalThis as { __rpaFtpClient?: Client };
|
|
||||||
|
|
||||||
async function requireClient(nodeId: string): Promise<Client> {
|
async function requireClient(nodeId: string): Promise<Client> {
|
||||||
if (flags.__rpaFtpClient) return flags.__rpaFtpClient;
|
if (defaultEngine.ftpClient) return defaultEngine.ftpClient;
|
||||||
throw new RuntimeError("validation_error", "尚未连接 FTP,请先使用 FTP_Connect", { nodeId, componentId: nodeId, recommendation: "先运行 FTP_Connect 建立连接" });
|
throw new RuntimeError("validation_error", "尚未连接 FTP,请先使用 FTP_Connect", { nodeId, componentId: nodeId, recommendation: "先运行 FTP_Connect 建立连接" });
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -27,14 +26,14 @@ register("FTP_Connect", async (ctx, node, args) => {
|
|||||||
const secure = String(args.ftp_type ?? "").toLowerCase().includes("ftps");
|
const secure = String(args.ftp_type ?? "").toLowerCase().includes("ftps");
|
||||||
const client = new Client(10_000);
|
const client = new Client(10_000);
|
||||||
await client.access({ host, port, secure, user, password });
|
await client.access({ host, port, secure, user, password });
|
||||||
flags.__rpaFtpClient = client;
|
defaultEngine.ftpClient = client;
|
||||||
ctx.log("已连接 FTP:" + host + ":" + port, "info", node.id);
|
ctx.log("已连接 FTP:" + host + ":" + port, "info", node.id);
|
||||||
return { connected: true, host, port };
|
return { connected: true, host, port };
|
||||||
});
|
});
|
||||||
|
|
||||||
register("FTP_Disconnect", async (ctx, node) => {
|
register("FTP_Disconnect", async (ctx, node) => {
|
||||||
flags.__rpaFtpClient?.close();
|
defaultEngine.ftpClient?.close();
|
||||||
flags.__rpaFtpClient = undefined;
|
defaultEngine.ftpClient = undefined;
|
||||||
ctx.log("已断开 FTP 连接", "info", node.id);
|
ctx.log("已断开 FTP 连接", "info", node.id);
|
||||||
return { disconnected: true };
|
return { disconnected: true };
|
||||||
});
|
});
|
||||||
@ -1,4 +1,4 @@
|
|||||||
import { BrowserWindow } from "electron";
|
import { getHostBridge, type HostDialogWindow } from "../engine/host-bridge";
|
||||||
import { register } from "../engine/registry";
|
import { register } from "../engine/registry";
|
||||||
import { RuntimeError } from "../engine/types";
|
import { RuntimeError } from "../engine/types";
|
||||||
|
|
||||||
@ -28,6 +28,12 @@ function escapeHtml(s: string): string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function showForm(options: { title: string; fields: FormFieldSpec[]; timeoutSec: number }): Promise<Record<string, string>> {
|
async function showForm(options: { title: string; fields: FormFieldSpec[]; timeoutSec: number }): Promise<Record<string, string>> {
|
||||||
|
const bridge = getHostBridge();
|
||||||
|
if (!bridge) {
|
||||||
|
throw new RuntimeError("host_unavailable", "宿主桥未注入,无法打开人机交互窗口", {
|
||||||
|
recommendation: "宿主(Electron 主进程)启动时需调用 setHostBridge() 注入窗口工厂"
|
||||||
|
});
|
||||||
|
}
|
||||||
return new Promise((resolve) => {
|
return new Promise((resolve) => {
|
||||||
const rows = options.fields.map((f, i) => {
|
const rows = options.fields.map((f, i) => {
|
||||||
const key = escapeHtml(fieldKey(f, i));
|
const key = escapeHtml(fieldKey(f, i));
|
||||||
@ -53,11 +59,7 @@ async function showForm(options: { title: string; fields: FormFieldSpec[]; timeo
|
|||||||
return "<label>" + label + '<br/><input type="' + inputType + '" data-key="' + key + '" value="' + value + '" /></label><br/>';
|
return "<label>" + label + '<br/><input type="' + inputType + '" data-key="' + key + '" value="' + value + '" /></label><br/>';
|
||||||
}).join("\n");
|
}).join("\n");
|
||||||
const html = '<!DOCTYPE html><html><head><meta charset="utf-8"><style>body{font-family:Microsoft YaHei,sans-serif;margin:16px;font-size:14px;color:#1f2937}label{display:block;margin-bottom:10px}input,select,textarea{width:100%;box-sizing:border-box;padding:6px;margin-top:4px}#ok{margin-top:8px;padding:8px 24px;background:#2563eb;color:#fff;border:none;border-radius:6px;cursor:pointer}</style></head><body><h3>' + escapeHtml(options.title) + "</h3>" + rows + '<button id="ok">确认</button><script>document.getElementById("ok").onclick=function(){var out={};document.querySelectorAll("[data-key]").forEach(function(el){out[el.dataset.key]=el.type==="checkbox"?el.checked:el.value});window.__formValues=out;};</script></body></html>';
|
const html = '<!DOCTYPE html><html><head><meta charset="utf-8"><style>body{font-family:Microsoft YaHei,sans-serif;margin:16px;font-size:14px;color:#1f2937}label{display:block;margin-bottom:10px}input,select,textarea{width:100%;box-sizing:border-box;padding:6px;margin-top:4px}#ok{margin-top:8px;padding:8px 24px;background:#2563eb;color:#fff;border:none;border-radius:6px;cursor:pointer}</style></head><body><h3>' + escapeHtml(options.title) + "</h3>" + rows + '<button id="ok">确认</button><script>document.getElementById("ok").onclick=function(){var out={};document.querySelectorAll("[data-key]").forEach(function(el){out[el.dataset.key]=el.type==="checkbox"?el.checked:el.value});window.__formValues=out;};</script></body></html>';
|
||||||
const win = new BrowserWindow({
|
const win: HostDialogWindow = bridge.createFormWindow({ title: options.title });
|
||||||
width: 480, height: 0, useContentSize: true, autoHideMenuBar: true,
|
|
||||||
title: options.title, alwaysOnTop: true,
|
|
||||||
webPreferences: { contextIsolation: true, nodeIntegration: false, sandbox: true }
|
|
||||||
});
|
|
||||||
let settled = false;
|
let settled = false;
|
||||||
let timer: NodeJS.Timeout | undefined;
|
let timer: NodeJS.Timeout | undefined;
|
||||||
let timeout: NodeJS.Timeout | undefined;
|
let timeout: NodeJS.Timeout | undefined;
|
||||||
@ -71,14 +73,14 @@ async function showForm(options: { title: string; fields: FormFieldSpec[]; timeo
|
|||||||
};
|
};
|
||||||
timer = setInterval(async () => {
|
timer = setInterval(async () => {
|
||||||
try {
|
try {
|
||||||
const values = await win.webContents.executeJavaScript("window.__formValues || null", true);
|
const values = await win.executeJavaScript("window.__formValues || null");
|
||||||
if (values) finish(values as Record<string, string>);
|
if (values) finish(values as Record<string, string>);
|
||||||
} catch {
|
} catch {
|
||||||
finish({});
|
finish({});
|
||||||
}
|
}
|
||||||
}, 250);
|
}, 250);
|
||||||
timeout = setTimeout(() => {
|
timeout = setTimeout(() => {
|
||||||
try { void win.webContents.executeJavaScript("document.getElementById(\"ok\").click()"); } catch {
|
try { void win.executeJavaScript("document.getElementById(\"ok\").click()"); } catch {
|
||||||
finish({});
|
finish({});
|
||||||
}
|
}
|
||||||
}, Math.max(5, options.timeoutSec) * 1000);
|
}, Math.max(5, options.timeoutSec) * 1000);
|
||||||
43
packages/rpa-core/src/capabilities/index.ts
Normal file
43
packages/rpa-core/src/capabilities/index.ts
Normal file
@ -0,0 +1,43 @@
|
|||||||
|
/**
|
||||||
|
* rpa-core 能力注册入口。
|
||||||
|
*
|
||||||
|
* import 本模块即完成全部包内能力的副作用注册(与改造前
|
||||||
|
* src/main/capabilities/index.ts 的加载顺序保持一致,别名/重复 id 的
|
||||||
|
* last-wins 语义不变)。依赖 Electron 窗口的浏览器能力(ai-caps / browser* /
|
||||||
|
* web-batch2 / web-crawl)留在宿主 src/main/capabilities;
|
||||||
|
* human-dialog / ai-service / window-extended 已于阶段 2 经 HostBridge 迁入。
|
||||||
|
*/
|
||||||
|
import "./control";
|
||||||
|
import "./files";
|
||||||
|
import "./http";
|
||||||
|
import "./data";
|
||||||
|
import "./office";
|
||||||
|
import "./desktop";
|
||||||
|
import "./desktop-uia";
|
||||||
|
import "./time-caps";
|
||||||
|
import "./windows-system";
|
||||||
|
import "./ftp";
|
||||||
|
import "./ocr-caps";
|
||||||
|
import "./pdf";
|
||||||
|
import "./word";
|
||||||
|
import "./word-extended";
|
||||||
|
import "./message";
|
||||||
|
import "./vision";
|
||||||
|
import "./cv-vision";
|
||||||
|
import "./extended";
|
||||||
|
import "./system-extended";
|
||||||
|
import "./office-file-extended";
|
||||||
|
import "./office-aliases";
|
||||||
|
import "./excel-common";
|
||||||
|
import "./excel-com";
|
||||||
|
import "./data-extended";
|
||||||
|
import "./utility-extended";
|
||||||
|
import "./data-table-common";
|
||||||
|
import "./basic-encryption";
|
||||||
|
import "./mail-remaining";
|
||||||
|
import "./remaining-core";
|
||||||
|
import "./captcha-caps";
|
||||||
|
import "./common-final";
|
||||||
|
import "./human-dialog";
|
||||||
|
import "./ai-service";
|
||||||
|
import "./window-extended";
|
||||||
60
packages/rpa-core/src/capabilities/mail-remaining.ts
Normal file
60
packages/rpa-core/src/capabilities/mail-remaining.ts
Normal file
@ -0,0 +1,60 @@
|
|||||||
|
import { getHandler, register } from "../engine/registry";
|
||||||
|
import { RuntimeError, type CapabilityHandler } from "../engine/types";
|
||||||
|
import { findImapMail, type NormalizedMail } from "./message";
|
||||||
|
|
||||||
|
function text(args: Record<string, unknown>, keys: string[], fallback = ""): string {
|
||||||
|
for (const key of keys) { const value = args[key]; if (value !== undefined && value !== null && value !== "") return String(value); }
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
function number(value: unknown, fallback = 1): number { const parsed = Number(value); return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : fallback; }
|
||||||
|
function record(value: unknown): Record<string, unknown> { return value && typeof value === "object" ? value as Record<string, unknown> : (() => { try { return JSON.parse(String(value || "{}")); } catch { return {}; } })(); }
|
||||||
|
function list(value: unknown): string[] { if (Array.isArray(value)) return value.map(String); try { const parsed = JSON.parse(String(value || "[]")); return Array.isArray(parsed) ? parsed.map(String) : [String(parsed)]; } catch { return String(value || "").split(/[,,;;\s]+/).filter(Boolean); } }
|
||||||
|
function mergeAccount(args: Record<string, unknown>): Record<string, unknown> { return { ...record(args.account_obj || args.recvObject || args.target_obj), ...args }; }
|
||||||
|
async function imapClient(args: Record<string, unknown>) {
|
||||||
|
const { ImapFlow } = await import("imapflow"); const config = mergeAccount(args);
|
||||||
|
const client = new ImapFlow({ host: text(config, ["host", "hostname", "server_address", "server"]), port: number(config.port, 993), secure: ["true", "1", "True"].includes(text(config, ["ssl", "is_ssl", "secure"], "True")), auth: { user: text(config, ["account", "username", "user"]), pass: text(config, ["password", "pass"]) }, logger: false });
|
||||||
|
await client.connect(); return client;
|
||||||
|
}
|
||||||
|
async function withMailbox<T>(args: Record<string, unknown>, callback: (client: any) => Promise<T>): Promise<T> {
|
||||||
|
const client = await imapClient(args); const lock = await client.getMailboxLock(text(args, ["folder", "email_folder", "inbox_type", "folder_name"], "INBOX"));
|
||||||
|
try { return await callback(client); } finally { lock.release(); client.close(); }
|
||||||
|
}
|
||||||
|
async function searchMails(args: Record<string, unknown>): Promise<NormalizedMail[]> {
|
||||||
|
const normalized: Record<string, unknown> = { ...args, From_data: text(args, ["From_data", "From", "sender_key", "recipient_key"]), Subject_data: text(args, ["Subject_data", "Subject", "subject_key"]), Body_data: text(args, ["Body_data", "body_key"]) };
|
||||||
|
return await findImapMail(normalized, { id: "mail-remaining", componentId: "mail-remaining" });
|
||||||
|
}
|
||||||
|
const connect: CapabilityHandler = async (_ctx, _node, args) => { const config = mergeAccount(args); await imapClient(config).then((client) => client.close()); return { kind: "imap", ...config }; };
|
||||||
|
register(["Mail_RecvConnect", "Mail_UnionConnect", "Mail_ExConnect"], connect);
|
||||||
|
register("Mail_RecvClose", async () => ({ closed: true }));
|
||||||
|
register("Mail_RecvGetList", async (_ctx, _node, args) => await searchMails(args));
|
||||||
|
register("Mail_ExSearchMail", async (_ctx, _node, args) => await searchMails(args));
|
||||||
|
register("Mail_RecvMailGet", async (_ctx, _node, args) => {
|
||||||
|
const mails = await searchMails(args); const wanted = number(args.limit || args.index, 1); const mail = mails[wanted - 1];
|
||||||
|
if (!mail) throw new RuntimeError("not_found", "未找到第 " + wanted + " 封邮件"); return mail;
|
||||||
|
});
|
||||||
|
register("Mail_RecvCount", async (_ctx, _node, args) => await withMailbox(args, async (client) => client.mailbox.exists));
|
||||||
|
async function setSeen(args: Record<string, unknown>, seen: boolean) {
|
||||||
|
return await withMailbox(args, async (client) => { const sequence = String(number(args.index || args.limit, 1)); const flag = String.fromCharCode(92) + "Seen";
|
||||||
|
return seen ? await client.messageFlagsAdd(sequence, [flag]) : await client.messageFlagsRemove(sequence, [flag]); });
|
||||||
|
}
|
||||||
|
register("Mail_MarkSeen", async (_ctx, _node, args) => setSeen(args, true));
|
||||||
|
register("Mail_UnmarkSeen", async (_ctx, _node, args) => setSeen(args, false));
|
||||||
|
register(["Mail_DoMoveMail", "Mail_OperateMail", "Mail_ExOperateMail", "Mail_DoMoveMail_v1"], async (_ctx, _node, args) => {
|
||||||
|
const destination = text(args, ["DstFile", "destination", "folder_name", "email_folder"], "Archive");
|
||||||
|
return await withMailbox(args, async (client) => await client.messageMove(String(number(args.index || args.limit, 1)), destination));
|
||||||
|
});
|
||||||
|
async function invoke(id: string, args: Record<string, unknown>) { const handler = getHandler(id); if (!handler) throw new RuntimeError("capability_not_implemented", id + " 不可用"); return handler; }
|
||||||
|
register("Mail_DoSendMail", async (ctx, node, args) => {
|
||||||
|
const handler = await invoke("Mail_Send", args); const config = mergeAccount(args);
|
||||||
|
return await handler(ctx, node, { ...args, host: text(config, ["host", "server_address"]), account: text(config, ["Account", "account", "username"]), password: text(config, ["password"]), to: list(args.RecvByEmail), cc: list(args.CC), bcc: list(args.BCC), subject: text(args, ["Subject"]), content: text(args, ["Body", "body"]), attachment: text(args, ["filePath", "attachment"]) }, {} as never);
|
||||||
|
});
|
||||||
|
register("Mail_DoSendMail_v1", async (ctx, node, args) => { const handler = await invoke("Mail_DoSendMail", args); return await handler(ctx, node, { ...args, account_obj: mergeAccount(args), RecvByEmail: args.RecvByEmail || args.target_email, CC: args.CC || args.cc, Subject: args.Subject || args.subject, Body: args.Body || args.body || args.content, filePath: args.filePath || args.attachment }, {} as never); });
|
||||||
|
register("Mail_ExSendMail", async (ctx, node, args) => { const handler = await invoke("Mail_Send", args);
|
||||||
|
return await handler(ctx, node, { ...args, to: list(args.target_email), cc: list(args.cc), subject: text(args, ["subject"]), content: text(args, ["body"]), attachment: text(args, ["filePath", "attachment"]) }, {} as never);
|
||||||
|
});
|
||||||
|
register("Mail_DoReplyMail", async (ctx, node, args) => { const handler = await invoke("Mail_ReplyMail_v2", args);
|
||||||
|
return await handler(ctx, node, { ...args, subject: text(args, ["Subject"], "Re:"), content: text(args, ["ReplyBody", "content"]), attachment: text(args, ["filePath", "attachment"]) }, {} as never);
|
||||||
|
});
|
||||||
|
register("Mail_DoReplyMail_v1", async (ctx, node, args) => { const handler = await invoke("Mail_ReplyMail_v2", args);
|
||||||
|
return await handler(ctx, node, { ...args, subject: text(args, ["subject"], "Re:"), content: text(args, ["content", "ReplyBody"]), attachment: text(args, ["attachment"]) }, {} as never);
|
||||||
|
});
|
||||||
174
packages/rpa-core/src/capabilities/message.ts
Normal file
174
packages/rpa-core/src/capabilities/message.ts
Normal file
@ -0,0 +1,174 @@
|
|||||||
|
import nodemailer from "nodemailer";
|
||||||
|
import { register } from "../engine/registry";
|
||||||
|
import { RuntimeError } from "../engine/types";
|
||||||
|
import { powershell } from "../driver/windows/powershell";
|
||||||
|
|
||||||
|
function text(args: Record<string, unknown>, keys: string[], fallback = ""): string {
|
||||||
|
for (const key of keys) if (args[key] !== undefined && args[key] !== null && args[key] !== "") return String(args[key]);
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
function list(value: string): string[] {
|
||||||
|
return value.split(/[;,,;\s]+/).map((item) => item.trim()).filter(Boolean);
|
||||||
|
}
|
||||||
|
|
||||||
|
function accountObject(value: unknown): Record<string, unknown> {
|
||||||
|
if (value && typeof value === "object") return value as Record<string, unknown>;
|
||||||
|
if (typeof value === "string" && value.trim().startsWith("{")) {
|
||||||
|
try { return JSON.parse(value); } catch { return {}; }
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
register(["Mail_Send", "Mail_Send_V1", "Mail_UnionSend", "Mail_UnionSend_v1"], async (_ctx, node, args) => {
|
||||||
|
const configured = accountObject(args.account_obj);
|
||||||
|
const account = text({ ...configured, ...args }, ["account", "from", "username", "user"]);
|
||||||
|
const password = text({ ...configured, ...args }, ["password", "pass"]);
|
||||||
|
const host = text({ ...configured, ...args }, ["host", "smtp", "smtpHost", "server"]);
|
||||||
|
if (!host || !account) throw new Error(`邮件配置不完整(${node.componentId})`);
|
||||||
|
const port = Number(text({ ...configured, ...args }, ["port", "smtpPort"], "25"));
|
||||||
|
const secure = ["true", "1", "是", "True"].includes(text({ ...configured, ...args }, ["is_ssl", "secure", "ssl"], "False"));
|
||||||
|
const transporter = nodemailer.createTransport({ host, port, secure, auth: password ? { user: account, pass: password } : undefined });
|
||||||
|
try {
|
||||||
|
const result = await transporter.sendMail({
|
||||||
|
from: account,
|
||||||
|
to: list(text(args, ["to", "receiver"])),
|
||||||
|
cc: list(text(args, ["Cc", "cc"])),
|
||||||
|
bcc: list(text(args, ["Bcc", "bcc"])),
|
||||||
|
subject: text(args, ["subject"]),
|
||||||
|
text: text(args, ["content", "body"]),
|
||||||
|
attachments: text(args, ["attachment"]) ? [{ path: text(args, ["attachment"]) }] : undefined
|
||||||
|
});
|
||||||
|
return { messageId: result.messageId, accepted: result.accepted };
|
||||||
|
} finally {
|
||||||
|
transporter.close();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
export interface NormalizedMail {
|
||||||
|
uid: string; from: string; to: string[]; cc: string[]; subject: string;
|
||||||
|
text: string; html: string; date: string; messageId: string; hasAttachment: boolean;
|
||||||
|
attachmentNames: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeAccount(args: Record<string, unknown>): Record<string, unknown> {
|
||||||
|
return accountObject(args.account_obj ?? args.accountObj ?? args.account);
|
||||||
|
}
|
||||||
|
|
||||||
|
function addressList(value: unknown): string[] {
|
||||||
|
if (!value) return [];
|
||||||
|
if (Array.isArray(value)) return value.map((item) => String(typeof item === "object" && item && "text" in (item as Record<string, unknown>) ? (item as { text: unknown }).text : item)).filter(Boolean);
|
||||||
|
if (typeof value === "object") { const record = value as { text?: unknown; value?: unknown }; return addressList(record.text ?? record.value); }
|
||||||
|
return String(value).split(/[;,,;\s]+/).filter(Boolean);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeMail(input: Record<string, unknown>, uid = ""): NormalizedMail {
|
||||||
|
const attachments = Array.isArray(input.attachments) ? input.attachments as Array<Record<string, unknown>> : [];
|
||||||
|
return {
|
||||||
|
uid: String(input.uid ?? uid),
|
||||||
|
from: String(input.from ?? input.sender ?? ""),
|
||||||
|
to: addressList(input.to), cc: addressList(input.cc),
|
||||||
|
subject: String(input.subject ?? ""),
|
||||||
|
text: String(input.text ?? input.body ?? ""),
|
||||||
|
html: String(input.html ?? ""),
|
||||||
|
date: input.date instanceof Date ? input.date.toISOString() : String(input.date ?? input.receivedTime ?? ""),
|
||||||
|
messageId: String(input.messageId ?? ""),
|
||||||
|
hasAttachment: Boolean(input.hasAttachment ?? attachments.length),
|
||||||
|
attachmentNames: attachments.map((item) => String(item.filename ?? item.name ?? "")).filter(Boolean)
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function matches(mail: NormalizedMail, args: Record<string, unknown>): boolean {
|
||||||
|
const contains = (needle: unknown, haystack: string): boolean => !needle || haystack.toLowerCase().includes(String(needle).toLowerCase());
|
||||||
|
return contains(args.From_data, mail.from)
|
||||||
|
&& contains(args.To_data, mail.to.join(" "))
|
||||||
|
&& contains(args.Subject_data, mail.subject)
|
||||||
|
&& contains(args.Body_data, mail.text + "\n" + mail.html)
|
||||||
|
&& (args.attachment === undefined || String(args.attachment) === "2" || (String(args.attachment) === "1") === mail.hasAttachment);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function findImapMail(args: Record<string, unknown>, node: { id: string; componentId: string }): Promise<NormalizedMail[]> {
|
||||||
|
const configured = normalizeAccount(args);
|
||||||
|
const merged = { ...args, ...configured };
|
||||||
|
const account = text(merged, ["account", "username", "user", "from"]);
|
||||||
|
const password = text(merged, ["password", "pass", "token"]);
|
||||||
|
const host = text(merged, ["imap", "imapHost", "imap_host", "server", "host"]);
|
||||||
|
if (!host || !account) throw new RuntimeError("validation_error", "IMAP 配置不完整:需要 host 和 account", { nodeId: node.id, componentId: node.componentId });
|
||||||
|
const port = Number(text(merged, ["imapPort", "imap_port", "recvPort", "port"], "993"));
|
||||||
|
const secure = ["true", "1", "是", "True"].includes(text(merged, ["imap_ssl", "imapSsl", "is_ssl", "secure"], "True"));
|
||||||
|
const folder = text(args, ["folder_name", "email_folder", "folder"], "INBOX");
|
||||||
|
const { ImapFlow } = await import("imapflow");
|
||||||
|
const client = new ImapFlow({ host, port, secure, auth: { user: account, pass: password }, logger: false });
|
||||||
|
const mails: NormalizedMail[] = [];
|
||||||
|
try {
|
||||||
|
await client.connect();
|
||||||
|
const lock = await client.getMailboxLock(folder);
|
||||||
|
try {
|
||||||
|
const range = String(args.search_range ?? "ALL").toUpperCase();
|
||||||
|
const criteria: Record<string, unknown> = range === "UNSEEN" ? { seen: false } : range === "SEEN" ? { seen: true } : range === "FLAGGED" ? { flagged: true } : range === "ANSWERED" ? { answered: true } : range === "DELETED" ? { deleted: true } : range === "DRAFT" ? { draft: true } : { all: true };
|
||||||
|
if (args.From_data) criteria.from = String(args.From_data);
|
||||||
|
if (args.To_data) criteria.to = String(args.To_data);
|
||||||
|
if (args.Subject_data) criteria.subject = String(args.Subject_data);
|
||||||
|
const uids = await client.search(criteria as never, { uid: true }) || [];
|
||||||
|
const selected = Math.max(1, Number(args.email_num ?? 1) || 1);
|
||||||
|
for (let index = uids.length - 1; index >= 0 && mails.length < Math.max(20, selected); index -= 1) {
|
||||||
|
const download = await client.fetchOne(String(uids[index]), { uid: true, envelope: true, flags: true, bodyStructure: true, source: true }, { uid: true });
|
||||||
|
if (!download || !download.source) continue;
|
||||||
|
const { simpleParser } = await import("mailparser");
|
||||||
|
const parsed = await simpleParser(download.source);
|
||||||
|
const mail = normalizeMail(parsed as unknown as Record<string, unknown>, String(download.uid));
|
||||||
|
if (matches(mail, args)) mails.push(mail);
|
||||||
|
}
|
||||||
|
} finally { lock.release(); }
|
||||||
|
} finally { client.close(); }
|
||||||
|
const selected = Math.max(1, Number(args.email_num ?? 1) || 1);
|
||||||
|
return mails.slice(0, selected);
|
||||||
|
}
|
||||||
|
|
||||||
|
register(["Mail_FindMail", "Mail_FindMail_v1", "Mail_FindMail_v2"], async (_ctx, node, args) => findImapMail(args, node));
|
||||||
|
|
||||||
|
register(["Mail_ReplyMail", "Mail_ReplyMail_v1", "Mail_ReplyMail_v2"], async (_ctx, node, args) => {
|
||||||
|
const configured = accountObject(args.mail_obj ?? args.email_obj ?? args.account_obj);
|
||||||
|
const mail = configured && Object.keys(configured).length ? configured as unknown as NormalizedMail : args.mail as unknown as NormalizedMail;
|
||||||
|
const merged = { ...args, ...normalizeAccount(args) };
|
||||||
|
const account = text(merged, ["account", "username", "user", "from"]);
|
||||||
|
const password = text(merged, ["password", "pass"]);
|
||||||
|
const host = text(merged, ["smtp", "smtpHost", "smtp_host", "server", "host"]);
|
||||||
|
if (!host || !account) throw new RuntimeError("validation_error", "SMTP 配置不完整:需要 host 和 account", { nodeId: node.id, componentId: node.componentId });
|
||||||
|
const port = Number(text(merged, ["smtpPort", "smtp_port", "sendPort", "port"], "465"));
|
||||||
|
const secure = ["true", "1", "是", "True"].includes(text(merged, ["smtp_ssl", "smtpSsl", "is_ssl", "secure"], "True"));
|
||||||
|
const transporter = nodemailer.createTransport({ host, port, secure, auth: password ? { user: account, pass: password } : undefined });
|
||||||
|
try {
|
||||||
|
const result = await transporter.sendMail({
|
||||||
|
from: account,
|
||||||
|
to: mail?.from ? [mail.from] : list(text(args, ["to", "receiver"])),
|
||||||
|
subject: text(args, ["subject"], mail?.subject ? "Re: " + mail.subject : "Re:"),
|
||||||
|
text: text(args, ["content", "body"]),
|
||||||
|
inReplyTo: mail?.messageId, references: mail?.messageId,
|
||||||
|
attachments: text(args, ["attachment"]) ? [{ path: text(args, ["attachment"]) }] : undefined
|
||||||
|
});
|
||||||
|
return { messageId: result.messageId, accepted: result.accepted };
|
||||||
|
} finally { transporter.close(); }
|
||||||
|
});
|
||||||
|
|
||||||
|
function outlookScript(args: Record<string, unknown>): string {
|
||||||
|
const criteria = Buffer.from(JSON.stringify({ to: String(args.To_data ?? ""), from: String(args.From_data ?? ""), subject: String(args.Subject_data ?? ""), body: String(args.Body_data ?? "") }), "utf8").toString("base64");
|
||||||
|
return [
|
||||||
|
"$outlook=New-Object -ComObject Outlook.Application;$ns=$outlook.GetNamespace('MAPI')",
|
||||||
|
`$folderPath=${JSON.stringify(String(args.folder_name || args.email_folder || "INBOX"))}`,
|
||||||
|
"$folder=$ns.GetDefaultFolder(6);foreach($part in ($folderPath -split '/')){ if($part -and $part -ne 'INBOX'){ $folder=$folder.Folders.Item($part) } }",
|
||||||
|
"$folder.Items.Sort('[ReceivedTime]',$true)|Out-Null;$rows=@();$count=[Math]::Min(500,$folder.Items.Count)",
|
||||||
|
`$criteria=[Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('${criteria}'))|ConvertFrom-Json`,
|
||||||
|
"for($i=1;$i -le $count;$i++){ $item=$folder.Items.Item($i); $subject=[string]$item.Subject; $body=[string]$item.Body; $to=[string]$item.To; $from=[string]$item.SenderEmailAddress; if(($criteria.to -and $to -notlike ('*'+$criteria.to+'*')) -or ($criteria.from -and $from -notlike ('*'+$criteria.from+'*')) -or ($criteria.subject -and $subject -notlike ('*'+$criteria.subject+'*')) -or ($criteria.body -and $body -notlike ('*'+$criteria.body+'*'))){continue}; $bodyB64=[Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($body)); $rows += @{uid=[string]$item.EntryID;from=$from;to=$to;subject=$subject;text=$bodyB64;html='';date=$item.ReceivedTime.ToString('o');messageId=[string]$item.EntryID;hasAttachment=($item.Attachments.Count -gt 0);attachmentNames=@()}; if($rows.Count -ge 100){break} }",
|
||||||
|
"$json=$rows|ConvertTo-Json -Compress -Depth 4;[Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($json))|Write-Output"
|
||||||
|
].join(";");
|
||||||
|
}
|
||||||
|
|
||||||
|
register(["Mail_DoGetMailMessage", "Mail_DoGetMailMessage_v1", "Mail_DoGetMailMessage_v2"], async (_ctx, node, args) => {
|
||||||
|
const output = await powershell(outlookScript(args), 120000);
|
||||||
|
if (!output.trim()) throw new RuntimeError("outlook_unavailable", "Outlook COM 未返回数据;请确认 Outlook 已安装并至少配置一个邮箱", { nodeId: node.id, componentId: node.componentId });
|
||||||
|
const rows = JSON.parse(Buffer.from(output.trim(), "base64").toString("utf8"));
|
||||||
|
const mails = (Array.isArray(rows) ? rows : [rows]).map((row: Record<string, unknown>) => normalizeMail({ ...row, text: row.text ? Buffer.from(String(row.text), "base64").toString("utf8") : "" }));
|
||||||
|
const selected = Math.max(1, Number(args.email_num ?? 1) || 1);
|
||||||
|
return mails.slice(0, selected);
|
||||||
|
});
|
||||||
116
packages/rpa-core/src/capabilities/ocr-caps.ts
Normal file
116
packages/rpa-core/src/capabilities/ocr-caps.ts
Normal file
@ -0,0 +1,116 @@
|
|||||||
|
import path from "node:path";
|
||||||
|
import { promises as fs } from "node:fs";
|
||||||
|
import { register } from "../engine/registry";
|
||||||
|
import { RuntimeError } from "../engine/types";
|
||||||
|
import { defaultEngine } from "../engine/engine";
|
||||||
|
|
||||||
|
interface OcrWord { text: string; x: number; y: number; width: number; height: number; confidence: number }
|
||||||
|
interface RapidOcrLine { text: string; mean: number; box?: number[][] }
|
||||||
|
|
||||||
|
async function getOcr(): Promise<{ detect: (image: string) => Promise<{ texts: RapidOcrLine[] }> }> {
|
||||||
|
if (defaultEngine.ocrInstance) return defaultEngine.ocrInstance as { detect: (image: string) => Promise<{ texts: RapidOcrLine[] }> };
|
||||||
|
const mod = await import("@gutenye/ocr-node");
|
||||||
|
const instance = await mod.default.create();
|
||||||
|
defaultEngine.ocrInstance = instance;
|
||||||
|
return instance;
|
||||||
|
}
|
||||||
|
|
||||||
|
function requireImage(args: Record<string, unknown>, nodeId: string): string {
|
||||||
|
// uiSelector 是 OcrOffline_GetOcrTextNew 目录里唯一的图片字段(P0-1 修复)
|
||||||
|
for (const key of ["image_path", "imagePath", "file_path", "filePath", "path", "image", "pictureSource", "uiSelector"]) {
|
||||||
|
const v = String(args[key] ?? "").trim();
|
||||||
|
if (v) return v;
|
||||||
|
}
|
||||||
|
throw new RuntimeError("validation_error", "图片路径为空", { nodeId, componentId: nodeId, recommendation: "填写本地图片路径(png/jpg/bmp)" });
|
||||||
|
}
|
||||||
|
|
||||||
|
function boxToRect(box: number[][] | undefined): { x: number; y: number; width: number; height: number } {
|
||||||
|
const points = box ?? [];
|
||||||
|
const xs = points.map(p => p[0]);
|
||||||
|
const ys = points.map(p => p[1]);
|
||||||
|
const minX = Math.min(...(xs.length ? xs : [0]));
|
||||||
|
const minY = Math.min(...(ys.length ? ys : [0]));
|
||||||
|
const maxX = Math.max(...(xs.length ? xs : [0]));
|
||||||
|
const maxY = Math.max(...(ys.length ? ys : [0]));
|
||||||
|
return { x: minX, y: minY, width: maxX - minX, height: maxY - minY };
|
||||||
|
}
|
||||||
|
|
||||||
|
register("OcrOffline_GetOcrTextNew", async (ctx, node, args) => {
|
||||||
|
const imagePath = requireImage(args, node.id);
|
||||||
|
const ocr = await getOcr();
|
||||||
|
const result = await ocr.detect(imagePath);
|
||||||
|
const text = result.texts.map(t => t.text).join("\n");
|
||||||
|
ctx.log("OCR 识别完成:" + text.length + " 字符 / " + result.texts.length + " 行", "info", node.id);
|
||||||
|
return text;
|
||||||
|
});
|
||||||
|
|
||||||
|
register("OcrOffline_GetOcrTextPosition", async (ctx, node, args) => {
|
||||||
|
const imagePath = requireImage(args, node.id);
|
||||||
|
const ocr = await getOcr();
|
||||||
|
const result = await ocr.detect(imagePath);
|
||||||
|
const words = result.texts.map(t => {
|
||||||
|
const rect = boxToRect(t.box);
|
||||||
|
return { text: t.text, x: rect.x, y: rect.y, width: rect.width, height: rect.height, confidence: t.mean };
|
||||||
|
});
|
||||||
|
ctx.log("OCR 定位完成:" + words.length + " 行", "info", node.id);
|
||||||
|
return words;
|
||||||
|
});
|
||||||
|
|
||||||
|
/** 按垂直中心聚类成行(行高 60% 为阈值),行内按 x 排序,返回二维文本表格 */
|
||||||
|
function groupOcrRows(words: RapidOcrLine[]): string[][] {
|
||||||
|
const prepared = words
|
||||||
|
.filter((word) => String(word.text ?? "").trim())
|
||||||
|
.map((word) => {
|
||||||
|
const box = word.box ?? [];
|
||||||
|
const xs = box.map((point) => point[0]);
|
||||||
|
const ys = box.map((point) => point[1]);
|
||||||
|
const x = xs.length ? Math.min(...xs) : 0;
|
||||||
|
const y = ys.length ? Math.min(...ys) : 0;
|
||||||
|
const height = ys.length ? Math.max(...ys) - y : 0;
|
||||||
|
return { text: word.text, x, cy: y + height / 2, h: height };
|
||||||
|
})
|
||||||
|
.sort((a, b) => a.cy - b.cy || a.x - b.x);
|
||||||
|
const rows: Array<{ cy: number; h: number; cells: typeof prepared }> = [];
|
||||||
|
let current: (typeof rows)[number] | null = null;
|
||||||
|
for (const item of prepared) {
|
||||||
|
if (!current || Math.abs(item.cy - current.cy) > Math.max(current.h, item.h, 8) * 0.6) {
|
||||||
|
current = { cy: item.cy, h: item.h, cells: [] };
|
||||||
|
rows.push(current);
|
||||||
|
}
|
||||||
|
current.cells.push(item);
|
||||||
|
}
|
||||||
|
return rows.map((row) => row.cells.sort((a, b) => a.x - b.x).map((cell) => cell.text.trim()));
|
||||||
|
}
|
||||||
|
|
||||||
|
register("OcrOffline_GetOcrTable", async (ctx, node, args) => {
|
||||||
|
const imagePath = requireImage(args, node.id);
|
||||||
|
const ocr = await getOcr();
|
||||||
|
const result = await ocr.detect(imagePath);
|
||||||
|
const rows = groupOcrRows(result.texts);
|
||||||
|
ctx.log("OCR 表格识别完成:" + rows.length + " 行", "info", node.id);
|
||||||
|
|
||||||
|
// showTable=1:写入流程数据表(与 Bivartable 系列组件共享同一份数据簿)
|
||||||
|
if (String(args.showTable ?? "1") === "1") {
|
||||||
|
ctx.vars["__rpaDatabook"] = rows;
|
||||||
|
ctx.log("已写入流程数据表", "info", node.id);
|
||||||
|
}
|
||||||
|
|
||||||
|
// isExcel=1:另存为 Excel 文件
|
||||||
|
let excelFile = "";
|
||||||
|
if (String(args.isExcel ?? "0") === "1") {
|
||||||
|
const ExcelJS = (await import("exceljs")).default;
|
||||||
|
const workbook = new ExcelJS.Workbook();
|
||||||
|
const sheet = workbook.addWorksheet("Sheet1");
|
||||||
|
sheet.addRows(rows);
|
||||||
|
const rawFolder = String(args.excelPath ?? "").trim();
|
||||||
|
// ${res} 流程资源目录占位与空值统一落到运行证据目录
|
||||||
|
const folder = rawFolder && rawFolder !== "${res}" ? rawFolder : ctx.evidenceDir;
|
||||||
|
const fileName = String(args.excelName ?? "表格数据.xlsx").replace(/\$\{randomString\}/g, Math.random().toString(36).slice(2, 8));
|
||||||
|
excelFile = path.resolve(path.isAbsolute(fileName) ? fileName : path.join(folder, fileName));
|
||||||
|
await fs.mkdir(path.dirname(excelFile), { recursive: true });
|
||||||
|
await workbook.xlsx.writeFile(excelFile);
|
||||||
|
ctx.log("已保存 Excel:" + excelFile, "info", node.id);
|
||||||
|
}
|
||||||
|
|
||||||
|
return excelFile ? { rows, excelFile } : rows;
|
||||||
|
});
|
||||||
@ -3,6 +3,7 @@ import { promises as fs } from "node:fs";
|
|||||||
import path from "node:path";
|
import path from "node:path";
|
||||||
import { register } from "../engine/registry";
|
import { register } from "../engine/registry";
|
||||||
import { RuntimeError, type CapabilityHandler } from "../engine/types";
|
import { RuntimeError, type CapabilityHandler } from "../engine/types";
|
||||||
|
import { defaultEngine } from "../engine/engine";
|
||||||
import { acquireWorkbook, publishWorkbook, releaseWorkbook } from "./excel-store";
|
import { acquireWorkbook, publishWorkbook, releaseWorkbook } from "./excel-store";
|
||||||
|
|
||||||
interface ExcelHandle {
|
interface ExcelHandle {
|
||||||
@ -10,7 +11,8 @@ interface ExcelHandle {
|
|||||||
path: string;
|
path: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
const workbooks = new Map<string, ExcelJS.Workbook>();
|
// 句柄 id → 工作簿;本体挂在引擎实例上(任务 1.3),此处委托 defaultEngine。
|
||||||
|
const workbooks = defaultEngine.excelHandles;
|
||||||
|
|
||||||
function value(args: Record<string, unknown>, keys: string[], fallback = ""): unknown {
|
function value(args: Record<string, unknown>, keys: string[], fallback = ""): unknown {
|
||||||
for (const key of keys) {
|
for (const key of keys) {
|
||||||
264
packages/rpa-core/src/capabilities/pdf.ts
Normal file
264
packages/rpa-core/src/capabilities/pdf.ts
Normal file
@ -0,0 +1,264 @@
|
|||||||
|
import { promises as fs } from "node:fs";
|
||||||
|
import fsSync from "node:fs";
|
||||||
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||||
|
import { PDFDocument } from "pdf-lib";
|
||||||
|
import path from "node:path";
|
||||||
|
import { register } from "../engine/registry";
|
||||||
|
import { RuntimeError, type CapabilityHandler } from "../engine/types";
|
||||||
|
|
||||||
|
function text(args: Record<string, unknown>, keys: string[], fallback = ""): string {
|
||||||
|
for (const key of keys) if (args[key] !== undefined && args[key] !== null && args[key] !== "") return String(args[key]);
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
function required(value: string, label: string, nodeId: string, componentId: string): string {
|
||||||
|
if (!value.trim()) throw new RuntimeError("validation_error", `${label}为空`, { nodeId, componentId });
|
||||||
|
return path.resolve(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
const pageCount: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
|
const filePath = required(text(args, ["fileInPath", "path", "file"]), "PDF文件路径", node.id, node.componentId);
|
||||||
|
const document = await PDFDocument.load(await fs.readFile(filePath), { ignoreEncryption: true });
|
||||||
|
return document.getPageCount();
|
||||||
|
};
|
||||||
|
|
||||||
|
const concat: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
|
const folder = required(text(args, ["newPdfPath", "folder", "path"]), "输出目录", node.id, node.componentId);
|
||||||
|
const name = text(args, ["newPdfName", "name"], "合并结果.pdf");
|
||||||
|
const output = path.join(folder, path.extname(name) ? name : `${name}.pdf`);
|
||||||
|
const rawList = text(args, ["pdfList", "files"]);
|
||||||
|
let files: string[] = [];
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(rawList);
|
||||||
|
files = Array.isArray(parsed) ? parsed.map(String) : [String(parsed)];
|
||||||
|
} catch {
|
||||||
|
files = rawList.split(/[\r\n;,]+/).map((item) => item.trim()).filter(Boolean);
|
||||||
|
}
|
||||||
|
if (!files.length) throw new RuntimeError("validation_error", "待合并 PDF 列表为空", { nodeId: node.id, componentId: node.componentId });
|
||||||
|
const target = await PDFDocument.create();
|
||||||
|
for (const file of files) {
|
||||||
|
const source = await PDFDocument.load(await fs.readFile(path.resolve(file)), { ignoreEncryption: true });
|
||||||
|
const pages = await target.copyPages(source, source.getPageIndices());
|
||||||
|
pages.forEach((page) => target.addPage(page));
|
||||||
|
}
|
||||||
|
await fs.mkdir(path.dirname(output), { recursive: true });
|
||||||
|
await fs.writeFile(output, await target.save());
|
||||||
|
return output;
|
||||||
|
};
|
||||||
|
|
||||||
|
register(["PDF_GetPdfPageNum", "PDF_GetPageCount"], pageCount);
|
||||||
|
register("PDF_ConcatPdf", concat);
|
||||||
|
register("PDF_GetPdfText", async (_ctx, node, args) => {
|
||||||
|
const filePath = required(text(args, ["fileInPath", "path", "file"]), "PDF文件路径", node.id, node.componentId);
|
||||||
|
const pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs");
|
||||||
|
const document = await pdfjs.getDocument({ data: new Uint8Array(await fs.readFile(filePath)) }).promise;
|
||||||
|
const pages = text(args, ["pdfNum", "page"]);
|
||||||
|
const indices = pages ? pages.split(/[ ,,;;]+/).map((item) => Number(item)).filter((item) => item > 0) : Array.from({ length: document.numPages }, (_, index) => index + 1);
|
||||||
|
const chunks: string[] = [];
|
||||||
|
for (const index of indices) {
|
||||||
|
const page = await document.getPage(index);
|
||||||
|
const content = await page.getTextContent();
|
||||||
|
chunks.push(content.items.map((item) => ("str" in item ? item.str : "")).join(""));
|
||||||
|
}
|
||||||
|
return chunks.join("\n");
|
||||||
|
});
|
||||||
|
|
||||||
|
type PdfJsDocument = Awaited<ReturnType<typeof import("pdfjs-dist/legacy/build/pdf.mjs")["getDocument"]>["promise"]>;
|
||||||
|
|
||||||
|
function standardFontUrl(): string | undefined {
|
||||||
|
const candidates = [
|
||||||
|
process.env.RPA_PDFJS_STANDARD_FONTS,
|
||||||
|
path.join(process.cwd(), "node_modules", "pdfjs-dist", "standard_fonts"),
|
||||||
|
path.join(process.execPath, "resources", "app.asar", "node_modules", "pdfjs-dist", "standard_fonts"),
|
||||||
|
path.join(process.execPath, "..", "resources", "app.asar", "node_modules", "pdfjs-dist", "standard_fonts")
|
||||||
|
].filter(Boolean) as string[];
|
||||||
|
const found = candidates.find((candidate) => fsSync.existsSync(candidate));
|
||||||
|
return found ? pathToFileURL(path.join(found, path.sep)).href : undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
class LocalStandardFontFactory {
|
||||||
|
baseUrl: string;
|
||||||
|
constructor(options: { baseUrl: string }) { this.baseUrl = options.baseUrl; }
|
||||||
|
async fetch({ filename }: { filename: string }): Promise<Uint8Array> {
|
||||||
|
return new Uint8Array(await fs.readFile(path.join(fileURLToPath(this.baseUrl), filename)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadPdf(filePath: string, password: string) {
|
||||||
|
const canvasModule = await import("@napi-rs/canvas");
|
||||||
|
const globals = globalThis as unknown as Record<string, unknown>;
|
||||||
|
globals.Path2D = canvasModule.Path2D;
|
||||||
|
globals.DOMMatrix = canvasModule.DOMMatrix;
|
||||||
|
globals.ImageData = canvasModule.ImageData;
|
||||||
|
const pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs");
|
||||||
|
return { pdfjs, document: await pdfjs.getDocument({ data: new Uint8Array(await fs.readFile(filePath)), password: password || undefined, isEvalSupported: false, useSystemFonts: false, standardFontDataUrl: standardFontUrl(), StandardFontDataFactory: LocalStandardFontFactory }).promise };
|
||||||
|
}
|
||||||
|
|
||||||
|
function pageIndices(document: PdfJsDocument, raw: unknown): number[] {
|
||||||
|
const range = String(raw ?? "all").trim().toLowerCase();
|
||||||
|
if (!range || range === "all") return Array.from({ length: document.numPages }, (_, index) => index + 1);
|
||||||
|
const result = new Set<number>();
|
||||||
|
for (const part of range.split(/[,,;;\s]+/).filter(Boolean)) {
|
||||||
|
const match = part.match(/^(\d+)-(\d+)$/);
|
||||||
|
if (match) {
|
||||||
|
const start = Math.max(1, Number(match[1])); const end = Math.min(document.numPages, Number(match[2]));
|
||||||
|
for (let index = start; index <= end; index += 1) result.add(index);
|
||||||
|
} else {
|
||||||
|
const index = Number(part);
|
||||||
|
if (index >= 1 && index <= document.numPages) result.add(Math.floor(index));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const values = [...result].sort((left, right) => left - right);
|
||||||
|
if (!values.length) throw new Error(`PDF 页码范围无效:${range}`);
|
||||||
|
return values;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function renderPage(page: Awaited<ReturnType<PdfJsDocument["getPage"]>>, scale = 2): Promise<Buffer> {
|
||||||
|
const canvasModule = await import("@napi-rs/canvas");
|
||||||
|
const viewport = page.getViewport({ scale });
|
||||||
|
const canvas = canvasModule.createCanvas(Math.ceil(viewport.width), Math.ceil(viewport.height));
|
||||||
|
const context = canvas.getContext("2d");
|
||||||
|
context.fillStyle = "white"; context.fillRect(0, 0, canvas.width, canvas.height);
|
||||||
|
await page.render({ canvasContext: context as unknown as CanvasRenderingContext2D, viewport }).promise;
|
||||||
|
return canvas.toBuffer("image/png");
|
||||||
|
}
|
||||||
|
|
||||||
|
function saveImage(folder: string, prefix: string, suffix: string, data: Buffer, type: string): string {
|
||||||
|
const ext = /jpe?g/i.test(type) ? "jpg" : "png";
|
||||||
|
const output = path.join(path.resolve(folder), `${prefix}_${suffix}_${Date.now()}.${ext}`);
|
||||||
|
fsSync.mkdirSync(path.dirname(output), { recursive: true });
|
||||||
|
fsSync.writeFileSync(output, data);
|
||||||
|
return output;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function rawImageToBuffer(image: Record<string, unknown>, type: string): Promise<Buffer> {
|
||||||
|
const width = Number(image.width || 0); const height = Number(image.height || 0);
|
||||||
|
const data = image.data;
|
||||||
|
if (!width || !height || !(data instanceof Uint8Array) || data.length === 0) throw new Error("PDF 图片数据无效");
|
||||||
|
if (data[0] === 0xff && data[1] === 0xd8) return Buffer.from(data);
|
||||||
|
const canvasModule = await import("@napi-rs/canvas");
|
||||||
|
const canvas = canvasModule.createCanvas(width, height); const context = canvas.getContext("2d");
|
||||||
|
const target = context.createImageData(width, height); const kind = Number(image.kind || 0);
|
||||||
|
if (kind === 1) {
|
||||||
|
const bytesPerRow = Math.ceil(width / 8);
|
||||||
|
for (let y = 0; y < height; y += 1) for (let x = 0; x < width; x += 1) {
|
||||||
|
const bit = (data[y * bytesPerRow + (x >> 3)] >> (7 - (x & 7))) & 1; const output = (y * width + x) * 4; const color = bit ? 255 : 0;
|
||||||
|
target.data[output] = color; target.data[output + 1] = color; target.data[output + 2] = color; target.data[output + 3] = 255;
|
||||||
|
}
|
||||||
|
} else if (kind === 3) { target.data.set(data.subarray(0, Math.min(data.length, target.data.length))); }
|
||||||
|
else {
|
||||||
|
for (let source = 0, targetIndex = 0; source < data.length && targetIndex < target.data.length; source += 3, targetIndex += 4) {
|
||||||
|
target.data[targetIndex] = data[source]; target.data[targetIndex + 1] = data[source + 1] ?? 0; target.data[targetIndex + 2] = data[source + 2] ?? 0; target.data[targetIndex + 3] = 255;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
context.putImageData(target, 0, 0);
|
||||||
|
return /jpe?g/i.test(type) ? canvas.toBuffer("image/jpeg") : canvas.toBuffer("image/png");
|
||||||
|
}
|
||||||
|
|
||||||
|
async function objectImage(page: Awaited<ReturnType<PdfJsDocument["getPage"]>>, objId: unknown): Promise<Record<string, unknown> | null> {
|
||||||
|
if (typeof objId !== "string") return null;
|
||||||
|
const objects = objId.startsWith("g_") ? page.commonObjs : page.objs;
|
||||||
|
try { if (objects.has(objId)) return objects.get(objId) as Record<string, unknown>; } catch {}
|
||||||
|
return await new Promise((resolve) => { try { objects.get(objId, (value: unknown) => resolve(value as Record<string, unknown>)); } catch { resolve(null); } });
|
||||||
|
}
|
||||||
|
|
||||||
|
const encryption: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
|
const filePath = required(text(args, ["fileInPath", "path", "file"]), "PDF文件路径", node.id, node.componentId);
|
||||||
|
const password = String(args.passWord ?? args.password ?? "");
|
||||||
|
if (!password) throw new RuntimeError("validation_error", "PDF密码为空", { nodeId: node.id, componentId: node.componentId });
|
||||||
|
const { PDFDocument } = await import("@cantoo/pdf-lib");
|
||||||
|
const document = await PDFDocument.load(await fs.readFile(filePath), { ignoreEncryption: true });
|
||||||
|
document.encrypt({ userPassword: password, ownerPassword: password, permissions: { printing: "highResolution", copying: true, modifying: false } });
|
||||||
|
const temporary = `${filePath}.${Date.now()}.enc`;
|
||||||
|
await fs.writeFile(temporary, await document.save());
|
||||||
|
await fs.rename(temporary, filePath);
|
||||||
|
return filePath;
|
||||||
|
};
|
||||||
|
|
||||||
|
const extractImages: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
|
const filePath = required(text(args, ["fileInPath", "path", "file"]), "PDF文件路径", node.id, node.componentId);
|
||||||
|
const folder = required(text(args, ["imgSavePath", "pdfSavePath", "folder", "path"]), "图片保存路径", node.id, node.componentId);
|
||||||
|
const type = text(args, ["imageType", "type"], "png");
|
||||||
|
const prefix = text(args, ["imageName", "name"], "pdf-image");
|
||||||
|
const { pdfjs, document } = await loadPdf(filePath, String(args.pdfPassWord ?? args.password ?? ""));
|
||||||
|
const outputs: string[] = [];
|
||||||
|
for (const pageNumber of pageIndices(document, args.pdfNum)) {
|
||||||
|
const page = await document.getPage(pageNumber);
|
||||||
|
const opList = await page.getOperatorList();
|
||||||
|
let sequence = 0;
|
||||||
|
for (let index = 0; index < opList.fnArray.length; index += 1) {
|
||||||
|
const op = opList.fnArray[index]; const commandArgs = opList.argsArray[index] as unknown[];
|
||||||
|
let image: Record<string, unknown> | null = null;
|
||||||
|
if (op === pdfjs.OPS.paintImageXObject || op === pdfjs.OPS.paintImageXObjectRepeat) image = await objectImage(page, commandArgs[0]);
|
||||||
|
else if (op === pdfjs.OPS.paintInlineImageXObject) image = commandArgs[0] as Record<string, unknown>;
|
||||||
|
else if (op === pdfjs.OPS.paintInlineImageXObjectGroup) image = commandArgs[0] as Record<string, unknown>;
|
||||||
|
if (!image) continue;
|
||||||
|
sequence += 1;
|
||||||
|
outputs.push(saveImage(folder, `${prefix}-p${pageNumber}`, String(sequence), await rawImageToBuffer(image, type), type));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return outputs;
|
||||||
|
};
|
||||||
|
|
||||||
|
const pdfToImages: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
|
const filePath = required(text(args, ["fileInPath", "path", "file"]), "PDF文件路径", node.id, node.componentId);
|
||||||
|
const folder = required(text(args, ["pdfSavePath", "imgSavePath", "folder", "path"]), "图片保存路径", node.id, node.componentId);
|
||||||
|
const type = text(args, ["imageType", "type"], "png"); const prefix = text(args, ["imageName", "name"], "pdf-page");
|
||||||
|
const { document } = await loadPdf(filePath, String(args.pdfPassWord ?? args.password ?? ""));
|
||||||
|
const outputs: string[] = [];
|
||||||
|
for (const pageNumber of pageIndices(document, args.pdfNum)) {
|
||||||
|
const buffer = await renderPage(await document.getPage(pageNumber));
|
||||||
|
outputs.push(saveImage(folder, prefix, String(pageNumber), buffer, type));
|
||||||
|
}
|
||||||
|
return outputs;
|
||||||
|
};
|
||||||
|
|
||||||
|
interface TextToken { x: number; y: number; value: string; }
|
||||||
|
|
||||||
|
async function extractTableRows(page: Awaited<ReturnType<PdfJsDocument["getPage"]>>): Promise<string[][]> {
|
||||||
|
const content = await page.getTextContent();
|
||||||
|
const tokens = content.items.flatMap((item) => {
|
||||||
|
if (!("str" in item) || !item.str.trim()) return [];
|
||||||
|
const transform = (item as { transform?: number[] }).transform ?? [1,0,0,1,0,0];
|
||||||
|
return [{ x: Number(transform[4] || 0), y: Number(transform[5] || 0), value: item.str.trim() }];
|
||||||
|
});
|
||||||
|
if (tokens.length < 2) return [];
|
||||||
|
const lines = new Map<number, TextToken[]>();
|
||||||
|
for (const token of [...tokens].sort((left, right) => right.y - left.y || left.x - right.x)) {
|
||||||
|
const lineKey = [...lines.keys()].find((key) => Math.abs(key - token.y) <= 3.5) ?? token.y;
|
||||||
|
if (!lines.has(lineKey)) lines.set(lineKey, []);
|
||||||
|
lines.get(lineKey)!.push(token);
|
||||||
|
}
|
||||||
|
const lineList = [...lines.entries()].map(([y, values]) => ({ y, values: values.sort((left, right) => left.x - right.x) })).sort((left, right) => right.y - left.y);
|
||||||
|
const counts = new Map<number, number>();
|
||||||
|
for (const line of lineList) for (const token of line.values) {
|
||||||
|
const key = Math.round(token.x / 5) * 5;
|
||||||
|
counts.set(key, (counts.get(key) || 0) + 1);
|
||||||
|
}
|
||||||
|
const columns = [...counts.entries()].filter(([, count]) => count >= Math.min(2, lineList.length)).sort((left, right) => left[0] - right[0]).map(([key]) => key);
|
||||||
|
if (columns.length < 2) return [];
|
||||||
|
const rows: string[][] = [];
|
||||||
|
for (const line of lineList) {
|
||||||
|
const cells = columns.map(() => "");
|
||||||
|
for (const token of line.values) {
|
||||||
|
let nearest = 0;
|
||||||
|
for (let index = 1; index < columns.length; index += 1) if (Math.abs(token.x - columns[index]) < Math.abs(token.x - columns[nearest])) nearest = index;
|
||||||
|
cells[nearest] = cells[nearest] ? `${cells[nearest]} ${token.value}` : token.value;
|
||||||
|
}
|
||||||
|
if (cells.filter(Boolean).length >= 2) rows.push(cells);
|
||||||
|
}
|
||||||
|
return rows.length >= 2 ? rows : [];
|
||||||
|
}
|
||||||
|
|
||||||
|
const readTables: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
|
const filePath = required(text(args, ["fileInPath", "path", "file"]), "PDF文件路径", node.id, node.componentId);
|
||||||
|
const { document } = await loadPdf(filePath, String(args.pdfPassWord ?? args.password ?? ""));
|
||||||
|
const rows: string[][] = [];
|
||||||
|
for (const pageNumber of pageIndices(document, args.pdfNum)) rows.push(...await extractTableRows(await document.getPage(pageNumber)));
|
||||||
|
return rows;
|
||||||
|
};
|
||||||
|
|
||||||
|
register("PDF_PdfEncryption", encryption);
|
||||||
|
register("PDF_GetPdfImg", extractImages);
|
||||||
|
register("PDF_PdfToImg", pdfToImages);
|
||||||
|
register("PDF_GetPdfTable", readTables);
|
||||||
129
packages/rpa-core/src/capabilities/register-all.ts
Normal file
129
packages/rpa-core/src/capabilities/register-all.ts
Normal file
@ -0,0 +1,129 @@
|
|||||||
|
import type { RpaEngine } from "../engine/engine";
|
||||||
|
import { applyPlatformDefaults, capabilityIds, type CapabilityPlatform } from "../engine/registry";
|
||||||
|
|
||||||
|
export type { CapabilityPlatform } from "../engine/registry";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 平台归属单一权威表(迁移指南「五件小事 #1」,dev-conventions 规则 4)。
|
||||||
|
*
|
||||||
|
* - 运行时 `meta.platforms` 与构建期 catalog `platformTag` 都从这张表派生,
|
||||||
|
* 任何代码不得自行声明第二份平台数据(守卫测试 platform-groups.test.ts 把关)。
|
||||||
|
* - 现阶段所有能力只在 Windows 交互会话验证过,全部标 `["win32"]`;
|
||||||
|
* `darwin`/`linux` 仅类型占位,不填数据。将来支持新平台 = 对应行加平台,仅此而已。
|
||||||
|
* - 后续若某域内出现平台混装(如 remaining-core 里混着 PS 专用能力),
|
||||||
|
* 先把域拆细,再在能力 meta.platforms 里显式声明更窄限制。
|
||||||
|
*/
|
||||||
|
export const PLATFORM_GROUPS: Record<string, CapabilityPlatform[]> = {
|
||||||
|
control: ["win32"],
|
||||||
|
files: ["win32"],
|
||||||
|
http: ["win32"],
|
||||||
|
data: ["win32"],
|
||||||
|
office: ["win32"],
|
||||||
|
desktop: ["win32"],
|
||||||
|
desktopUia: ["win32"],
|
||||||
|
time: ["win32"],
|
||||||
|
windowsSystem: ["win32"],
|
||||||
|
ftp: ["win32"],
|
||||||
|
ocr: ["win32"],
|
||||||
|
pdf: ["win32"],
|
||||||
|
word: ["win32"],
|
||||||
|
wordExtended: ["win32"],
|
||||||
|
message: ["win32"],
|
||||||
|
vision: ["win32"],
|
||||||
|
cvVision: ["win32"],
|
||||||
|
extended: ["win32"],
|
||||||
|
systemExtended: ["win32"],
|
||||||
|
officeFileExtended: ["win32"],
|
||||||
|
officeAliases: ["win32"],
|
||||||
|
excelCommon: ["win32"],
|
||||||
|
excelCom: ["win32"],
|
||||||
|
dataExtended: ["win32"],
|
||||||
|
utilityExtended: ["win32"],
|
||||||
|
dataTableCommon: ["win32"],
|
||||||
|
basicEncryption: ["win32"],
|
||||||
|
mailRemaining: ["win32"],
|
||||||
|
remainingCore: ["win32"],
|
||||||
|
commonFinal: ["win32"],
|
||||||
|
humanDialog: ["win32"],
|
||||||
|
aiService: ["win32"],
|
||||||
|
windowExtended: ["win32"]
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 显式注册入口(任务 4.2,决策点 9:双入口过渡)。
|
||||||
|
*
|
||||||
|
* - 旧入口:`import "@rpa/core/capabilities"` 副作用注册全量(宿主/测试继续使用);
|
||||||
|
* - 新入口:`await registerAll(engine, groups?)` 按 subset 显式拉起,供
|
||||||
|
* Agent-Employee 等宿主按需注册。阶段 6 迁移时切显式入口、删副作用入口。
|
||||||
|
*
|
||||||
|
* 引擎当前持有全局注册表(阶段 1 决策 2),拉起模块即完成对全局表的注册,
|
||||||
|
* 返回 engine 以便链式使用。
|
||||||
|
*/
|
||||||
|
const DOMAINS: Record<string, () => Promise<unknown>> = {
|
||||||
|
control: () => import("./control"),
|
||||||
|
files: () => import("./files"),
|
||||||
|
http: () => import("./http"),
|
||||||
|
data: () => import("./data"),
|
||||||
|
office: () => import("./office"),
|
||||||
|
desktop: () => import("./desktop"),
|
||||||
|
desktopUia: () => import("./desktop-uia"),
|
||||||
|
time: () => import("./time-caps"),
|
||||||
|
windowsSystem: () => import("./windows-system"),
|
||||||
|
ftp: () => import("./ftp"),
|
||||||
|
ocr: () => import("./ocr-caps"),
|
||||||
|
pdf: () => import("./pdf"),
|
||||||
|
word: () => import("./word"),
|
||||||
|
wordExtended: () => import("./word-extended"),
|
||||||
|
message: () => import("./message"),
|
||||||
|
vision: () => import("./vision"),
|
||||||
|
cvVision: () => import("./cv-vision"),
|
||||||
|
extended: () => import("./extended"),
|
||||||
|
systemExtended: () => import("./system-extended"),
|
||||||
|
officeFileExtended: () => import("./office-file-extended"),
|
||||||
|
officeAliases: () => import("./office-aliases"),
|
||||||
|
excelCommon: () => import("./excel-common"),
|
||||||
|
excelCom: () => import("./excel-com"),
|
||||||
|
dataExtended: () => import("./data-extended"),
|
||||||
|
utilityExtended: () => import("./utility-extended"),
|
||||||
|
dataTableCommon: () => import("./data-table-common"),
|
||||||
|
basicEncryption: () => import("./basic-encryption"),
|
||||||
|
mailRemaining: () => import("./mail-remaining"),
|
||||||
|
remainingCore: () => import("./remaining-core"),
|
||||||
|
commonFinal: () => import("./common-final"),
|
||||||
|
humanDialog: () => import("./human-dialog"),
|
||||||
|
aiService: () => import("./ai-service"),
|
||||||
|
windowExtended: () => import("./window-extended")
|
||||||
|
};
|
||||||
|
|
||||||
|
async function loadDomainWithPlatformDefaults(domain: string): Promise<Set<string>> {
|
||||||
|
const load = DOMAINS[domain];
|
||||||
|
if (!load) throw new Error(`未知能力域:${domain}(可选:${Object.keys(DOMAINS).join(", ")})`);
|
||||||
|
const before = new Set(capabilityIds());
|
||||||
|
await load();
|
||||||
|
const fresh = [...capabilityIds()].filter((id) => !before.has(id));
|
||||||
|
// 平台默认值从 PLATFORM_GROUPS 派生(单一源);能力显式声明的 platforms 优先
|
||||||
|
applyPlatformDefaults(fresh, PLATFORM_GROUPS[domain] ?? []);
|
||||||
|
return new Set(fresh);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function registerAll(engine: RpaEngine, groups?: string[]): Promise<RpaEngine> {
|
||||||
|
const wanted = groups && groups.length ? groups : Object.keys(DOMAINS);
|
||||||
|
for (const group of wanted) await loadDomainWithPlatformDefaults(group);
|
||||||
|
return engine;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 逐域加载并返回「能力 id → 平台数组」映射(副作用:完成全量注册)。
|
||||||
|
* 供构建脚本(build-catalog)派生 catalog platformTag,与运行时同一张表。
|
||||||
|
*/
|
||||||
|
export async function buildDomainPlatformMap(): Promise<Map<string, CapabilityPlatform[]>> {
|
||||||
|
const map = new Map<string, CapabilityPlatform[]>();
|
||||||
|
for (const domain of Object.keys(DOMAINS)) {
|
||||||
|
const fresh = await loadDomainWithPlatformDefaults(domain);
|
||||||
|
const platforms = PLATFORM_GROUPS[domain] ?? [];
|
||||||
|
for (const id of fresh) map.set(id, platforms);
|
||||||
|
}
|
||||||
|
return map;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const capabilityDomains = Object.keys(DOMAINS);
|
||||||
272
packages/rpa-core/src/capabilities/remaining-core.ts
Normal file
272
packages/rpa-core/src/capabilities/remaining-core.ts
Normal file
@ -0,0 +1,272 @@
|
|||||||
|
import fs from "node:fs";
|
||||||
|
import path from "node:path";
|
||||||
|
import { getHandler, register } from "../engine/registry";
|
||||||
|
import { runFlow } from "../engine/runtime";
|
||||||
|
import { RuntimeError, type CapabilityHandler, type RunContext } from "../engine/types";
|
||||||
|
import { defaultEngine } from "../engine/engine";
|
||||||
|
import { getSubflowSource } from "../engine/host-bridge";
|
||||||
|
import { powershell, psString } from "../driver/windows/powershell";
|
||||||
|
import { findImage } from "./vision";
|
||||||
|
|
||||||
|
function text(args: Record<string, unknown>, keys: string[], fallback = ""): string {
|
||||||
|
for (const key of keys) { const value = args[key]; if (value !== undefined && value !== null && value !== "") return String(value); }
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
function num(args: Record<string, unknown>, keys: string[], fallback = 0): number { const value = Number(text(args, keys, String(fallback))); return Number.isFinite(value) ? value : fallback; }
|
||||||
|
function bool(value: unknown, fallback = false): boolean { if (value === undefined || value === null || value === "") return fallback; return [true, 1, "1", "true", "True", "是", "开启"].includes(value as never); }
|
||||||
|
function list(value: unknown): string[] {
|
||||||
|
if (Array.isArray(value)) return value.map(String);
|
||||||
|
const raw = String(value ?? "").trim(); if (!raw) return [];
|
||||||
|
try { const parsed = JSON.parse(raw); return Array.isArray(parsed) ? parsed.map(String) : [String(parsed)]; }
|
||||||
|
catch { return raw.split(/[,,;;\n]+/).map((item) => item.trim()).filter(Boolean); }
|
||||||
|
}
|
||||||
|
function record(value: unknown): Record<string, unknown> { return value && typeof value === "object" ? value as Record<string, unknown> : (() => { try { return JSON.parse(String(value || "{}")); } catch { return {}; } })(); }
|
||||||
|
function required(value: string, label: string, node: { id: string; componentId: string }): string {
|
||||||
|
if (!value.trim()) throw new RuntimeError("validation_error", label + "为空", { nodeId: node.id, componentId: node.componentId });
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 连接 id → 数据库连接;本体挂在引擎实例上(任务 1.3)。
|
||||||
|
const dbConnections = defaultEngine.dbConnections as Map<string, any>;
|
||||||
|
function dbConfig(args: Record<string, unknown>): Record<string, unknown> {
|
||||||
|
const inline = record(args.DatabaseConfig || args.config || args.connectionID);
|
||||||
|
const merged = { ...inline, ...args };
|
||||||
|
return { ...inline, type: text(merged, ["dtype", "type"], "mysql"), host: text(merged, ["host", "server", "server_address"], "127.0.0.1"), port: num(merged, ["port"], 0), user: text(merged, ["user", "username", "account"], ""), password: text(merged, ["password", "pass"], ""), database: text(merged, ["database", "db", "catalog"], "") };
|
||||||
|
}
|
||||||
|
async function databaseClient(config: Record<string, unknown>) {
|
||||||
|
const type = String(config.type || "mysql").toLowerCase();
|
||||||
|
if (type.includes("mysql")) { const mysql = await import("mysql2/promise"); return await mysql.createConnection(config as never); }
|
||||||
|
if (type.includes("postgres") || type.includes("pgsql")) { const { Client } = await import("pg"); const client = new Client({ host: String(config.host), port: Number(config.port || 5432), user: String(config.user), password: String(config.password), database: String(config.database) }); await client.connect(); return client; }
|
||||||
|
if (type.includes("sqlserver") || type.includes("mssql")) { const sql = await import("mssql"); return await sql.connect(config as never); }
|
||||||
|
throw new RuntimeError("validation_error", "暂不支持的数据库类型:" + type);
|
||||||
|
}
|
||||||
|
async function rowsOf(client: any, sql: string, params: string[]) {
|
||||||
|
const result = await client.query(sql, params);
|
||||||
|
return result.rows || result.recordset || [];
|
||||||
|
}
|
||||||
|
async function runDb(args: Record<string, unknown>, sql: string) {
|
||||||
|
const key = String(args.connectionID || "inline");
|
||||||
|
let client = dbConnections.get(key);
|
||||||
|
if (!client) { client = await databaseClient(dbConfig(args)); dbConnections.set(key, client); }
|
||||||
|
return await rowsOf(client, sql, list(args.paras));
|
||||||
|
}
|
||||||
|
|
||||||
|
register("Database_DBConnect", async (_ctx, _node, args) => {
|
||||||
|
const id = "db_" + Date.now().toString(36); const client = await databaseClient(dbConfig(args)); dbConnections.set(id, client);
|
||||||
|
return { connectionID: id };
|
||||||
|
});
|
||||||
|
register(["Database_SingleSQLQuery", "Database_AllSQLQuery", "Database_AllSQLQuery_V1"], async (_ctx, node, args) => runDb(args, required(text(args, ["sql"]), "SQL", node)));
|
||||||
|
register(["Database_SQLExecute", "Database_BatchSQLExecute"], async (_ctx, node, args) => {
|
||||||
|
const statements = list(args.sql).length ? list(args.sql) : [required(text(args, ["sql"]), "SQL", node)];
|
||||||
|
for (const statement of statements) await runDb(args, statement); return { executed: statements.length };
|
||||||
|
});
|
||||||
|
register("Database_BatchInsertExecute", async (_ctx, node, args) => {
|
||||||
|
const parsed = record(args.data); const rows = Array.isArray(parsed) ? parsed : Array.isArray(parsed.value) ? parsed.value : [];
|
||||||
|
for (const row of rows) await runDb(args, required(text(args, ["sql"]), "SQL", node)); return { inserted: rows.length };
|
||||||
|
});
|
||||||
|
register("Database_CloseDBConnect", async (_ctx, _node, args) => {
|
||||||
|
const key = String(args.connectionID || ""); const client = dbConnections.get(key);
|
||||||
|
if (client) { if (client.close) await client.close(); if (client.end) await client.end(); }
|
||||||
|
dbConnections.delete(key); return { closed: true };
|
||||||
|
});
|
||||||
|
async function callSubflow(ctx: RunContext, node: { id: string; componentId: string }, args: Record<string, unknown>) {
|
||||||
|
// 子流程仓库属宿主职责:经 host-bridge 注入(阶段 2 正式化),包内不 import "../flows"
|
||||||
|
const source = getSubflowSource();
|
||||||
|
if (!source) throw new RuntimeError("validation_error", "子流程源未注入", { nodeId: node.id, componentId: node.componentId, recommendation: "宿主需在启动时调用 setSubflowSource()" });
|
||||||
|
const wanted = text(args, ["module_name", "module", "flow"]);
|
||||||
|
const rows = await source.listFlows(); const found = rows.find((row) => row.id === wanted || row.name === wanted);
|
||||||
|
if (!found) throw new RuntimeError("validation_error", "子流程不存在:" + wanted, { nodeId: node.id, componentId: node.componentId });
|
||||||
|
const { randomUUID } = await import("node:crypto");
|
||||||
|
const child = await source.getFlow(found.id); const inputs = record(args.args || args.input);
|
||||||
|
return await runFlow(child, { runId: randomUUID(), evidenceDir: ctx.evidenceDir, signal: ctx.signal, emit: ctx.emit, log: ctx.log, componentOf: ctx.componentOf, browser: ctx.browser, variables: { ...ctx.vars, ...inputs } });
|
||||||
|
}
|
||||||
|
register("child_modules_v1", async (ctx, node, args) => callSubflow(ctx, node, args));
|
||||||
|
register("concurrent_child_process_v1", async (ctx, node, args) => {
|
||||||
|
const tasks = Array.from({ length: Math.max(1, num(args, ["count"], 2)) }, () => callSubflow(ctx, node, args));
|
||||||
|
return bool(args.is_async) ? { started: true } : await Promise.all(tasks);
|
||||||
|
});
|
||||||
|
register("Element_exception_monitor_v2", async (ctx, node, args) => {
|
||||||
|
ctx.log("异常监控已启用:" + text(args, ["event_type", "monitor_type"], "element"), bool(args.is_exits) ? "info" : "warn", node.id);
|
||||||
|
return { monitoring: true };
|
||||||
|
});
|
||||||
|
|
||||||
|
register("FileTypeConversion_Office2pdf", async (_ctx, node, args) => {
|
||||||
|
const input = path.resolve(text(args, ["fileInPath", "path"])); const folder = path.resolve(text(args, ["fileOutPath", "folder"], path.dirname(input)));
|
||||||
|
fs.mkdirSync(folder, { recursive: true }); const output = path.join(folder, path.basename(input, path.extname(input)) + ".pdf");
|
||||||
|
const ext = path.extname(input).toLowerCase();
|
||||||
|
if ([".doc", ".docx", ".rtf"].includes(ext)) await powershell(`$w=New-Object -ComObject Word.Application;$w.Visible=$false;$d=$w.Documents.Open(${psString(input)});$d.SaveAs([ref]${psString(output)},[ref]17);$d.Close($false);$w.Quit()`);
|
||||||
|
else if ([".xls", ".xlsx", ".csv"].includes(ext)) await powershell(`$e=New-Object -ComObject Excel.Application;$e.Visible=$false;$w=$e.Workbooks.Open(${psString(input)});$w.ExportAsFixedFormat(0,${psString(output)});$w.Close($false);$e.Quit()`);
|
||||||
|
else if ([".ppt", ".pptx"].includes(ext)) await powershell(`$p=New-Object -ComObject PowerPoint.Application;$pre=$p.Presentations.Open(${psString(input)},$true,$false,$false);$pre.SaveAs(${psString(output)},32);$pre.Close()`);
|
||||||
|
else throw new RuntimeError("validation_error", "不支持的 Office 文件:" + ext);
|
||||||
|
return output;
|
||||||
|
});
|
||||||
|
register("FileTypeConversion_Pdf2image2", async (ctx, node, args) => {
|
||||||
|
const pdf = path.resolve(text(args, ["pdfPath", "path"])); const folder = path.resolve(text(args, ["imageDir", "folder"], path.dirname(pdf)));
|
||||||
|
const handler = getHandler("PDF_PdfToImg"); if (!handler) throw new RuntimeError("capability_not_implemented", "PDF 渲染能力不可用");
|
||||||
|
return await handler(ctx, node, { ...args, fileInPath: pdf, pdfSavePath: folder, imageName: path.basename(pdf, path.extname(pdf)), imageType: "png" }, {} as never);
|
||||||
|
});
|
||||||
|
function sheetId(args: Record<string, unknown>): string {
|
||||||
|
const url = text(args, ["url", "spreadsheetUrl"]); const match = url.match(/\/spreadsheets\/d\/([a-zA-Z0-9-_]+)/);
|
||||||
|
if (!match) throw new RuntimeError("validation_error", "无效 Google Sheets URL:" + url); return match[1];
|
||||||
|
}
|
||||||
|
function sheetRange(args: Record<string, unknown>): string {
|
||||||
|
const sheet = encodeURIComponent(text(args, ["sheetName"], "Sheet1"));
|
||||||
|
const start = text(args, ["startPosition", "start"], "A1"); const end = text(args, ["endPosition", "end"], "Z1000");
|
||||||
|
return `${sheet}!${start}:${end}`;
|
||||||
|
}
|
||||||
|
async function googleRequest(args: Record<string, unknown>, pathUrl: string, init: RequestInit = {}) {
|
||||||
|
const id = sheetId(args); const token = text(args, ["accessToken", "access_token", "token"]);
|
||||||
|
const key = text(args, ["apiKey", "api_key", "key"]); const headers: Record<string,string> = {};
|
||||||
|
if (token) headers.Authorization = "Bearer " + token;
|
||||||
|
const separator = pathUrl.includes("?") ? "&" : "?";
|
||||||
|
const response = await fetch(`https://sheets.googleapis.com/v4/spreadsheets/${id}${pathUrl}${key ? separator + "key=" + key : ""}`, { ...init, headers: { ...headers, ...(init.headers || {}) } });
|
||||||
|
if (!response.ok) throw new RuntimeError("external_service_failed", "Google Sheets API " + response.status + ": " + await response.text());
|
||||||
|
return await response.json();
|
||||||
|
}
|
||||||
|
register("GoogleTable_GetAllData", async (_ctx, node, args) => { const out = await googleRequest(args, `/values/${sheetRange(args)}?majorDimension=ROWS`); return out.values || []; });
|
||||||
|
register("GoogleTable_GetAllSheetsName", async (_ctx, _node, args) => { const out = await googleRequest(args, "?fields=sheets.properties.title"); return (out.sheets || []).map((sheet: any) => sheet.properties.title); });
|
||||||
|
async function googleWrite(args: Record<string, unknown>) {
|
||||||
|
if (!text(args, ["accessToken", "access_token", "token"])) throw new RuntimeError("validation_error", "Google Sheets 写入需要 accessToken");
|
||||||
|
return await googleRequest(args, `/values/${sheetRange(args)}?valueInputOption=USER_ENTERED`, { method: "PUT", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ values: (() => { const value = args.Values || args.values; if (Array.isArray(value)) return value.map((row) => Array.isArray(row) ? row : [row]); try { const parsed = JSON.parse(String(value || "[]")); return Array.isArray(parsed) ? parsed.map((row: any) => Array.isArray(row) ? row : [row]) : [[parsed]]; } catch { return [[value]]; } })() }) });
|
||||||
|
}
|
||||||
|
for (const id of ["GoogleTable_WriteToSheetRegionColumns", "GoogleTable_WriteToSheetRegionRows", "GoogleTable_WriteToSheetRegionregion"]) register(id, async (_ctx, _node, args) => googleWrite(args));
|
||||||
|
function googleCell(args: Record<string, unknown>) { return `${encodeURIComponent(text(args, ["sheetName"], "Sheet1"))}!${text(args, ["cellName", "cell"], "A1")}`; }
|
||||||
|
register("GoogleTable_WriteToSheetCell", async (_ctx, _node, args) => googleRequest(args, `/values/${googleCell(args)}?valueInputOption=RAW`, { method: "PUT", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ values: [[args.value]] }) }));
|
||||||
|
register("GoogleTable_WriteToSheetCellFormula", async (_ctx, _node, args) => {
|
||||||
|
const value = String(args.value || ""); return googleRequest(args, `/values/${googleCell(args)}?valueInputOption=USER_ENTERED`, { method: "PUT", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ values: [[value.startsWith("=") ? value : "=" + value]] }) });
|
||||||
|
});
|
||||||
|
register("GoogleTable_GetSheetRowCount", async (_ctx, _node, args) => { const out = await googleRequest(args, `/values/${encodeURIComponent(text(args, ["sheetName"], "Sheet1"))}`); return (out.values || []).length; });
|
||||||
|
register("GoogleTable_DownloadSheets", async (_ctx, _node, args) => {
|
||||||
|
const format = /xlsx/i.test(text(args, ["format"], "csv")) ? "xlsx" : "csv";
|
||||||
|
const response = await fetch(`https://docs.google.com/spreadsheets/d/${sheetId(args)}/export?format=${format}`);
|
||||||
|
if (!response.ok) throw new RuntimeError("external_service_failed", "Google 导出失败:" + response.status);
|
||||||
|
const file = path.resolve(process.cwd(), `google-sheet-${Date.now()}.${format}`);
|
||||||
|
fs.mkdirSync(path.dirname(file), { recursive: true }); fs.writeFileSync(file, Buffer.from(await response.arrayBuffer())); return file;
|
||||||
|
});
|
||||||
|
register("GoogleTable_CopyPasteSheet", async (_ctx, _node, args) => {
|
||||||
|
const source = `${encodeURIComponent(text(args, ["sourceSheetName"], "Sheet1"))}!${text(args, ["sourceStart"], "A1")}:${text(args, ["sourceEnd"], "Z1000")}`;
|
||||||
|
const target = `${encodeURIComponent(text(args, ["destSheetName"], "Copy"))}!${text(args, ["destStart"], "A1")}:${text(args, ["destEnd"], "Z1000")}`;
|
||||||
|
const values = await googleRequest(args, `/values/${source}`);
|
||||||
|
return await googleRequest(args, `/values/${target}?valueInputOption=USER_ENTERED`, { method: "PUT", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ values: values.values || [] }) });
|
||||||
|
});
|
||||||
|
register("GoogleTable_GetColumnsByTableTitle", async (_ctx, _node, args) => {
|
||||||
|
const titles = list(args.tableTitles); const values = await googleRequest(args, `/values/${encodeURIComponent(text(args, ["sheetName"], "Sheet1"))}`);
|
||||||
|
const row = (values.values || [])[num(args, ["rowNum"], 0)] || [];
|
||||||
|
return titles.map((title) => ({ title, value: row[Number(title)] ?? "" }));
|
||||||
|
});
|
||||||
|
register("GoogleTable_ChangeSheet", async (_ctx, _node, args) => {
|
||||||
|
const out = await googleRequest(args, "?fields=sheets.properties"); const wanted = text(args, ["sheetName"], "Sheet1");
|
||||||
|
const sheet = (out.sheets || []).find((item: any) => item.properties.title === wanted); if (!sheet) throw new RuntimeError("not_found", "Google Sheet 不存在:" + wanted); return sheet.properties;
|
||||||
|
});
|
||||||
|
register("GoogleTable_RenameSheet", async (_ctx, _node, args) => {
|
||||||
|
const out = await googleRequest(args, "?fields=sheets.properties.sheetId"); const oldName = text(args, ["originalSheetName"]);
|
||||||
|
const sheet = (out.sheets || []).find((item: any) => item.properties.title === oldName); if (!sheet) throw new RuntimeError("not_found", "Google Sheet 不存在:" + oldName);
|
||||||
|
return await googleRequest(args, ":batchUpdate", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ requests: [{ updateSheet: { properties: { sheetId: sheet.properties.sheetId, title: text(args, ["modifySheetName"]) }, fields: "title" } }] }) });
|
||||||
|
});
|
||||||
|
// Image_Exists / Image_Location / Image_Wait / Image_Move 统一由 vision.ts 注册到新查找管线
|
||||||
|
// (虚拟桌面多屏拼接 + 相似度语义 + JPG 模板)。此处旧实现用 GDI CopyFromScreen 只截主屏
|
||||||
|
// 且 accuracy 语义颠倒, 已整体迁出删除。
|
||||||
|
async function captureScreen(file: string) {
|
||||||
|
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||||
|
await powershell(`Add-Type -AssemblyName System.Drawing;Add-Type -AssemblyName System.Windows.Forms;$bounds=[System.Windows.Forms.Screen]::PrimaryScreen.Bounds;$bitmap=New-Object System.Drawing.Bitmap $bounds.Width,$bounds.Height;$graphics=[System.Drawing.Graphics]::FromImage($bitmap);$graphics.CopyFromScreen($bounds.Location,[System.Drawing.Point]::Empty,$bounds.Size);$bitmap.Save(${psString(file)});$graphics.Dispose();$bitmap.Dispose()`);
|
||||||
|
}
|
||||||
|
// 图片Base64编解码: 统一组件按 mode 分流(encode=图片→Base64, decode=Base64→图片)。
|
||||||
|
// Image_Base64Decode 已隐藏进该组件,但保留注册以防旧流程直接引用旧 id。
|
||||||
|
register("Image_Base64Encode", async (_ctx, node, args) => {
|
||||||
|
const mode = text(args, ["mode"], "encode");
|
||||||
|
if (mode === "decode") {
|
||||||
|
const value = text(args, ["val", "value"]).replace(/^data:[^;]+;base64,/, "");
|
||||||
|
if (!value.trim()) throw new RuntimeError("validation_error", "Base64编码不能为空", { nodeId: node.id, componentId: node.componentId, recommendation: "填写 base64 字符串,或用 ${变量} 引用上游编码输出" });
|
||||||
|
const custom = text(args, ["output"]).trim();
|
||||||
|
const file = custom ? path.resolve(custom) : path.resolve(process.cwd(), `image-${Date.now()}.png`);
|
||||||
|
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||||
|
fs.writeFileSync(file, Buffer.from(value, "base64"));
|
||||||
|
return file;
|
||||||
|
}
|
||||||
|
const imageArg = text(args, ["path", "image"]);
|
||||||
|
if (!imageArg.trim()) throw new RuntimeError("validation_error", "图片对象不能为空", { nodeId: node.id, componentId: node.componentId, recommendation: "填写源图片路径,或用 ${变量} 引用上游截图输出" });
|
||||||
|
const source = path.resolve(imageArg);
|
||||||
|
if (!fs.existsSync(source)) throw new RuntimeError("validation_error", `找不到图片文件:${source}`, { nodeId: node.id, componentId: node.componentId, recommendation: "确认路径正确,或先用「屏幕截图」组件生成图片" });
|
||||||
|
return fs.readFileSync(source).toString("base64");
|
||||||
|
});
|
||||||
|
register("Image_Base64Decode", async (_ctx, _node, args) => {
|
||||||
|
const value = text(args, ["val", "value"]).replace(/^data:[^;]+;base64,/, ""); const file = path.resolve(process.cwd(), `image-${Date.now()}.png`);
|
||||||
|
fs.mkdirSync(path.dirname(file), { recursive: true }); fs.writeFileSync(file, Buffer.from(value, "base64")); return file;
|
||||||
|
});
|
||||||
|
register("Image_CropImg", async (_ctx, node, args) => {
|
||||||
|
const rawInput = text(args, ["file_path", "imagePath", "path"]);
|
||||||
|
if (!rawInput.trim()) throw new RuntimeError("validation_error", "图片地址不能为空", { nodeId: node.id, componentId: node.componentId, recommendation: "填写要裁剪的源图片完整路径(如 D:\\images\\shot.png),该组件是从已有图片文件里裁剪区域,不是截屏" });
|
||||||
|
const input = path.resolve(rawInput);
|
||||||
|
if (!fs.existsSync(input)) throw new RuntimeError("validation_error", `找不到图片文件:${input}`, { nodeId: node.id, componentId: node.componentId, recommendation: "确认路径正确;若要截取屏幕内容,请改用「截图图像 (Image_Screenshot)」组件" });
|
||||||
|
const output = path.resolve(text(args, ["output"], input.replace(/(\.[^.]+)$/, "-crop$1")));
|
||||||
|
const x = num(args, ["xstart"], 0), y = num(args, ["ystart"], 0), width = num(args, ["xend"], 100) - x, height = num(args, ["yend"], 100) - y;
|
||||||
|
if (width <= 0 || height <= 0) throw new RuntimeError("validation_error", `裁剪区域无效(宽 ${width} × 高 ${height})`, { nodeId: node.id, componentId: node.componentId, recommendation: "终点坐标需要大于起点坐标(单位是图片上的像素)" });
|
||||||
|
fs.mkdirSync(path.dirname(output), { recursive: true });
|
||||||
|
// 注意: .NET 方法的参数表里只能放表达式, 不能放 New-Object 语句——
|
||||||
|
// DrawImage($img, New-Object Rectangle(...), ...) 会解析报错"','后面缺少表达式"(组件此前从未跑通的原因);
|
||||||
|
// 统一改用 [Type]::new() 表达式写法
|
||||||
|
await powershell(`Add-Type -AssemblyName System.Drawing;$image=[System.Drawing.Image]::FromFile(${psString(input)});$rect=[System.Drawing.Rectangle]::new(${x},${y},${width},${height});$bitmap=[System.Drawing.Bitmap]::new($rect.Width,$rect.Height);$graphics=[System.Drawing.Graphics]::FromImage($bitmap);$graphics.DrawImage($image,[System.Drawing.Rectangle]::new(0,0,$rect.Width,$rect.Height),$rect,[System.Drawing.GraphicsUnit]::Pixel);$bitmap.Save(${psString(output)});$graphics.Dispose();$bitmap.Dispose();$image.Dispose()`);
|
||||||
|
return output;
|
||||||
|
});
|
||||||
|
register("Image_RelativePosition", async (ctx, node, args) => {
|
||||||
|
const source = path.resolve(text(args, ["source", "image"]));
|
||||||
|
const template = path.resolve(text(args, ["template", "template_path"]));
|
||||||
|
if (!source.trim() || !template.trim()) throw new RuntimeError("validation_error", "被搜索图像与查找目标都需填写图片路径", { nodeId: node.id, componentId: node.componentId });
|
||||||
|
if (!fs.existsSync(source)) throw new RuntimeError("validation_error", `找不到被搜索图像:${source}`, { nodeId: node.id, componentId: node.componentId });
|
||||||
|
if (!fs.existsSync(template)) throw new RuntimeError("validation_error", `找不到查找目标图片:${template}`, { nodeId: node.id, componentId: node.componentId });
|
||||||
|
// 兼容填反:模板比源图大时(首次查找直接抛"还大"错误)自动互换,
|
||||||
|
// 大图当被搜索图、小图当查找目标,语义仍是「小图在大图中的位置」
|
||||||
|
const accuracy = num(args, ["accuracy"], 0.2);
|
||||||
|
let hit: Awaited<ReturnType<typeof findImage>> = null;
|
||||||
|
let firstError: unknown = null;
|
||||||
|
try {
|
||||||
|
hit = await findImage(template, source, accuracy);
|
||||||
|
} catch (error) {
|
||||||
|
firstError = error;
|
||||||
|
try {
|
||||||
|
hit = await findImage(source, template, accuracy);
|
||||||
|
if (hit) ctx.log("两张图疑似填反,已自动按「小图在大图中」查找并命中", "warn", node.id);
|
||||||
|
} catch {
|
||||||
|
// 互换也失败则回落到原始方向的错误信息
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!hit) {
|
||||||
|
// 互换也没命中:两张图内容互不包含(或真的不存在),给出综合诊断而不是误导性的尺寸错误
|
||||||
|
throw new RuntimeError("not_found", "查找目标未出现在被搜索图像中(已自动尝试反向查找仍未命中,两张图内容互不包含)", {
|
||||||
|
nodeId: node.id,
|
||||||
|
componentId: node.componentId,
|
||||||
|
detail: firstError instanceof Error ? firstError.message : String(firstError ?? ""),
|
||||||
|
recommendation: "「被搜索图像」应是较大的底图(如整屏/整窗截图),「查找目标」应是底图中实际出现的小图案(按钮/图标/文字块)的截图;两张完全无关的图无法匹配"
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return hit;
|
||||||
|
});
|
||||||
|
|
||||||
|
register("logic_control_if_ai_vision", async (ctx, node, args) => {
|
||||||
|
const screenshot = path.join(ctx.evidenceDir, `ai-vision-${Date.now()}.png`); await captureScreen(screenshot);
|
||||||
|
return { image: screenshot, prompt: text(args, ["input_data", "prompt", "condition"], "描述屏幕内容并判断条件是否成立"), matched: true };
|
||||||
|
});
|
||||||
|
// CV_GetText / CV_SetText / CV_MouseHover / CV_MouseClick 统一由 cv-vision.ts 注册:
|
||||||
|
// 此处曾重复注册并覆盖 cv-vision 的实现,其中内联 here-string(@'...'@ 单行)是
|
||||||
|
// 非法 PowerShell 语法,导致 CV_MouseClick/CV_MouseHover 必然解析失败(报 CLIXML 错误)。
|
||||||
|
// 如需调整 CV 行为请修改 cv-vision.ts,不要在此重复注册。
|
||||||
|
|
||||||
|
async function xunfeiRequest(args: Record<string, unknown>, operation: string, payload: Record<string, unknown>) {
|
||||||
|
const endpoint = text(args, ["endpoint"]); if (!endpoint) throw new RuntimeError("validation_error", "需要配置讯飞 API endpoint");
|
||||||
|
const response = await fetch(endpoint, { method: "POST", headers: { "Content-Type": "application/json", "X-Appid": text(args, ["APPID", "appId"]), Authorization: "Bearer " + text(args, ["APIKey", "apiKey"]) }, body: JSON.stringify({ operation, ...payload }) });
|
||||||
|
if (!response.ok) throw new RuntimeError("external_service_failed", "讯飞 API " + response.status + ": " + await response.text()); return await response.json();
|
||||||
|
}
|
||||||
|
register("NLP_XunFei_MachineTranslation", async (_ctx, _node, args) => xunfeiRequest(args, "translation", { text: text(args, ["text"]), from: text(args, ["BeforLanguage"], "auto"), to: text(args, ["AfterLanguage"], "en") }));
|
||||||
|
register("NLP_XunFei_FeelingAnalysis", async (_ctx, _node, args) => xunfeiRequest(args, "sentiment", { text: text(args, ["text"]) }));
|
||||||
|
register("NLP_XunFei_KeywordExtraction", async (_ctx, _node, args) => xunfeiRequest(args, "keyword", { text: text(args, ["text"]) }));
|
||||||
|
|
||||||
|
async function ocrImage(args: Record<string, unknown>, node: { id: string; componentId: string }) {
|
||||||
|
const image = path.resolve(text(args, ["path", "image_url", "data", "pictureSource", "image_path", "image"]));
|
||||||
|
if (!fs.existsSync(image)) throw new RuntimeError("validation_error", "OCR 图片不存在", { nodeId: node.id, componentId: node.componentId, detail: image });
|
||||||
|
const module = await import("@gutenye/ocr-node"); const ocr = await module.default.create(); const result = await ocr.detect(image);
|
||||||
|
return (result.texts || []).map((line: any) => line.text).join("\n");
|
||||||
|
}
|
||||||
|
const ocrIds = ["OCR_MergeCardOCR", "OCR_CalculationCode", "OCR_IdCardV2", "OCR_BusinessCardV2", "OCR_VehicleInvoiceV2", "OCR_MergeOtherOCR", "OCR_BusinessLicenseV3", "OCR_PassportV2", "OCR_MergeGeneralOCR", "OCR_KeyValueIdentification", "OCR_AmountIdentification", "OCR_GeneralTextV2", "OCR_VatInvoiceV2", "OCR_DrivingLicenseV2", "OCR_AccountPageV2", "OCR_PlateLicenseV2", "OCR_BankCardV2", "OCR_BankCardV3", "OCR_IdCardV3", "OCR_VatInvoiceV3", "OCR_MergePaperOCR", "OCR_DriverLicenseV2", "OCR_RealEstateCertV2", "OCR_TrainTicketV2", "OCR_TaxiInvoiceV2", "OCR_OfficialSealV2", "OCR_MergeCarOCR", "OCR_MergeEducationOCR", "OCR_HouseCertV2", "OCR_CommonEnglishNumbersCode", "OCR_BusinessLicenseV2", "OCR_VinV2"];
|
||||||
|
for (const id of ocrIds) register(id, async (ctx, node, args) => {
|
||||||
|
const value = await ocrImage(args, node); ctx.log(id + " OCR 完成:" + value.length + " 字符", "info", node.id); return value;
|
||||||
|
});
|
||||||
@ -2,7 +2,8 @@ import os from "node:os";
|
|||||||
import path from "node:path";
|
import path from "node:path";
|
||||||
import { register } from "../engine/registry";
|
import { register } from "../engine/registry";
|
||||||
import { RuntimeError, type CapabilityHandler } from "../engine/types";
|
import { RuntimeError, type CapabilityHandler } from "../engine/types";
|
||||||
import { powershell, psString } from "./desktop";
|
import { powershell, psString, psText } from "../driver/windows/powershell";
|
||||||
|
import { defaultEngine } from "../engine/engine";
|
||||||
|
|
||||||
function value(args: Record<string, unknown>, keys: string[], fallback = ""): string {
|
function value(args: Record<string, unknown>, keys: string[], fallback = ""): string {
|
||||||
for (const key of keys) {
|
for (const key of keys) {
|
||||||
@ -31,20 +32,8 @@ function selector(args: Record<string, unknown>): string {
|
|||||||
return String(raw ?? "");
|
return String(raw ?? "");
|
||||||
}
|
}
|
||||||
|
|
||||||
const WINDOW_API = `Add-Type @'
|
// 窗口矩形/状态操作统一走驱动(任务 3.3)
|
||||||
using System;
|
const getDriver = () => defaultEngine.driver;
|
||||||
using System.Runtime.InteropServices;
|
|
||||||
public static class RpaWindowApi {
|
|
||||||
[StructLayout(LayoutKind.Sequential)] public struct RECT { public int Left; public int Top; public int Right; public int Bottom; }
|
|
||||||
[DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr hWnd, out RECT rect);
|
|
||||||
[DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr hWnd, int command);
|
|
||||||
[DllImport("user32.dll")] public static extern bool SetWindowPos(IntPtr hWnd, IntPtr after, int x, int y, int width, int height, uint flags);
|
|
||||||
}
|
|
||||||
'@`;
|
|
||||||
|
|
||||||
function findWindowScript(title: string): string {
|
|
||||||
return `$p = Get-Process | Where-Object { $_.MainWindowHandle -ne 0 -and $_.MainWindowTitle -like ${psString(`*${title}*`)} } | Select-Object -First 1; if (-not $p) { throw '找不到窗口:${title.replace(/'/g, "''")}' }`;
|
|
||||||
}
|
|
||||||
|
|
||||||
register("System_GetEnv", (_ctx, node, args) => {
|
register("System_GetEnv", (_ctx, node, args) => {
|
||||||
const name = required(args, ["name"], "环境变量名称", node.id, node.componentId);
|
const name = required(args, ["name"], "环境变量名称", node.id, node.componentId);
|
||||||
@ -105,7 +94,7 @@ register(["App_Kill", "App_KillV1"], async (_ctx, node, args) => {
|
|||||||
register("Clipboard_GetTextV1", () => powershell("Get-Clipboard -Raw"));
|
register("Clipboard_GetTextV1", () => powershell("Get-Clipboard -Raw"));
|
||||||
register("Clipboard_SetTextV1", async (_ctx, _node, args) => {
|
register("Clipboard_SetTextV1", async (_ctx, _node, args) => {
|
||||||
const content = value(args, ["text", "value"]);
|
const content = value(args, ["text", "value"]);
|
||||||
await powershell(`Set-Clipboard -Value ${psString(content)}`);
|
await powershell(`Set-Clipboard -Value ${psText(content)}`);
|
||||||
return content;
|
return content;
|
||||||
});
|
});
|
||||||
register("Clipboard_ClearClipboard", async () => {
|
register("Clipboard_ClearClipboard", async () => {
|
||||||
@ -115,15 +104,14 @@ register("Clipboard_ClearClipboard", async () => {
|
|||||||
|
|
||||||
register("Window_GetWindowRect", async (_ctx, node, args) => {
|
register("Window_GetWindowRect", async (_ctx, node, args) => {
|
||||||
const title = required({ title: selector(args) }, ["title"], "窗口标题", node.id, node.componentId);
|
const title = required({ title: selector(args) }, ["title"], "窗口标题", node.id, node.componentId);
|
||||||
const output = await powershell(`${WINDOW_API}; ${findWindowScript(title)}; $r = New-Object RpaWindowApi+RECT; [RpaWindowApi]::GetWindowRect($p.MainWindowHandle,[ref]$r) | Out-Null; @{x=$r.Left;y=$r.Top;width=$r.Right-$r.Left;height=$r.Bottom-$r.Top;title=$p.MainWindowTitle;pid=$p.Id} | ConvertTo-Json -Compress`);
|
return getDriver().window.findByTitle(title);
|
||||||
return JSON.parse(output);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
register(["Window_SetWindowState", "Window_SetWindowState_v1"], async (_ctx, node, args) => {
|
register(["Window_SetWindowState", "Window_SetWindowState_v1"], async (_ctx, node, args) => {
|
||||||
const title = required({ title: selector(args) }, ["title"], "窗口标题", node.id, node.componentId);
|
const title = required({ title: selector(args) }, ["title"], "窗口标题", node.id, node.componentId);
|
||||||
const state = value(args, ["WindowType", "switchType"], "maximize").toLowerCase();
|
const state = value(args, ["WindowType", "switchType"], "maximize").toLowerCase();
|
||||||
const command = state.includes("min") ? 6 : state.includes("max") ? 3 : 9;
|
const command = state.includes("min") ? 6 : state.includes("max") ? 3 : 9;
|
||||||
await powershell(`${WINDOW_API}; ${findWindowScript(title)}; [RpaWindowApi]::ShowWindow($p.MainWindowHandle,${command}) | Out-Null`);
|
await getDriver().window.setState(title, command);
|
||||||
return state;
|
return state;
|
||||||
});
|
});
|
||||||
|
|
||||||
@ -136,6 +124,6 @@ register("Window_SizeWindow_v1", async (_ctx, node, args) => {
|
|||||||
}
|
}
|
||||||
const [width, height] = Array.isArray(size) ? size.map(Number) : [0, 0];
|
const [width, height] = Array.isArray(size) ? size.map(Number) : [0, 0];
|
||||||
if (!(width > 0 && height > 0)) throw new RuntimeError("validation_error", "窗口尺寸无效", { nodeId: node.id, componentId: node.componentId });
|
if (!(width > 0 && height > 0)) throw new RuntimeError("validation_error", "窗口尺寸无效", { nodeId: node.id, componentId: node.componentId });
|
||||||
await powershell(`${WINDOW_API}; ${findWindowScript(title)}; $r = New-Object RpaWindowApi+RECT; [RpaWindowApi]::GetWindowRect($p.MainWindowHandle,[ref]$r) | Out-Null; [RpaWindowApi]::SetWindowPos($p.MainWindowHandle,[IntPtr]::Zero,$r.Left,$r.Top,${width},${height},4) | Out-Null`);
|
await getDriver().window.setSize(title, width, height);
|
||||||
return { width, height };
|
return { width, height };
|
||||||
});
|
});
|
||||||
@ -4,6 +4,7 @@ import path from "node:path";
|
|||||||
import { promisify } from "node:util";
|
import { promisify } from "node:util";
|
||||||
import { register } from "../engine/registry";
|
import { register } from "../engine/registry";
|
||||||
import { RuntimeError } from "../engine/types";
|
import { RuntimeError } from "../engine/types";
|
||||||
|
import { isWindows } from "../driver/platform";
|
||||||
|
|
||||||
type Table = unknown[][];
|
type Table = unknown[][];
|
||||||
|
|
||||||
@ -257,7 +258,7 @@ register("System_ExecuteCommandV1", async (_ctx, node, args) => {
|
|||||||
const timeout = Math.max(1_000, Number(args.timeoutMs || 120_000));
|
const timeout = Math.max(1_000, Number(args.timeoutMs || 120_000));
|
||||||
let executable: string;
|
let executable: string;
|
||||||
let commandArgs: string[];
|
let commandArgs: string[];
|
||||||
if (process.platform === "win32") {
|
if (isWindows) {
|
||||||
if (mode === "2") {
|
if (mode === "2") {
|
||||||
const file = path.resolve(requiredText(args.bat_file, "BAT文件", node.id, node.componentId));
|
const file = path.resolve(requiredText(args.bat_file, "BAT文件", node.id, node.componentId));
|
||||||
await fs.access(file);
|
await fs.access(file);
|
||||||
@ -280,7 +281,7 @@ register("System_ExecuteCommandV1", async (_ctx, node, args) => {
|
|||||||
commandArgs = ["-lc", command];
|
commandArgs = ["-lc", command];
|
||||||
}
|
}
|
||||||
// cmd.exe 自己解析 /c 后的命令;不能让 Node 再把 BAT 路径引号转成反斜杠引号。
|
// cmd.exe 自己解析 /c 后的命令;不能让 Node 再把 BAT 路径引号转成反斜杠引号。
|
||||||
const result = await execFileAsync(executable, commandArgs, { cwd, timeout, windowsHide: true, windowsVerbatimArguments: process.platform === "win32" && mode !== "3", maxBuffer: 10 * 1024 * 1024 });
|
const result = await execFileAsync(executable, commandArgs, { cwd, timeout, windowsHide: true, windowsVerbatimArguments: isWindows && mode !== "3", maxBuffer: 10 * 1024 * 1024 });
|
||||||
return bool(args.get_ret, true) ? String(result.stdout).trimEnd() : true;
|
return bool(args.get_ret, true) ? String(result.stdout).trimEnd() : true;
|
||||||
});
|
});
|
||||||
|
|
||||||
550
packages/rpa-core/src/capabilities/vision.ts
Normal file
550
packages/rpa-core/src/capabilities/vision.ts
Normal file
@ -0,0 +1,550 @@
|
|||||||
|
import path from "node:path";
|
||||||
|
import { promises as fs } from "node:fs";
|
||||||
|
import { PNG } from "pngjs";
|
||||||
|
import { register } from "../engine/registry";
|
||||||
|
import { RuntimeError } from "../engine/types";
|
||||||
|
import { requireScreenService, type HostDisplay } from "../engine/host-bridge";
|
||||||
|
import { powershell, psString } from "../driver/windows/powershell";
|
||||||
|
|
||||||
|
function text(args: Record<string, unknown>, keys: string[], fallback = ""): string {
|
||||||
|
for (const key of keys) if (args[key] !== undefined && args[key] !== null && args[key] !== "") return String(args[key]);
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 全虚拟桌面(所有显示器拼接)物理分辨率截图。
|
||||||
|
* desktopCapturer thumbnailSize 乘 scaleFactor 与「屏幕框选」模板(物理像素裁剪)尺度一致;
|
||||||
|
* 逐屏截图后按各自物理原点拼到一张画布上——目标在副屏也能搜到。
|
||||||
|
* 返回 originX/originY = 拼接画布左上角对应的屏幕物理坐标(副屏在主屏左侧时为负),用于结果坐标平移。 */
|
||||||
|
async function captureVirtualScreenPng(): Promise<{ png: PNG; originX: number; originY: number }> {
|
||||||
|
const screenHost = requireScreenService();
|
||||||
|
const displays = screenHost.listDisplays();
|
||||||
|
if (displays.length === 0) throw new RuntimeError("desktop_error", "无法获取屏幕信息");
|
||||||
|
// 每屏按自己的物理分辨率截图(宿主按 displayId 对位 → 按物理原点排序对位)
|
||||||
|
const captures: { display: HostDisplay; origin: { x: number; y: number }; png: PNG }[] = [];
|
||||||
|
for (const display of displays) {
|
||||||
|
const scale = display.scaleFactor || 1;
|
||||||
|
const wantW = Math.round(display.bounds.width * scale);
|
||||||
|
const wantH = Math.round(display.bounds.height * scale);
|
||||||
|
const pngBytes = await screenHost.captureDisplay(display.id, wantW, wantH);
|
||||||
|
if (!pngBytes?.length) throw new RuntimeError("desktop_error", "无法获取屏幕截图");
|
||||||
|
// 显示器 DIP 原点的物理坐标 = 该屏物理原点(屏内线性映射: 物理 = (DIP-原点)*缩放 + 物理原点)
|
||||||
|
const origin = screenHost.dipToScreenPoint({ x: display.bounds.x, y: display.bounds.y });
|
||||||
|
captures.push({ display, origin, png: PNG.sync.read(Buffer.from(pngBytes)) });
|
||||||
|
}
|
||||||
|
const originX = Math.min(...captures.map(c => c.origin.x));
|
||||||
|
const originY = Math.min(...captures.map(c => c.origin.y));
|
||||||
|
const endX = Math.max(...captures.map(c => c.origin.x + c.png.width));
|
||||||
|
const endY = Math.max(...captures.map(c => c.origin.y + c.png.height));
|
||||||
|
const width = endX - originX;
|
||||||
|
const height = endY - originY;
|
||||||
|
const { createCanvas } = await import("@napi-rs/canvas");
|
||||||
|
const canvas = createCanvas(width, height);
|
||||||
|
const context = canvas.getContext("2d");
|
||||||
|
for (const capture of captures) {
|
||||||
|
const sourceCanvas = createCanvas(capture.png.width, capture.png.height);
|
||||||
|
const sourceContext = sourceCanvas.getContext("2d");
|
||||||
|
const imageData = sourceContext.createImageData(capture.png.width, capture.png.height);
|
||||||
|
imageData.data.set(capture.png.data);
|
||||||
|
sourceContext.putImageData(imageData, 0, 0);
|
||||||
|
context.drawImage(sourceCanvas, capture.origin.x - originX, capture.origin.y - originY);
|
||||||
|
}
|
||||||
|
const stitched = context.getImageData(0, 0, width, height);
|
||||||
|
const png = new PNG({ width, height });
|
||||||
|
png.data = Buffer.from(stitched.data);
|
||||||
|
return { png, originX, originY };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** DIP 区域 → 物理像素区域:左上角经 dipToScreenPoint 换算(天然支持副屏/负坐标),宽高乘所在屏缩放比 */
|
||||||
|
async function dipRegionToPhysical(region: { x: number; y: number; width: number; height: number }): Promise<{ x: number; y: number; width: number; height: number }> {
|
||||||
|
const screenHost = requireScreenService();
|
||||||
|
const topLeft = screenHost.dipToScreenPoint({ x: Math.round(region.x), y: Math.round(region.y) });
|
||||||
|
const display = screenHost.displayMatching({
|
||||||
|
x: region.x, y: region.y,
|
||||||
|
width: Math.max(1, Math.round(region.width)), height: Math.max(1, Math.round(region.height))
|
||||||
|
});
|
||||||
|
const scale = display.scaleFactor || 1;
|
||||||
|
return {
|
||||||
|
x: topLeft.x,
|
||||||
|
y: topLeft.y,
|
||||||
|
width: Math.max(1, Math.round(region.width * scale)),
|
||||||
|
height: Math.max(1, Math.round(region.height * scale))
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 按魔数判断是否 PNG(89 50 4E 47) */
|
||||||
|
function isPngBuffer(buffer: Buffer): boolean {
|
||||||
|
return buffer.length > 8 && buffer[0] === 0x89 && buffer[1] === 0x50 && buffer[2] === 0x4e && buffer[3] === 0x47;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 解码用户提供的图片文件为 pngjs PNG。pngjs 只认 PNG, 模板/输入图常是 JPG——
|
||||||
|
* 非 PNG 时走 @napi-rs/canvas 解码(JPG/BMP/WebP 等), 否则报 "unrecognised content at end of stream" */
|
||||||
|
async function decodeImageFile(filePath: string): Promise<PNG> {
|
||||||
|
let buffer: Buffer;
|
||||||
|
try {
|
||||||
|
buffer = await fs.readFile(filePath);
|
||||||
|
} catch (error) {
|
||||||
|
if ((error as NodeJS.ErrnoException)?.code === "ENOENT") {
|
||||||
|
throw new RuntimeError("validation_error", `找不到图片文件:${filePath}`, {
|
||||||
|
recommendation: "确认路径正确,或直接点参数旁的「屏幕框选」截取一张"
|
||||||
|
});
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
if (isPngBuffer(buffer)) return PNG.sync.read(buffer);
|
||||||
|
const { loadImage, createCanvas } = await import("@napi-rs/canvas");
|
||||||
|
const image = await loadImage(buffer);
|
||||||
|
if (!image.width || !image.height) {
|
||||||
|
throw new RuntimeError("validation_error", "图片解码失败", { detail: filePath });
|
||||||
|
}
|
||||||
|
const canvas = createCanvas(image.width, image.height);
|
||||||
|
const context = canvas.getContext("2d");
|
||||||
|
context.drawImage(image, 0, 0);
|
||||||
|
const imageData = context.getImageData(0, 0, image.width, image.height);
|
||||||
|
const png = new PNG({ width: image.width, height: image.height });
|
||||||
|
png.data = Buffer.from(imageData.data);
|
||||||
|
return png;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 解析区域 JSON:对象 {x,y,width,height}(兼容 left/top、x1y1x2y2 角点)或数组 [x,y,width,height],并夹取到图片边界内 */
|
||||||
|
export function parseRegion(raw: string, image: PNG): { x: number; y: number; width: number; height: number } {
|
||||||
|
let parsed: unknown;
|
||||||
|
try {
|
||||||
|
parsed = JSON.parse(raw);
|
||||||
|
} catch {
|
||||||
|
throw new RuntimeError("validation_error", "区域 JSON 无法解析", { detail: raw.slice(0, 200) });
|
||||||
|
}
|
||||||
|
const numberOf = (value: unknown): number | null => {
|
||||||
|
const n = Number(value);
|
||||||
|
return Number.isFinite(n) ? n : null;
|
||||||
|
};
|
||||||
|
if (Array.isArray(parsed) && parsed.length === 4) {
|
||||||
|
const [x, y, width, height] = parsed.map(numberOf);
|
||||||
|
if (x !== null && y !== null && width !== null && height !== null) {
|
||||||
|
return clampRegion({ x, y, width, height }, image);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (parsed && typeof parsed === "object") {
|
||||||
|
const record = parsed as Record<string, unknown>;
|
||||||
|
const x = numberOf(record.x ?? record.left);
|
||||||
|
const y = numberOf(record.y ?? record.top);
|
||||||
|
const width = numberOf(record.width);
|
||||||
|
const height = numberOf(record.height);
|
||||||
|
if (x !== null && y !== null && width !== null && height !== null) {
|
||||||
|
return clampRegion({ x, y, width, height }, image);
|
||||||
|
}
|
||||||
|
const x2 = numberOf(record.x2 ?? record.right);
|
||||||
|
const y2 = numberOf(record.y2 ?? record.bottom);
|
||||||
|
if (x !== null && y !== null && x2 !== null && y2 !== null) {
|
||||||
|
return clampRegion({ x, y, width: x2 - x, height: y2 - y }, image);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw new RuntimeError("validation_error", "区域 JSON 不符合要求", {
|
||||||
|
detail: raw.slice(0, 200),
|
||||||
|
recommendation: '使用 {"x":0,"y":0,"width":100,"height":80} 或 [x, y, width, height]'
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function clampRegion(region: { x: number; y: number; width: number; height: number }, image: PNG): { x: number; y: number; width: number; height: number } {
|
||||||
|
const x = Math.max(0, Math.min(Math.round(region.x), image.width - 1));
|
||||||
|
const y = Math.max(0, Math.min(Math.round(region.y), image.height - 1));
|
||||||
|
const width = Math.max(1, Math.min(Math.round(region.width), image.width - x));
|
||||||
|
const height = Math.max(1, Math.min(Math.round(region.height), image.height - y));
|
||||||
|
return { x, y, width, height };
|
||||||
|
}
|
||||||
|
|
||||||
|
export function cropPng(source: PNG, region: { x: number; y: number; width: number; height: number }): PNG {
|
||||||
|
const cropped = new PNG({ width: region.width, height: region.height });
|
||||||
|
PNG.bitblt(source, cropped, region.x, region.y, region.width, region.height, 0, 0);
|
||||||
|
return cropped;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 目标路径容错:填的是文件夹(已存在或以分隔符结尾)时,自动往里存默认文件名 */
|
||||||
|
async function resolveOutputPath(outputPath: string, defaultName: string): Promise<string> {
|
||||||
|
if (/[\\/]$/.test(outputPath)) return path.join(outputPath, defaultName);
|
||||||
|
try {
|
||||||
|
const stat = await fs.stat(outputPath);
|
||||||
|
if (stat.isDirectory()) return path.join(outputPath, defaultName);
|
||||||
|
} catch {
|
||||||
|
// 路径不存在说明是要新建的文件,原样返回
|
||||||
|
}
|
||||||
|
return outputPath;
|
||||||
|
}
|
||||||
|
|
||||||
|
register("Image_Screenshot", async (ctx, node, args) => {
|
||||||
|
const folder = text(args, ["outImagePath", "folder"]);
|
||||||
|
const name = text(args, ["outImageName", "name"], `screenshot_${Date.now()}.png`);
|
||||||
|
const imageArg = text(args, ["image_path"]);
|
||||||
|
const source = text(args, ["pictureSource"], "1");
|
||||||
|
const range = text(args, ["shotRange"], "1");
|
||||||
|
|
||||||
|
let image: PNG;
|
||||||
|
let output: string;
|
||||||
|
let screenOrigin: { x: number; y: number } | null = null;
|
||||||
|
if (source === "2") {
|
||||||
|
// 指定图片文件来源:image_path 为输入图片;输出走 输出目录+文件名,两者均未配置时直接返回源文件
|
||||||
|
if (!imageArg.trim()) {
|
||||||
|
throw new RuntimeError("validation_error", "图像来源为「指定图片文件」时,截图保存路径需填写源图片路径", {
|
||||||
|
recommendation: "在截图保存路径选择一张已存在的图片,或将图像来源改回「当前屏幕」"
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const sourceFile = path.resolve(imageArg);
|
||||||
|
image = await decodeImageFile(sourceFile);
|
||||||
|
output = folder ? path.resolve(path.join(folder, name)) : sourceFile;
|
||||||
|
} else {
|
||||||
|
// 当前屏幕来源:image_path 即保存路径(文件夹则自动追加默认文件名),缺省用 输出目录+文件名;再缺省落流程证据目录(路径可预期)
|
||||||
|
// 截取整个虚拟桌面(含副屏),物理分辨率
|
||||||
|
const shot = await captureVirtualScreenPng();
|
||||||
|
image = shot.png;
|
||||||
|
screenOrigin = { x: shot.originX, y: shot.originY };
|
||||||
|
output = await resolveOutputPath(
|
||||||
|
path.resolve(imageArg || (folder ? path.join(folder, name) : path.join(ctx.evidenceDir, name))),
|
||||||
|
name
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (range === "2") {
|
||||||
|
const regionRaw = text(args, ["position", "action", "position_img"]);
|
||||||
|
if (!regionRaw.trim()) {
|
||||||
|
throw new RuntimeError("validation_error", "截图范围为「框选区域」时,截图区域需填写区域 JSON", {
|
||||||
|
recommendation: '在「截图区域」点屏幕框选,或填写 {"x":0,"y":0,"width":100,"height":80},或将截图范围改回「整个屏幕」'
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (screenOrigin) {
|
||||||
|
// 屏幕来源: 区域 JSON 是 DIP 坐标(支持副屏),换算物理坐标后减去拼接画布原点再裁剪
|
||||||
|
const phys = await dipRegionToPhysical(parseRegion(regionRaw, image));
|
||||||
|
image = cropPng(image, clampRegion({
|
||||||
|
x: phys.x - screenOrigin.x,
|
||||||
|
y: phys.y - screenOrigin.y,
|
||||||
|
width: phys.width,
|
||||||
|
height: phys.height
|
||||||
|
}, image));
|
||||||
|
} else {
|
||||||
|
// 图片文件来源: 区域按该图片自身像素坐标
|
||||||
|
image = cropPng(image, parseRegion(regionRaw, image));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await fs.mkdir(path.dirname(output), { recursive: true });
|
||||||
|
await fs.writeFile(output, PNG.sync.write(image));
|
||||||
|
return output;
|
||||||
|
});
|
||||||
|
|
||||||
|
interface MatchResult { x: number; y: number; width: number; height: number; score: number; }
|
||||||
|
|
||||||
|
interface WindowRect { x: number; y: number; width: number; height: number }
|
||||||
|
|
||||||
|
/** 按标题(精确→包含)或进程名查找窗口,前置并返回窗口矩形(物理屏幕坐标) */
|
||||||
|
async function focusWindowRect(windowObj: string): Promise<WindowRect> {
|
||||||
|
const quoted = psString(windowObj);
|
||||||
|
const stdout = await powershell(`Add-Type @'
|
||||||
|
using System; using System.Runtime.InteropServices;
|
||||||
|
public static class WinRect {
|
||||||
|
[DllImport("user32.dll")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value);
|
||||||
|
[DllImport("user32.dll")] public static extern IntPtr FindWindow(string cls, string title);
|
||||||
|
[DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr h);
|
||||||
|
[DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr h, int cmd);
|
||||||
|
[DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr h, out RECT r);
|
||||||
|
public struct RECT { public int Left; public int Top; public int Right; public int Bottom; }
|
||||||
|
}
|
||||||
|
'@;
|
||||||
|
# 不声明 DPI 感知时 GetWindowRect 返回虚拟化坐标, 与物理分辨率截图对不上
|
||||||
|
try { [void][WinRect]::SetProcessDpiAwarenessContext([IntPtr](-4)) } catch {};
|
||||||
|
$h = [WinRect]::FindWindow($null, ${quoted});
|
||||||
|
if ($h -eq [IntPtr]::Zero) { $p = Get-Process | Where-Object { $_.MainWindowTitle -and ($_.MainWindowTitle -eq ${quoted} -or $_.MainWindowTitle.Contains(${quoted})) } | Select-Object -First 1; if ($p) { $h = $p.MainWindowHandle } }
|
||||||
|
if ($h -eq [IntPtr]::Zero) { Write-Output 'NOT_FOUND'; exit }
|
||||||
|
[WinRect]::ShowWindow($h, 9) | Out-Null;
|
||||||
|
[WinRect]::SetForegroundWindow($h) | Out-Null;
|
||||||
|
Start-Sleep -Milliseconds 300;
|
||||||
|
$r = New-Object WinRect+RECT;
|
||||||
|
[WinRect]::GetWindowRect($h, [ref]$r) | Out-Null;
|
||||||
|
Write-Output ("{0},{1},{2},{3}" -f $r.Left, $r.Top, $r.Right, $r.Bottom)`);
|
||||||
|
const line = stdout.trim().split(/\r?\n/).pop() ?? "";
|
||||||
|
if (line === "NOT_FOUND") {
|
||||||
|
throw new RuntimeError("not_found", `未找到窗口「${windowObj}」`, { recommendation: "确认窗口已打开,窗口对象可填完整标题、标题包含关键字或进程名" });
|
||||||
|
}
|
||||||
|
const parts = line.split(",").map(Number);
|
||||||
|
if (parts.length !== 4 || parts.some((v) => !Number.isFinite(v))) {
|
||||||
|
throw new RuntimeError("desktop_error", "窗口矩形获取失败", { detail: line.slice(0, 120) });
|
||||||
|
}
|
||||||
|
const [left, top, right, bottom] = parts;
|
||||||
|
return { x: left, y: top, width: Math.max(1, right - left), height: Math.max(1, bottom - top) };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 单通道噪声死区:JPG 压缩伪影/抗锯齿带来 ±10 级内的像素波动,不应计入差异 */
|
||||||
|
const NOISE_FLOOR = 10;
|
||||||
|
|
||||||
|
function pixelDifference(screen: PNG, template: PNG, offsetX: number, offsetY: number, sampleStep: number, cutoff: number): number {
|
||||||
|
let difference = 0;
|
||||||
|
let samples = 0;
|
||||||
|
for (let y = 0; y < template.height; y += sampleStep) {
|
||||||
|
for (let x = 0; x < template.width; x += sampleStep) {
|
||||||
|
const sourceIndex = ((offsetY + y) * screen.width + offsetX + x) * 4;
|
||||||
|
const templateIndex = (y * template.width + x) * 4;
|
||||||
|
for (let c = 0; c < 3; c++) {
|
||||||
|
const d = Math.abs(screen.data[sourceIndex + c] - template.data[templateIndex + c]);
|
||||||
|
difference += d <= NOISE_FLOOR ? 0 : d - NOISE_FLOOR;
|
||||||
|
}
|
||||||
|
samples += 3;
|
||||||
|
// cutoff 是 0~1 的归一化差异上限; difference 是通道差累加值(0~255 每通道),
|
||||||
|
// 提前退出必须按 255 折算——之前漏乘导致几乎所有位置在第一个像素就被剪掉, 得分全乱
|
||||||
|
if (difference > cutoff * samples * 255) return difference / (samples * 255);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return samples ? difference / (samples * 255) : 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 在屏幕图里扫描模板的最佳位置(粗扫网格 + 精化) */
|
||||||
|
function scanBest(screen: PNG, template: PNG, maximumDifference: number): MatchResult | null {
|
||||||
|
const sampleStep = Math.max(1, Math.floor(Math.min(template.width, template.height) / 18));
|
||||||
|
const scanStep = Math.max(1, Math.floor(Math.min(template.width, template.height) / 12));
|
||||||
|
let best: MatchResult | null = null;
|
||||||
|
for (let y = 0; y <= screen.height - template.height; y += scanStep) {
|
||||||
|
for (let x = 0; x <= screen.width - template.width; x += scanStep) {
|
||||||
|
// 尚无 best 时不剪枝(完整评估): 若初始就按 maximumDifference 剪枝,
|
||||||
|
// 均匀背景位置在少数采样后返回"刚好压线"的部分均值抢占 best,
|
||||||
|
// 而真位置(常不在网格上)附近的网格点因部分重叠被剪, 精化永远够不到——
|
||||||
|
// 相似度 0.95 时实测 score=1.0 的真位置被判"未达到阈值"
|
||||||
|
const cutoff = best ? 1 - best.score : 1;
|
||||||
|
const difference = pixelDifference(screen, template, x, y, sampleStep, cutoff);
|
||||||
|
const score = 1 - difference;
|
||||||
|
if (!best || score > best.score) best = { x, y, width: template.width, height: template.height, score };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!best) return null;
|
||||||
|
// 注意: 阈值判定必须放在精化之后——粗扫网格(步长=模板短边/12)通常够不到精确位置,
|
||||||
|
// 粗扫 best 常差在阈值边缘, 精化后才能提到满分; 先判阈值会把真命中误杀
|
||||||
|
const refine = Math.max(1, scanStep);
|
||||||
|
const minX = Math.max(0, best.x - refine);
|
||||||
|
const maxX = Math.min(screen.width - template.width, best.x + refine);
|
||||||
|
const minY = Math.max(0, best.y - refine);
|
||||||
|
const maxY = Math.min(screen.height - template.height, best.y + refine);
|
||||||
|
for (let y = minY; y <= maxY; y += 1) {
|
||||||
|
for (let x = minX; x <= maxX; x += 1) {
|
||||||
|
// 精化区域只有几百个位置, 完整评估避免同样的剪枝偏置
|
||||||
|
const difference = pixelDifference(screen, template, x, y, sampleStep, 1);
|
||||||
|
const score = 1 - difference;
|
||||||
|
if (score > best.score) best = { x, y, width: template.width, height: template.height, score };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// 阈值判定: 精化后的最终得分才与 maximumDifference 比较
|
||||||
|
if (1 - best.score > maximumDifference) return null;
|
||||||
|
return best;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 用 canvas 双线性缩放模板(匹配尺度兜底用) */
|
||||||
|
async function resizeTemplate(template: PNG, scale: number): Promise<PNG> {
|
||||||
|
const { createCanvas } = await import("@napi-rs/canvas");
|
||||||
|
const width = Math.max(1, Math.round(template.width * scale));
|
||||||
|
const height = Math.max(1, Math.round(template.height * scale));
|
||||||
|
const source = createCanvas(template.width, template.height);
|
||||||
|
const sourceContext = source.getContext("2d");
|
||||||
|
const imageData = sourceContext.createImageData(template.width, template.height);
|
||||||
|
imageData.data.set(template.data);
|
||||||
|
sourceContext.putImageData(imageData, 0, 0);
|
||||||
|
const target = createCanvas(width, height);
|
||||||
|
const targetContext = target.getContext("2d");
|
||||||
|
targetContext.imageSmoothingEnabled = true;
|
||||||
|
targetContext.imageSmoothingQuality = "high";
|
||||||
|
targetContext.drawImage(source, 0, 0, width, height);
|
||||||
|
const scaled = targetContext.getImageData(0, 0, width, height);
|
||||||
|
const png = new PNG({ width, height });
|
||||||
|
png.data = Buffer.from(scaled.data);
|
||||||
|
return png;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 模板匹配。extraScales 提供尺度兜底:截图与模板的获取渠道分辨率不一致时
|
||||||
|
* (如旧版 DIP 截图存下的模板 vs 现在的物理分辨率截图),按给定缩放比重试。
|
||||||
|
*/
|
||||||
|
export async function findImage(templatePath: string, screenshotPath: string, accuracy: number, extraScales: number[] = []): Promise<MatchResult | null> {
|
||||||
|
const [screenBuffer, templateBuffer] = await Promise.all([fs.readFile(screenshotPath), fs.readFile(templatePath)]);
|
||||||
|
const screen = PNG.sync.read(screenBuffer);
|
||||||
|
// 模板图可能是 JPG 等非 PNG 格式, 按魔数选择解码器
|
||||||
|
const template = isPngBuffer(templateBuffer) ? PNG.sync.read(templateBuffer) : await decodeImageFile(templatePath);
|
||||||
|
if (template.width > screen.width || template.height > screen.height) {
|
||||||
|
throw new RuntimeError("validation_error", `模板图片(${template.width}×${template.height})比截图画面(${screen.width}×${screen.height})还大,无法查找`, {
|
||||||
|
recommendation: "模板应是屏幕上要寻找的小图案(按钮、图标、文字块),不要用整屏大图"
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const maximumDifference = Math.max(0.01, Math.min(0.95, accuracy || 0.2));
|
||||||
|
const best = scanBest(screen, template, maximumDifference);
|
||||||
|
if (best) return best;
|
||||||
|
for (const scale of extraScales) {
|
||||||
|
if (!Number.isFinite(scale) || scale === 1) continue;
|
||||||
|
const scaled = await resizeTemplate(template, scale);
|
||||||
|
if (scaled.width > screen.width || scaled.height > screen.height) continue;
|
||||||
|
const scaledBest = scanBest(screen, scaled, maximumDifference);
|
||||||
|
if (scaledBest) return scaledBest;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function locate(args: Record<string, unknown>, evidenceDir: string, nodeId: string): Promise<MatchResult> {
|
||||||
|
const rawTemplate = text(args, ["image", "image_path", "template"]);
|
||||||
|
if (!rawTemplate.trim()) throw new RuntimeError("validation_error", "模板图片路径为空", { nodeId });
|
||||||
|
const template = path.resolve(rawTemplate);
|
||||||
|
|
||||||
|
// 查找范围:1 整个虚拟桌面(默认,含副屏)/ 2 指定窗口 / 3 指定区域;
|
||||||
|
// 截图统一为物理分辨率拼接画布, 结果坐标加回画布原点即屏幕物理坐标(可直接用于点击)
|
||||||
|
const rangeMode = text(args, ["find_range"], "1");
|
||||||
|
const screenshot = path.join(evidenceDir, `${nodeId}_screen.png`);
|
||||||
|
const shot = await captureVirtualScreenPng();
|
||||||
|
let offsetX = 0;
|
||||||
|
let offsetY = 0;
|
||||||
|
if (rangeMode === "2") {
|
||||||
|
const windowObj = text(args, ["window_obj"]);
|
||||||
|
if (!windowObj.trim()) throw new RuntimeError("validation_error", "查找范围为「指定窗口」时,窗口对象不能为空", { nodeId, recommendation: "填写窗口标题(或标题关键字、进程名)" });
|
||||||
|
const rect = await focusWindowRect(windowObj);
|
||||||
|
const cropRect = clampRegion({ x: rect.x - shot.originX, y: rect.y - shot.originY, width: rect.width, height: rect.height }, shot.png);
|
||||||
|
await fs.writeFile(screenshot, PNG.sync.write(cropPng(shot.png, cropRect)));
|
||||||
|
offsetX = rect.x;
|
||||||
|
offsetY = rect.y;
|
||||||
|
} else if (rangeMode === "3" || rangeMode === "4") {
|
||||||
|
const regionRaw = text(args, ["shot_position", "action", "position"]);
|
||||||
|
if (!regionRaw.trim()) throw new RuntimeError("validation_error", "查找范围为「指定区域」时,需要填写区域 JSON", { nodeId, recommendation: '在 shot_position 填写 {"x":0,"y":0,"width":100,"height":80} 或 [x, y, width, height]' });
|
||||||
|
// 区域 JSON 是 DIP 坐标(框选工具按 DIP 返回, 支持副屏负坐标),换算物理坐标后减画布原点裁剪
|
||||||
|
const phys = await dipRegionToPhysical(parseRegion(regionRaw, shot.png));
|
||||||
|
const cropRect = clampRegion({ x: phys.x - shot.originX, y: phys.y - shot.originY, width: phys.width, height: phys.height }, shot.png);
|
||||||
|
await fs.writeFile(screenshot, PNG.sync.write(cropPng(shot.png, cropRect)));
|
||||||
|
offsetX = phys.x;
|
||||||
|
offsetY = phys.y;
|
||||||
|
} else {
|
||||||
|
await fs.writeFile(screenshot, PNG.sync.write(shot.png));
|
||||||
|
// 拼接画布坐标 → 屏幕物理坐标必须加回画布原点(副屏在主屏左侧/上方时原点为负);
|
||||||
|
// 漏加会导致点击偏移(实测副屏上移 177px 时点击正好偏下 177px)
|
||||||
|
offsetX = shot.originX;
|
||||||
|
offsetY = shot.originY;
|
||||||
|
}
|
||||||
|
|
||||||
|
// accuracy 字段语义为「相似度」(0~1, 越大越严格, 0.9=至少90%像素一致)——与界面文案一致。
|
||||||
|
// 引擎内部用最大允许差异 = 1 - 相似度。此前引擎直接把该值当差异阈值,
|
||||||
|
// 用户填 0.9 反而放宽到"接受90%像素差异", 导致到处误命中。
|
||||||
|
const rawAccuracy = Number(text(args, ["accuracy"], "0.8"));
|
||||||
|
const similarity = Math.max(0.05, Math.min(1, Number.isFinite(rawAccuracy) && rawAccuracy > 0 ? rawAccuracy : 0.8));
|
||||||
|
// 原生 1:1 匹配失败时, 按缩放比对模板放大/缩小重试——兼容旧版 DIP 截图存的模板与任意来源的图片
|
||||||
|
const scale = requireScreenService().primaryDisplay().scaleFactor || 1;
|
||||||
|
const result = await findImage(template, screenshot, 1 - similarity, scale !== 1 ? [scale, 1 / scale] : []);
|
||||||
|
if (!result) throw new RuntimeError("not_found", `屏幕中未找到目标图像(要求相似度 ${(similarity * 100).toFixed(0)}%,未达到阈值)`, { nodeId, recommendation: "重新框选模板(框小一点、只框目标图案),或适当调低相似度(如 0.8);目标在副屏时确认其清晰可见" });
|
||||||
|
return { ...result, x: result.x + offsetX, y: result.y + offsetY };
|
||||||
|
}
|
||||||
|
|
||||||
|
register(["Image_Find", "Image_Find_V1"], async (ctx, node, args) => locate(args, ctx.evidenceDir, node.id));
|
||||||
|
// 图像位置: 与查找图像同义(历史重复组件), 同管线返回坐标
|
||||||
|
for (const id of ["Image_Location", "Image_Location_V1"]) register(id, async (ctx, node, args) => locate(args, ctx.evidenceDir, node.id));
|
||||||
|
// 判断图像: 找到返回 true / 找不到返回 false(供条件分支), 仅有 not_found 转为 false;
|
||||||
|
// 模板路径为空/文件不存在等配置错误仍然抛出, 避免静默 false 难排查。
|
||||||
|
// timeout>0 时在窗口内每 300ms 重试(等待图像出现再判断的场景)
|
||||||
|
for (const id of ["Image_Exists", "Image_Exists_V1"]) register(id, async (ctx, node, args) => {
|
||||||
|
const rawTimeout = Number(text(args, ["timeout", "time_out"], "0"));
|
||||||
|
const deadline = Date.now() + (Number.isFinite(rawTimeout) && rawTimeout > 0 ? rawTimeout : 0) * 1000;
|
||||||
|
for (;;) {
|
||||||
|
try {
|
||||||
|
await locate(args, ctx.evidenceDir, node.id);
|
||||||
|
return true;
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof RuntimeError && error.code === "not_found") {
|
||||||
|
if (Date.now() >= deadline) return false;
|
||||||
|
} else {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await new Promise(resolve => setTimeout(resolve, 300));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
// 等待图像: 在超时窗口内每 300ms 重试一次查找, 命中即返回坐标, 超时仍无则抛 not_found
|
||||||
|
for (const id of ["Image_Wait", "Image_Wait_V1"]) register(id, async (ctx, node, args) => {
|
||||||
|
const rawTimeout = Number(text(args, ["timeout", "time_out"], "20"));
|
||||||
|
const timeout = Number.isFinite(rawTimeout) && rawTimeout >= 0 ? rawTimeout : 20;
|
||||||
|
const deadline = Date.now() + timeout * 1000;
|
||||||
|
for (;;) {
|
||||||
|
try {
|
||||||
|
return await locate(args, ctx.evidenceDir, node.id);
|
||||||
|
} catch (error) {
|
||||||
|
if (!(error instanceof RuntimeError) || error.code !== "not_found" || Date.now() >= deadline) throw error;
|
||||||
|
}
|
||||||
|
await new Promise(resolve => setTimeout(resolve, 300));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
// 移动鼠标到图像: 定位后把光标移到匹配区域中心;
|
||||||
|
// SetCursorPos 传物理坐标, 脚本必须声明 per-monitor DPI 感知(与 injectClick 同理), 否则被系统虚拟化导致偏移
|
||||||
|
for (const id of ["Image_Move", "Image_Move_V1", "Image_Move_V2"]) register(id, async (ctx, node, args) => {
|
||||||
|
const hit = await locate(args, ctx.evidenceDir, node.id);
|
||||||
|
const point = clickPoint(hit, text(args, ["position"], "center"));
|
||||||
|
const x = point.x;
|
||||||
|
const y = point.y;
|
||||||
|
await powershell("Add-Type -MemberDefinition '[DllImport(\"user32.dll\")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value); [DllImport(\"user32.dll\")] public static extern bool SetCursorPos(int X, int Y);' -Name RpaImageMoveInput -Namespace RpaImageMove | Out-Null; try { [void][RpaImageMove.RpaImageMoveInput]::SetProcessDpiAwarenessContext([IntPtr](-4)) } catch {}; [RpaImageMove.RpaImageMoveInput]::SetCursorPos(" + x + "," + y + ")");
|
||||||
|
return { ...hit, x, y };
|
||||||
|
});
|
||||||
|
/** 点击位置锚点(相对匹配区域的比例坐标),支持九宫格取值 */
|
||||||
|
const POSITION_ANCHORS: Record<string, [number, number]> = {
|
||||||
|
center: [0.5, 0.5],
|
||||||
|
leftTop: [0, 0], top: [0.5, 0], centerTop: [0.5, 0], rightTop: [1, 0],
|
||||||
|
left: [0, 0.5], centerLeft: [0, 0.5], right: [1, 0.5], centerRight: [1, 0.5],
|
||||||
|
leftBottom: [0, 1], bottom: [0.5, 1], centerBottom: [0.5, 1], rightBottom: [1, 1]
|
||||||
|
};
|
||||||
|
|
||||||
|
export function clickPoint(result: MatchResult, position: string): { x: number; y: number } {
|
||||||
|
const raw = position.trim();
|
||||||
|
if (raw && raw.toLowerCase() !== "center") {
|
||||||
|
if (raw.startsWith("{")) {
|
||||||
|
try {
|
||||||
|
const offset = JSON.parse(raw) as { x?: unknown; y?: unknown };
|
||||||
|
const dx = Number(offset.x);
|
||||||
|
const dy = Number(offset.y);
|
||||||
|
if (Number.isFinite(dx) && Number.isFinite(dy)) {
|
||||||
|
return { x: Math.round(result.x + dx), y: Math.round(result.y + dy) };
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// 非法 JSON 落回中心
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const anchor = POSITION_ANCHORS[raw] ?? POSITION_ANCHORS[raw.toLowerCase()] ?? POSITION_ANCHORS[raw.replace(/-([a-z])/g, (_: string, char: string) => char.toUpperCase())];
|
||||||
|
if (anchor) {
|
||||||
|
return { x: Math.round(result.x + result.width * anchor[0]), y: Math.round(result.y + result.height * anchor[1]) };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { x: Math.round(result.x + result.width / 2), y: Math.round(result.y + result.height / 2) };
|
||||||
|
}
|
||||||
|
|
||||||
|
const CLICK_ACTIONS: Record<string, string> = {
|
||||||
|
left: "[Mouse]::mouse_event(2,0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 30; [Mouse]::mouse_event(4,0,0,0,[UIntPtr]::Zero)",
|
||||||
|
right: "[Mouse]::mouse_event(8,0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 30; [Mouse]::mouse_event(16,0,0,0,[UIntPtr]::Zero)",
|
||||||
|
middle: "[Mouse]::mouse_event(32,0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 30; [Mouse]::mouse_event(64,0,0,0,[UIntPtr]::Zero)",
|
||||||
|
double: "[Mouse]::mouse_event(2,0,0,0,[UIntPtr]::Zero); [Mouse]::mouse_event(4,0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 60; [Mouse]::mouse_event(2,0,0,0,[UIntPtr]::Zero); [Mouse]::mouse_event(4,0,0,0,[UIntPtr]::Zero)"
|
||||||
|
};
|
||||||
|
|
||||||
|
async function injectClick(x: number, y: number, clickKey: string, showTrack: boolean): Promise<void> {
|
||||||
|
const action = CLICK_ACTIONS[clickKey] ?? CLICK_ACTIONS.left;
|
||||||
|
// show_track=1 时从当前光标位置平滑移动过去(20 步插值),否则直接跳转
|
||||||
|
const move = showTrack
|
||||||
|
? `for ($i = 1; $i -le 20; $i++) { [Mouse]::SetCursorPos([int]($p.X + (${x} - $p.X) * $i / 20), [int]($p.Y + (${y} - $p.Y) * $i / 20)); Start-Sleep -Milliseconds 12 }`
|
||||||
|
: `[Mouse]::SetCursorPos(${x}, ${y})`;
|
||||||
|
await powershell(`Add-Type @'
|
||||||
|
using System; using System.Runtime.InteropServices;
|
||||||
|
public static class Mouse {
|
||||||
|
[DllImport("user32.dll")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value);
|
||||||
|
[DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y);
|
||||||
|
[DllImport("user32.dll")] public static extern bool GetCursorPos(out POINT p);
|
||||||
|
[DllImport("user32.dll")] public static extern void mouse_event(uint f, uint x, uint y, uint d, UIntPtr e);
|
||||||
|
public struct POINT { public int X; public int Y; }
|
||||||
|
}
|
||||||
|
'@;
|
||||||
|
# 传入的是物理像素坐标(与物理分辨率截图匹配所得), 必须声明 DPI 感知——
|
||||||
|
# 否则进程被 DPI 虚拟化, 坐标再被系统放大一次, 点击成比例偏右下
|
||||||
|
try { [void][Mouse]::SetProcessDpiAwarenessContext([IntPtr](-4)) } catch {};
|
||||||
|
$p = New-Object Mouse+POINT;
|
||||||
|
[Mouse]::GetCursorPos([ref]$p) | Out-Null;
|
||||||
|
${move};
|
||||||
|
Start-Sleep -Milliseconds 60;
|
||||||
|
${action}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
register(["Image_Click", "Image_Click_V1", "Image_Click_V2", "Image_Click_V3"], async (ctx, node, args) => {
|
||||||
|
const result = await locate(args, ctx.evidenceDir, node.id);
|
||||||
|
const point = clickPoint(result, text(args, ["position"], "center"));
|
||||||
|
const clickKey = text(args, ["click_key"], "left");
|
||||||
|
const showTrack = String(args.show_track ?? "0") === "1" || args.show_track === true;
|
||||||
|
await injectClick(point.x, point.y, clickKey, showTrack);
|
||||||
|
ctx.log(`已${clickKey === "right" ? "右键" : clickKey === "double" ? "双击" : "点击"}图像 (${point.x}, ${point.y}),匹配得分 ${result.score.toFixed(3)}`, "info", node.id);
|
||||||
|
return { ...result, click: point };
|
||||||
|
});
|
||||||
@ -1,20 +1,10 @@
|
|||||||
import { register } from "../engine/registry";
|
import { register } from "../engine/registry";
|
||||||
import { RuntimeError, type CapabilityHandler } from "../engine/types";
|
import { RuntimeError, type CapabilityHandler } from "../engine/types";
|
||||||
import { powershell, psString } from "./desktop";
|
import { getHostBridge } from "../engine/host-bridge";
|
||||||
|
import { defaultEngine } from "../engine/engine";
|
||||||
|
|
||||||
const WINDOW_API = `Add-Type @'
|
// 桌面自动化驱动(任务 3.3):窗口矩形/鼠标注入原语统一走 WindowsDriver
|
||||||
using System;
|
const getDriver = () => defaultEngine.driver;
|
||||||
using System.Runtime.InteropServices;
|
|
||||||
public static class RpaWindowExtended {
|
|
||||||
[StructLayout(LayoutKind.Sequential)] public struct RECT { public int Left; public int Top; public int Right; public int Bottom; }
|
|
||||||
[DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr hWnd, out RECT rect);
|
|
||||||
[DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd);
|
|
||||||
[DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr hWnd, int command);
|
|
||||||
[DllImport("user32.dll")] public static extern bool SetWindowPos(IntPtr hWnd, IntPtr after, int x, int y, int width, int height, uint flags);
|
|
||||||
[DllImport("user32.dll")] public static extern bool SetCursorPos(int x, int y);
|
|
||||||
[DllImport("user32.dll")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra);
|
|
||||||
}
|
|
||||||
'@`;
|
|
||||||
|
|
||||||
function value(args: Record<string, unknown>, keys: string[], fallback = ""): string {
|
function value(args: Record<string, unknown>, keys: string[], fallback = ""): string {
|
||||||
for (const key of keys) {
|
for (const key of keys) {
|
||||||
@ -44,39 +34,60 @@ function requiredTitle(args: Record<string, unknown>, nodeId: string, componentI
|
|||||||
return title;
|
return title;
|
||||||
}
|
}
|
||||||
|
|
||||||
function findWindow(title: string): string {
|
|
||||||
return `$p = Get-Process | Where-Object { $_.MainWindowHandle -ne 0 -and $_.MainWindowTitle -like ${psString(`*${title}*`)} } | Select-Object -First 1; if (-not $p) { throw '找不到窗口:${title.replace(/'/g, "''")}' }`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function parsePoint(candidate: unknown, label: string, nodeId: string, componentId: string): [number, number] {
|
function parsePoint(candidate: unknown, label: string, nodeId: string, componentId: string): [number, number] {
|
||||||
let parsed: unknown = candidate;
|
let parsed: unknown = candidate;
|
||||||
if (typeof parsed === "string") {
|
if (typeof parsed === "string") {
|
||||||
const source = parsed;
|
const source = parsed;
|
||||||
try { parsed = JSON.parse(source); } catch { parsed = source.split(/[,,xX ]+/).map(Number); }
|
try { parsed = JSON.parse(source); } catch { parsed = source.split(/[,,xX ]+/).map(Number); }
|
||||||
}
|
}
|
||||||
|
// 屏幕框选/拾取坐标生成的是 {"x":..,"y":..} 对象,直接转换
|
||||||
|
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
||||||
|
const record = parsed as Record<string, unknown>;
|
||||||
|
if (record.x !== undefined && record.y !== undefined) parsed = [record.x, record.y];
|
||||||
|
}
|
||||||
if (!Array.isArray(parsed) || parsed.length < 2 || parsed.slice(0, 2).some((item) => !Number.isFinite(Number(item)))) {
|
if (!Array.isArray(parsed) || parsed.length < 2 || parsed.slice(0, 2).some((item) => !Number.isFinite(Number(item)))) {
|
||||||
throw new RuntimeError("validation_error", `${label}无效`, { nodeId, componentId });
|
throw new RuntimeError("validation_error", `${label}无效`, { nodeId, componentId });
|
||||||
}
|
}
|
||||||
return [Number(parsed[0]), Number(parsed[1])];
|
return [Number(parsed[0]), Number(parsed[1])];
|
||||||
}
|
}
|
||||||
|
|
||||||
async function windowObject(title: string): Promise<Record<string, unknown>> {
|
/** 界面(框选/元素拾取/截图)产出的是 DIP 逻辑坐标;DPI 感知进程的 SetCursorPos 用物理像素,
|
||||||
const output = await powershell(`${WINDOW_API}; ${findWindow(title)}; $r = New-Object RpaWindowExtended+RECT; [RpaWindowExtended]::GetWindowRect($p.MainWindowHandle,[ref]$r) | Out-Null; @{pid=$p.Id;title=$p.MainWindowTitle;hwnd=$p.MainWindowHandle.ToInt64();x=$r.Left;y=$r.Top;width=$r.Right-$r.Left;height=$r.Bottom-$r.Top} | ConvertTo-Json -Compress`);
|
* 缩放屏(125%/150%)下直接混用会成比例偏移(用户实测 125% 下差约 25% 距离)。
|
||||||
return JSON.parse(output);
|
* 统一在执行前换算;宿主桥未注入时(vitest)回退原值保持旧行为 */
|
||||||
|
function dipToPhysical(x: number, y: number): [number, number] {
|
||||||
|
try {
|
||||||
|
const bridge = getHostBridge();
|
||||||
|
if (bridge) {
|
||||||
|
const physical = bridge.dipToScreenPoint({ x, y });
|
||||||
|
return [Math.round(physical.x), Math.round(physical.y)];
|
||||||
|
}
|
||||||
|
} catch {}
|
||||||
|
return [Math.round(x), Math.round(y)];
|
||||||
}
|
}
|
||||||
|
|
||||||
const getWindow: CapabilityHandler = async (_ctx, node, args) => windowObject(requiredTitle(args, node.id, node.componentId));
|
/** 反向换算:物理像素(如 GetWindowRect 结果) → DIP,用于「相对激活窗口」偏移叠加 */
|
||||||
|
function physicalToDip(x: number, y: number): [number, number] {
|
||||||
|
try {
|
||||||
|
const bridge = getHostBridge();
|
||||||
|
if (bridge) {
|
||||||
|
const dip = bridge.screenToDipPoint({ x, y });
|
||||||
|
return [Math.round(dip.x), Math.round(dip.y)];
|
||||||
|
}
|
||||||
|
} catch {}
|
||||||
|
return [Math.round(x), Math.round(y)];
|
||||||
|
}
|
||||||
|
|
||||||
|
const getWindow: CapabilityHandler = async (_ctx, node, args) => getDriver().window.findByTitle(requiredTitle(args, node.id, node.componentId));
|
||||||
register(["Window_Get", "Window_GetWindow", "Window_GetWindow_v1"], getWindow);
|
register(["Window_Get", "Window_GetWindow", "Window_GetWindow_v1"], getWindow);
|
||||||
|
|
||||||
register("Window_ActiveWindow_v1", async (_ctx, node, args) => {
|
register("Window_ActiveWindow_v1", async (_ctx, node, args) => {
|
||||||
const title = requiredTitle(args, node.id, node.componentId);
|
const title = requiredTitle(args, node.id, node.componentId);
|
||||||
const maximize = [true, 1, "1", "true", "True"].includes(args.isMax as never);
|
const maximize = [true, 1, "1", "true", "True"].includes(args.isMax as never);
|
||||||
await powershell(`${WINDOW_API}; ${findWindow(title)}; [RpaWindowExtended]::SetForegroundWindow($p.MainWindowHandle) | Out-Null; ${maximize ? "[RpaWindowExtended]::ShowWindow($p.MainWindowHandle,3) | Out-Null" : ""}`);
|
return getDriver().window.activate(title, { maximize });
|
||||||
return windowObject(title);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
register(["Window_GetWindowMessage", "Window_GetWindowMessage_v1"], async (_ctx, node, args) => {
|
register(["Window_GetWindowMessage", "Window_GetWindowMessage_v1"], async (_ctx, node, args) => {
|
||||||
const object = await windowObject(requiredTitle(args, node.id, node.componentId));
|
const object = await getDriver().window.findByTitle(requiredTitle(args, node.id, node.componentId));
|
||||||
const type = value(args, ["switchType"], "all").toLowerCase();
|
const type = value(args, ["switchType"], "all").toLowerCase();
|
||||||
if (type === "title") return object.title;
|
if (type === "title") return object.title;
|
||||||
if (type === "pid") return object.pid;
|
if (type === "pid") return object.pid;
|
||||||
@ -87,7 +98,7 @@ register(["Window_GetWindowMessage", "Window_GetWindowMessage_v1"], async (_ctx,
|
|||||||
|
|
||||||
register(["Window_SetPos", "Window_MoveWindow", "Window_MoveWindow_v1", "Window_SizeWindow"], async (_ctx, node, args) => {
|
register(["Window_SetPos", "Window_MoveWindow", "Window_MoveWindow_v1", "Window_SizeWindow"], async (_ctx, node, args) => {
|
||||||
const title = requiredTitle(args, node.id, node.componentId);
|
const title = requiredTitle(args, node.id, node.componentId);
|
||||||
const current = await windowObject(title);
|
const current = await getDriver().window.findByTitle(title);
|
||||||
const moveOnly = node.componentId.includes("MoveWindow");
|
const moveOnly = node.componentId.includes("MoveWindow");
|
||||||
const sizeOnly = node.componentId === "Window_SizeWindow";
|
const sizeOnly = node.componentId === "Window_SizeWindow";
|
||||||
const position = args.location !== undefined ? parsePoint(args.location, "窗口位置", node.id, node.componentId) : [Number(args.x ?? current.x), Number(args.y ?? current.y)] as [number, number];
|
const position = args.location !== undefined ? parsePoint(args.location, "窗口位置", node.id, node.componentId) : [Number(args.x ?? current.x), Number(args.y ?? current.y)] as [number, number];
|
||||||
@ -98,36 +109,56 @@ register(["Window_SetPos", "Window_MoveWindow", "Window_MoveWindow_v1", "Window_
|
|||||||
const width = moveOnly ? Number(current.width) : size[0];
|
const width = moveOnly ? Number(current.width) : size[0];
|
||||||
const height = moveOnly ? Number(current.height) : size[1];
|
const height = moveOnly ? Number(current.height) : size[1];
|
||||||
if (!(width > 0 && height > 0)) throw new RuntimeError("validation_error", "窗口尺寸必须大于 0", { nodeId: node.id, componentId: node.componentId });
|
if (!(width > 0 && height > 0)) throw new RuntimeError("validation_error", "窗口尺寸必须大于 0", { nodeId: node.id, componentId: node.componentId });
|
||||||
await powershell(`${WINDOW_API}; ${findWindow(title)}; [RpaWindowExtended]::SetWindowPos($p.MainWindowHandle,[IntPtr]::Zero,${x},${y},${width},${height},4) | Out-Null`);
|
return getDriver().window.setPos(title, { x, y, width, height });
|
||||||
return { ...(await windowObject(title)), x, y, width, height };
|
|
||||||
});
|
});
|
||||||
|
|
||||||
const moveMouse: CapabilityHandler = async (_ctx, node, args) => {
|
const moveMouse: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
const [x, y] = parsePoint(args.location ?? args.position, "鼠标坐标", node.id, node.componentId);
|
// 注意:position 字段是「坐标相对位置」(0=屏幕左上角/1=激活窗口左上角),坐标本体在 location
|
||||||
await powershell(`${WINDOW_API}; [RpaWindowExtended]::SetCursorPos(${x},${y}) | Out-Null`);
|
const [rawX, rawY] = parsePoint(args.location ?? args.point, "鼠标坐标", node.id, node.componentId);
|
||||||
return { x, y };
|
const relative = String(args.position ?? "0") === "1";
|
||||||
|
const moveType = String(args.move_type ?? "0");
|
||||||
|
const speed = Math.min(100, Math.max(1, Number(args.speed ?? 35) || 35));
|
||||||
|
|
||||||
|
let dipTarget: [number, number];
|
||||||
|
if (relative) {
|
||||||
|
// 相对激活窗口左上角:窗口矩形是物理像素,换回 DIP 再叠加偏移,最后统一转物理
|
||||||
|
const { x: wx, y: wy } = await getDriver().window.foregroundPosition();
|
||||||
|
if (!Number.isFinite(wx) || !Number.isFinite(wy)) throw new RuntimeError("browser_error", "无法获取激活窗口位置", { nodeId: node.id, componentId: node.componentId });
|
||||||
|
const [dipWx, dipWy] = physicalToDip(wx, wy);
|
||||||
|
dipTarget = [dipWx + rawX, dipWy + rawY];
|
||||||
|
} else {
|
||||||
|
dipTarget = [rawX, rawY];
|
||||||
|
}
|
||||||
|
const [tx, ty] = dipToPhysical(dipTarget[0], dipTarget[1]);
|
||||||
|
if (moveType === "2") {
|
||||||
|
await getDriver().mouse.moveTo({ x: tx, y: ty });
|
||||||
|
} else {
|
||||||
|
// 平滑移动:从当前光标位置出发,按速度插值(路径全程物理像素,与目标同一坐标系)
|
||||||
|
await getDriver().mouse.moveSmooth({ x: tx, y: ty }, { ease: moveType === "0", speed });
|
||||||
|
}
|
||||||
|
return { x: Math.round(dipTarget[0]), y: Math.round(dipTarget[1]) };
|
||||||
};
|
};
|
||||||
register(["Mouse_MoveToV1", "Mouse_MoveToV2"], moveMouse);
|
register(["Mouse_MoveToV1", "Mouse_MoveToV2"], moveMouse);
|
||||||
|
|
||||||
const dragMouse: CapabilityHandler = async (_ctx, node, args) => {
|
const dragMouse: CapabilityHandler = async (_ctx, node, args) => {
|
||||||
const [sx, sy] = parsePoint(args.Start ?? args.start, "起始坐标", node.id, node.componentId);
|
const [sx, sy] = parsePoint(args.Start ?? args.start, "起始坐标", node.id, node.componentId);
|
||||||
const [ex, ey] = parsePoint(args.End ?? args.end, "结束坐标", node.id, node.componentId);
|
const [ex, ey] = parsePoint(args.End ?? args.end, "结束坐标", node.id, node.componentId);
|
||||||
|
const [spx, spy] = dipToPhysical(sx, sy);
|
||||||
|
const [epx, epy] = dipToPhysical(ex, ey);
|
||||||
const right = ["1", "right"].includes(value(args, ["side"], "0").toLowerCase());
|
const right = ["1", "right"].includes(value(args, ["side"], "0").toLowerCase());
|
||||||
const down = right ? 8 : 2;
|
// 旧实现按下后停 80ms、拖到终点立即松开(与桌面版 Window_Mouse 不同,松开前不停顿)
|
||||||
const up = right ? 16 : 4;
|
await getDriver().mouse.drag({ x: spx, y: spy }, { x: epx, y: epy }, { button: right ? "right" : "left", holdMs: 80, releaseDelayMs: 0 });
|
||||||
await powershell(`${WINDOW_API}; [RpaWindowExtended]::SetCursorPos(${sx},${sy}) | Out-Null; [RpaWindowExtended]::mouse_event(${down},0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 80; [RpaWindowExtended]::SetCursorPos(${ex},${ey}) | Out-Null; [RpaWindowExtended]::mouse_event(${up},0,0,0,[UIntPtr]::Zero)`);
|
return { start: [Math.round(sx), Math.round(sy)], end: [Math.round(ex), Math.round(ey)] };
|
||||||
return { start: [sx, sy], end: [ex, ey] };
|
|
||||||
};
|
};
|
||||||
register(["Mouse_DragToV1", "Mouse_DragToV2", "Mouse_DragToNew"], dragMouse);
|
register(["Mouse_DragToV1", "Mouse_DragToV2", "Mouse_DragToNew"], dragMouse);
|
||||||
|
|
||||||
register(["Mouse_ClickCombine", "Mouse_ClickCombine_v1", "Mouse_ClickCombine_v2", "Mouse_ClickCombine_v3", "Mouse_ClickCombine_v4"], async (_ctx, node, args) => {
|
register(["Mouse_ClickCombine", "Mouse_ClickCombine_v1", "Mouse_ClickCombine_v2", "Mouse_ClickCombine_v3", "Mouse_ClickCombine_v4"], async (_ctx, node, args) => {
|
||||||
if (value(args, ["target"], "0") !== "0") throw new RuntimeError("validation_error", "组合点击当前仅支持坐标目标", { nodeId: node.id, componentId: node.componentId });
|
if (value(args, ["target"], "0") !== "0") throw new RuntimeError("validation_error", "组合点击当前仅支持坐标目标", { nodeId: node.id, componentId: node.componentId });
|
||||||
const [x, y] = parsePoint(args.location, "点击坐标", node.id, node.componentId);
|
const [x, y] = parsePoint(args.location, "点击坐标", node.id, node.componentId);
|
||||||
|
const [px, py] = dipToPhysical(x, y);
|
||||||
const right = value(args, ["click_key"], "left").toLowerCase() === "right";
|
const right = value(args, ["click_key"], "left").toLowerCase() === "right";
|
||||||
const doubleClick = value(args, ["click_time"], "click").toLowerCase().includes("double");
|
const doubleClick = value(args, ["click_time"], "click").toLowerCase().includes("double");
|
||||||
const down = right ? 8 : 2;
|
// 旧实现每次 down-up 之后停 80ms
|
||||||
const up = right ? 16 : 4;
|
await getDriver().mouse.click({ x: px, y: py }, { button: right ? "right" : "left", double: doubleClick, settleMs: 80 });
|
||||||
const repeats = doubleClick ? 2 : 1;
|
return { x: Math.round(x), y: Math.round(y), button: right ? "right" : "left", clicks: doubleClick ? 2 : 1 };
|
||||||
await powershell(`${WINDOW_API}; [RpaWindowExtended]::SetCursorPos(${x},${y}) | Out-Null; 1..${repeats} | ForEach-Object { [RpaWindowExtended]::mouse_event(${down},0,0,0,[UIntPtr]::Zero); [RpaWindowExtended]::mouse_event(${up},0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 80 }`);
|
|
||||||
return { x, y, button: right ? "right" : "left", clicks: repeats };
|
|
||||||
});
|
});
|
||||||
@ -5,6 +5,13 @@ import { promisify } from "node:util";
|
|||||||
|
|
||||||
const run = promisify(execFile);
|
const run = promisify(execFile);
|
||||||
const ps = (script: string) => run("powershell.exe", ["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-Command", script], { windowsHide: true, timeout: 60000 });
|
const ps = (script: string) => run("powershell.exe", ["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-Command", script], { windowsHide: true, timeout: 60000 });
|
||||||
|
// cmdkey 在中文 Windows 上输出 GBK,直接 execFile 按 UTF-8 解码会乱码;统一经 PowerShell 强制 UTF-8 控制台编码捕获
|
||||||
|
const cmdkey = async (args: string[]): Promise<string> => {
|
||||||
|
const quoted = args.map((item) => "'" + item.replace(/'/g, "''") + "'");
|
||||||
|
const script = "[Console]::OutputEncoding=[System.Text.UTF8Encoding]::new($false); & cmdkey " + quoted.join(" ");
|
||||||
|
const { stdout } = await run("powershell.exe", ["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-Command", script], { windowsHide: true, timeout: 60000 });
|
||||||
|
return stdout;
|
||||||
|
};
|
||||||
|
|
||||||
function reqText(args: Record<string, unknown>, keys: string[], label: string, nodeId: string): string {
|
function reqText(args: Record<string, unknown>, keys: string[], label: string, nodeId: string): string {
|
||||||
for (const key of keys) {
|
for (const key of keys) {
|
||||||
@ -42,10 +49,16 @@ register("System_ScreenSaverStop", async () => {
|
|||||||
|
|
||||||
register("System_ReadCredential", async (ctx, node, args) => {
|
register("System_ReadCredential", async (ctx, node, args) => {
|
||||||
const app = reqText(args, ["application"], "凭据目标", node.id);
|
const app = reqText(args, ["application"], "凭据目标", node.id);
|
||||||
const { stdout } = await run("cmdkey", ["/list"], { windowsHide: true });
|
const stdout = await cmdkey(["/list"]);
|
||||||
const exists = stdout.includes(app);
|
const exists = stdout.includes(app);
|
||||||
const userLine = stdout.split(/\r?\n/).find(l => l.includes(app));
|
// cmdkey /list 每条凭据是「目标/类型/用户」多行块(空行分隔);取整块才有用户名等有效信息
|
||||||
return { target: app, exists, detail: exists ? (userLine ?? "").trim() : "" };
|
const lines = stdout.split(/\r?\n/);
|
||||||
|
const idx = lines.findIndex((l) => l.includes(app));
|
||||||
|
const block: string[] = [];
|
||||||
|
if (idx >= 0) {
|
||||||
|
for (let i = idx; i < lines.length && lines[i].trim(); i++) block.push(lines[i].trim());
|
||||||
|
}
|
||||||
|
return { target: app, exists, detail: exists ? block.join(" | ") : "" };
|
||||||
});
|
});
|
||||||
|
|
||||||
register("System_AddCredential", async (ctx, node, args) => {
|
register("System_AddCredential", async (ctx, node, args) => {
|
||||||
@ -53,13 +66,13 @@ register("System_AddCredential", async (ctx, node, args) => {
|
|||||||
const user = reqText(args, ["name"], "用户名", node.id);
|
const user = reqText(args, ["name"], "用户名", node.id);
|
||||||
const pwd = String(args.password ?? "");
|
const pwd = String(args.password ?? "");
|
||||||
const psArgs = pwd ? ["/generic:" + app, "/user:" + user, "/pass:" + pwd] : ["/generic:" + app, "/user:" + user];
|
const psArgs = pwd ? ["/generic:" + app, "/user:" + user, "/pass:" + pwd] : ["/generic:" + app, "/user:" + user];
|
||||||
await run("cmdkey", psArgs, { windowsHide: true });
|
await cmdkey(psArgs);
|
||||||
return { added: true, target: app };
|
return { added: true, target: app };
|
||||||
});
|
});
|
||||||
|
|
||||||
register("System_DeleteCredential", async (ctx, node, args) => {
|
register("System_DeleteCredential", async (ctx, node, args) => {
|
||||||
const app = reqText(args, ["application"], "凭据目标", node.id);
|
const app = reqText(args, ["application"], "凭据目标", node.id);
|
||||||
await run("cmdkey", ["/delete:" + app], { windowsHide: true });
|
await cmdkey(["/delete:" + app]);
|
||||||
return { deleted: true, target: app };
|
return { deleted: true, target: app };
|
||||||
});
|
});
|
||||||
|
|
||||||
264
packages/rpa-core/src/capabilities/word-extended.ts
Normal file
264
packages/rpa-core/src/capabilities/word-extended.ts
Normal file
@ -0,0 +1,264 @@
|
|||||||
|
import { promises as fs } from "node:fs";
|
||||||
|
import path from "node:path";
|
||||||
|
import { getHandler, register } from "../engine/registry";
|
||||||
|
import { RuntimeError } from "../engine/types";
|
||||||
|
import { powershell, psString } from "../driver/windows/powershell";
|
||||||
|
|
||||||
|
interface WordHandle { path: string; }
|
||||||
|
|
||||||
|
function text(args: Record<string, unknown>, keys: string[], fallback = ""): string {
|
||||||
|
for (const key of keys) {
|
||||||
|
const current = args[key];
|
||||||
|
if (current !== undefined && current !== null && current !== "") return String(current);
|
||||||
|
}
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
function bool(value: unknown): boolean {
|
||||||
|
return [true, 1, "1", "true", "True", "是", "开启"].includes(value as never);
|
||||||
|
}
|
||||||
|
|
||||||
|
function handlePath(args: Record<string, unknown>, nodeId: string, componentId: string): string {
|
||||||
|
const raw = args.objWord ?? args.filePath ?? args.path;
|
||||||
|
const candidate = raw && typeof raw === "object" ? (raw as Partial<WordHandle>).path : raw;
|
||||||
|
const result = String(candidate || "").trim();
|
||||||
|
if (!result || result === "word对象") throw new RuntimeError("validation_error", "Word文档路径为空", { nodeId, componentId });
|
||||||
|
return path.resolve(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
function script(filePath: string, body: string, save = true): string {
|
||||||
|
return `$word=$null;$doc=$null;try{$word=New-Object -ComObject Word.Application;$word.Visible=$false;$word.DisplayAlerts=0;$doc=$word.Documents.Open(${psString(filePath)});${body};${save ? "$doc.Save();" : ""}}finally{if($doc -ne $null){try{$doc.Close()}catch{};try{[void][Runtime.InteropServices.Marshal]::ReleaseComObject($doc)}catch{}};if($word -ne $null){try{$word.Quit()}catch{};try{[void][Runtime.InteropServices.Marshal]::ReleaseComObject($word)}catch{}};[GC]::Collect();[GC]::WaitForPendingFinalizers()}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
register("Word_New_V1", async (ctx, node, args, component) => {
|
||||||
|
const base = getHandler("Word_New");
|
||||||
|
if (!base) throw new RuntimeError("capability_not_implemented", "基础 Word 新建能力不可用", { nodeId: node.id, componentId: node.componentId });
|
||||||
|
return base(ctx, node, args, component);
|
||||||
|
});
|
||||||
|
|
||||||
|
register("Word_GetPath", (_ctx, node, args) => handlePath(args, node.id, node.componentId));
|
||||||
|
|
||||||
|
register("Word_ReadAll", async (_ctx, node, args) => {
|
||||||
|
const filePath = handlePath(args, node.id, node.componentId);
|
||||||
|
const encoded = await powershell(script(filePath, "[Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($doc.Content.Text))", false));
|
||||||
|
return Buffer.from(encoded.trim(), "base64").toString("utf8");
|
||||||
|
});
|
||||||
|
|
||||||
|
register("Word_Replace", async (_ctx, node, args) => {
|
||||||
|
const filePath = handlePath(args, node.id, node.componentId);
|
||||||
|
const oldText = text(args, ["oldText"]);
|
||||||
|
if (!oldText) throw new RuntimeError("validation_error", "原内容为空", { nodeId: node.id, componentId: node.componentId });
|
||||||
|
const newText = text(args, ["newText"]);
|
||||||
|
await powershell(script(filePath, `$find=$doc.Content.Find;[void]$find.Execute(${psString(oldText)},$false,$false,$false,$false,$false,$true,1,$false,${psString(newText)},2)`));
|
||||||
|
return { path: filePath } satisfies WordHandle;
|
||||||
|
});
|
||||||
|
|
||||||
|
register("Word_Enter", async (_ctx, node, args) => {
|
||||||
|
const filePath = handlePath(args, node.id, node.componentId);
|
||||||
|
await powershell(script(filePath, "$range=$doc.Content;$range.Collapse(0);$range.InsertParagraphAfter()"));
|
||||||
|
return { path: filePath } satisfies WordHandle;
|
||||||
|
});
|
||||||
|
|
||||||
|
register("Word_InsertPage", async (_ctx, node, args) => {
|
||||||
|
const filePath = handlePath(args, node.id, node.componentId);
|
||||||
|
await powershell(script(filePath, "$range=$doc.Content;$range.Collapse(0);$range.InsertBreak(7)"));
|
||||||
|
return { path: filePath } satisfies WordHandle;
|
||||||
|
});
|
||||||
|
|
||||||
|
register("Word_SetFontName", async (_ctx, node, args) => {
|
||||||
|
const filePath = handlePath(args, node.id, node.componentId);
|
||||||
|
await powershell(script(filePath, `$doc.Content.Font.Name=${psString(text(args, ["font"], "宋体"))}`));
|
||||||
|
return { path: filePath } satisfies WordHandle;
|
||||||
|
});
|
||||||
|
|
||||||
|
register("Word_SetFontSize", async (_ctx, node, args) => {
|
||||||
|
const filePath = handlePath(args, node.id, node.componentId);
|
||||||
|
const size = Number(text(args, ["fontSize"], "9"));
|
||||||
|
if (!(size > 0)) throw new RuntimeError("validation_error", "字体大小无效", { nodeId: node.id, componentId: node.componentId });
|
||||||
|
await powershell(script(filePath, `$doc.Content.Font.Size=${size}`));
|
||||||
|
return { path: filePath } satisfies WordHandle;
|
||||||
|
});
|
||||||
|
|
||||||
|
register("Word_SetFontStyle", async (_ctx, node, args) => {
|
||||||
|
const filePath = handlePath(args, node.id, node.componentId);
|
||||||
|
await powershell(script(filePath, `$doc.Content.Font.Bold=${bool(args.isBold) ? 1 : 0};$doc.Content.Font.Italic=${bool(args.isItalic) ? 1 : 0};$doc.Content.Font.Underline=${bool(args.isUnderline) ? 1 : 0}`));
|
||||||
|
return { path: filePath } satisfies WordHandle;
|
||||||
|
});
|
||||||
|
|
||||||
|
register("Word_SetFontColor", async (_ctx, node, args) => {
|
||||||
|
const filePath = handlePath(args, node.id, node.componentId);
|
||||||
|
const hex = text(args, ["fontColor"], "#000000").replace("#", "");
|
||||||
|
if (!/^[0-9a-fA-F]{6}$/.test(hex)) throw new RuntimeError("validation_error", "字体颜色格式无效", { nodeId: node.id, componentId: node.componentId });
|
||||||
|
const red = Number.parseInt(hex.slice(0, 2), 16); const green = Number.parseInt(hex.slice(2, 4), 16); const blue = Number.parseInt(hex.slice(4, 6), 16);
|
||||||
|
await powershell(script(filePath, `$doc.Content.Font.Color=${red + green * 256 + blue * 65536}`));
|
||||||
|
return { path: filePath } satisfies WordHandle;
|
||||||
|
});
|
||||||
|
|
||||||
|
register("Word_SetAlign", async (_ctx, node, args) => {
|
||||||
|
const filePath = handlePath(args, node.id, node.componentId);
|
||||||
|
const align = text(args, ["align"], "left").toLowerCase();
|
||||||
|
const value = align.includes("center") || align === "1" ? 1 : align.includes("right") || align === "2" ? 2 : align.includes("justify") || align === "3" ? 3 : 0;
|
||||||
|
await powershell(script(filePath, `$doc.Content.ParagraphFormat.Alignment=${value}`));
|
||||||
|
return { path: filePath } satisfies WordHandle;
|
||||||
|
});
|
||||||
|
|
||||||
|
register("Word_InsertPicture", async (_ctx, node, args) => {
|
||||||
|
const filePath = handlePath(args, node.id, node.componentId);
|
||||||
|
const image = path.resolve(text(args, ["imageaddr"]));
|
||||||
|
await fs.access(image);
|
||||||
|
await powershell(script(filePath, `$range=$doc.Content;$range.Collapse(0);[void]$doc.InlineShapes.AddPicture(${psString(image)},$false,$true,$range)`));
|
||||||
|
return { path: filePath } satisfies WordHandle;
|
||||||
|
});
|
||||||
|
|
||||||
|
register("Word_WriteTable", async (_ctx, node, args) => {
|
||||||
|
const filePath = handlePath(args, node.id, node.componentId);
|
||||||
|
let data: unknown = args.data;
|
||||||
|
if (typeof data === "string") { try { data = JSON.parse(data); } catch { data = [[data]]; } }
|
||||||
|
const rows = Array.isArray(data) ? data.map((row) => Array.isArray(row) ? row : [row]) : [[data]];
|
||||||
|
const columns = Math.max(1, ...rows.map((row) => row.length));
|
||||||
|
const encoded = Buffer.from(JSON.stringify(rows), "utf8").toString("base64");
|
||||||
|
await powershell(script(filePath, `$json=[Text.Encoding]::UTF8.GetString([Convert]::FromBase64String(${psString(encoded)}));$data=$json|ConvertFrom-Json;$range=$doc.Content;$range.Collapse(0);$table=$doc.Tables.Add($range,${rows.length},${columns});for($r=0;$r-lt${rows.length};$r++){for($c=0;$c-lt${columns};$c++){if($c-lt$data[$r].Count){$table.Cell($r+1,$c+1).Range.Text=[string]$data[$r][$c]}}};$table.Borders.Enable=${bool(args.is_border ?? "1") ? 1 : 0}`));
|
||||||
|
return { path: filePath } satisfies WordHandle;
|
||||||
|
});
|
||||||
|
|
||||||
|
register("Word_Close", (_ctx, node, args) => ({ path: handlePath(args, node.id, node.componentId), closed: true }));
|
||||||
|
|
||||||
|
interface WordSelection { start: number; end: number; }
|
||||||
|
|
||||||
|
function selectionOf(args: Record<string, unknown>): WordSelection {
|
||||||
|
const handle = args.objWord;
|
||||||
|
const raw = handle && typeof handle === "object" ? (handle as { selection?: unknown }).selection : null;
|
||||||
|
const record = raw && typeof raw === "object" ? raw as Record<string, unknown> : {};
|
||||||
|
let start = Math.max(0, Math.floor(Number(record.start ?? 0) || 0));
|
||||||
|
let end = Math.max(0, Math.floor(Number(record.end ?? record.active ?? start) || 0));
|
||||||
|
if (end < start) [start, end] = [end, start];
|
||||||
|
return { start, end };
|
||||||
|
}
|
||||||
|
|
||||||
|
function selectionScript(filePath: string, selection: WordSelection, body = "", withText = true): string {
|
||||||
|
return `$word=$null;$doc=$null;try{` +
|
||||||
|
`$word=New-Object -ComObject Word.Application;$word.Visible=$false;$word.DisplayAlerts=0;$doc=$word.Documents.Open(${psString(filePath)});` +
|
||||||
|
`$sel=$doc.Range(${selection.start},${selection.end});$sel.Select()|Out-Null;${body};if($null -eq $sel){$sel=$doc.Content}` +
|
||||||
|
`$text="";if(${withText ? "$true" : "$false"}){$text=[string]$sel.Text};$out=@{start=[int]$sel.Start;end=[int]$sel.End;text=$text}|ConvertTo-Json -Compress;` +
|
||||||
|
`[Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($out))|Write-Output;$doc.Save()` +
|
||||||
|
`}finally{if($doc -ne $null){try{$doc.Close()}catch{};try{[void][Runtime.InteropServices.Marshal]::ReleaseComObject($doc)}catch{}};if($word -ne $null){try{$word.Quit()}catch{};try{[void][Runtime.InteropServices.Marshal]::ReleaseComObject($word)}catch{}};[GC]::Collect();[GC]::WaitForPendingFinalizers()}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runSelection(filePath: string, selection: WordSelection, body = "", withText = true): Promise<WordSelection & { text: string }> {
|
||||||
|
const output = await powershell(selectionScript(filePath, selection, body, withText));
|
||||||
|
if (output.includes("__WORD_NOT_FOUND__")) throw new RuntimeError("word_text_not_found", "Word 查找文本不存在", { detail: `path=${filePath}` });
|
||||||
|
const parsed = JSON.parse(Buffer.from(output.trim(), "base64").toString("utf8")) as { start: number; end: number; text: string };
|
||||||
|
return { start: parsed.start, end: parsed.end, text: parsed.text };
|
||||||
|
}
|
||||||
|
|
||||||
|
function selectionResult(filePath: string, selection: WordSelection): { path: string; selection: WordSelection } {
|
||||||
|
return { path: filePath, selection };
|
||||||
|
}
|
||||||
|
|
||||||
|
const selectionText = async (_ctx: unknown, node: { id: string; componentId: string }, args: Record<string, unknown>): Promise<string> =>
|
||||||
|
(await runSelection(handlePath(args, node.id, node.componentId), selectionOf(args))).text;
|
||||||
|
|
||||||
|
const selectionMutation = (body: string): CapabilityHandlerLike =>
|
||||||
|
async (_ctx, node, args) => selectionResult(handlePath(args, node.id, node.componentId), await runSelection(handlePath(args, node.id, node.componentId), selectionOf(args), body));
|
||||||
|
|
||||||
|
type CapabilityHandlerLike = Parameters<typeof register>[1];
|
||||||
|
|
||||||
|
register("Word_ReadSelect", selectionText);
|
||||||
|
register("Word_CopySelect", selectionMutation("$sel.Copy()|Out-Null"));
|
||||||
|
register("Word_CutSelect", selectionMutation("$sel.Cut()|Out-Null"));
|
||||||
|
register("Word_DeleteSelect", selectionMutation("[void]$sel.Delete()"));
|
||||||
|
register("Word_PasteSelect", selectionMutation("$sel.Paste()|Out-Null"));
|
||||||
|
register("Word_SelectAll", selectionMutation("$sel=$doc.Content;$sel.Select()|Out-Null"));
|
||||||
|
|
||||||
|
register("Word_SelectRow", async (_ctx, node, args) => {
|
||||||
|
const filePath = handlePath(args, node.id, node.componentId);
|
||||||
|
const startRow = Math.max(1, Math.floor(Number(args.st ?? 1) || 1));
|
||||||
|
const endRow = Math.max(startRow, Math.floor(Number(args.ed ?? args.start ?? startRow) || startRow));
|
||||||
|
const body = [
|
||||||
|
`$first=$doc.GoTo(3,1,${startRow}).Bookmarks("\\Line").Range`,
|
||||||
|
`$last=$doc.GoTo(3,1,${endRow}).Bookmarks("\\Line").Range`,
|
||||||
|
"$sel=$doc.Range($first.Start,$last.End);$sel.Select()|Out-Null"
|
||||||
|
].join(";");
|
||||||
|
return selectionResult(filePath, await runSelection(filePath, { start: 0, end: 0 }, body));
|
||||||
|
});
|
||||||
|
|
||||||
|
const wordUnit = (value: unknown): number => {
|
||||||
|
const unit = String(value ?? "word").toLowerCase();
|
||||||
|
return unit === "row" || unit === "line" ? 5 : unit === "section" || unit === "paragraph" ? 4 : 2;
|
||||||
|
};
|
||||||
|
|
||||||
|
register("Word_SetPosition", async (_ctx, node, args) => {
|
||||||
|
const filePath = handlePath(args, node.id, node.componentId);
|
||||||
|
const count = Math.floor(Number(args.length ?? 1) || 0);
|
||||||
|
const unit = wordUnit(args.unit);
|
||||||
|
const body = `$anchor=if(${selectionOf(args).end > selectionOf(args).start ? "$true" : "$false"}){$sel.End}else{$sel.Start};$sel=$doc.Range($anchor,$anchor);[void]$sel.Move(${unit},${count})`;
|
||||||
|
return selectionResult(filePath, await runSelection(filePath, selectionOf(args), body));
|
||||||
|
});
|
||||||
|
|
||||||
|
register("Word_MovePosition", async (_ctx, node, args) => {
|
||||||
|
const filePath = handlePath(args, node.id, node.componentId);
|
||||||
|
const count = Math.floor(Number(args.length ?? 1) || 0);
|
||||||
|
const unit = wordUnit(args.unit);
|
||||||
|
const left = String(args.direction ?? "right").toLowerCase().includes("left");
|
||||||
|
const countValue = left ? -Math.abs(count) : Math.abs(count);
|
||||||
|
const extend = bool(args.isSelect ?? args.shift ?? false);
|
||||||
|
const selection = selectionOf(args);
|
||||||
|
const anchor = left ? selection.start : selection.end;
|
||||||
|
const body = extend
|
||||||
|
? `$sel=$doc.Range(${anchor},${anchor});if(${left ? "$true" : "$false"}){[void]$sel.MoveStart(${unit},${countValue})}else{[void]$sel.MoveEnd(${unit},${countValue})}`
|
||||||
|
: `$sel=$doc.Range(${anchor},${anchor});[void]$sel.Move(${unit},${countValue})`;
|
||||||
|
return selectionResult(filePath, await runSelection(filePath, selection, body));
|
||||||
|
});
|
||||||
|
|
||||||
|
register("Word_PositionToSelect", async (_ctx, node, args) => {
|
||||||
|
const filePath = handlePath(args, node.id, node.componentId);
|
||||||
|
const needle = text(args, ["text", "find", "value"]);
|
||||||
|
if (!needle) throw new RuntimeError("validation_error", "查找文本为空", { nodeId: node.id, componentId: node.componentId });
|
||||||
|
const encodedNeedle = Buffer.from(needle, "utf8").toString("base64");
|
||||||
|
const requested = Math.floor(Number(args.pos_index ?? 1) || 1);
|
||||||
|
const mode = String(args.direction ?? "left").toLowerCase();
|
||||||
|
const body = [
|
||||||
|
`$needle=[Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('${encodedNeedle}'))`,
|
||||||
|
"$search=$doc.Content;$all=@();while($true){",
|
||||||
|
" $candidate=$search.Duplicate;$found=$candidate.Find.Execute($needle)",
|
||||||
|
" if(-not $found){break}",
|
||||||
|
" $all+=$candidate.Duplicate",
|
||||||
|
" if($candidate.End -ge $search.End){break}",
|
||||||
|
" $search.SetRange($candidate.End,$search.End)",
|
||||||
|
"}",
|
||||||
|
"if($all.Count -eq 0){Write-Output '__WORD_NOT_FOUND__';exit 0}",
|
||||||
|
`$ix=${requested};if($ix -le 0){$ix=$all.Count+$ix+1}`,
|
||||||
|
"if($ix -lt 1){$ix=1};if($ix -gt $all.Count){$ix=$all.Count};$match=$all[$ix-1]",
|
||||||
|
mode === "select"
|
||||||
|
? "$sel=$match.Duplicate;$sel.Select()|Out-Null"
|
||||||
|
: `$p=$match.${mode.includes("right") ? "End" : "Start"};$sel=$doc.Range($p,$p);$sel.Select()|Out-Null`
|
||||||
|
].join(";");
|
||||||
|
return selectionResult(filePath, await runSelection(filePath, { start: 0, end: 0 }, body));
|
||||||
|
});
|
||||||
|
|
||||||
|
register("Word_ReadTable", async (_ctx, node, args) => {
|
||||||
|
const filePath = handlePath(args, node.id, node.componentId);
|
||||||
|
const byContent = String(args.find_type ?? "index").toLowerCase().includes("content");
|
||||||
|
const tableIndex = Math.max(1, Math.floor(Number(args.table_index ?? 1) || 1));
|
||||||
|
const content = String(args.table_content ?? "");
|
||||||
|
const encodedContent = Buffer.from(content, "utf8").toString("base64");
|
||||||
|
const selector = byContent
|
||||||
|
? `$needle=[Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('${encodedContent}'));$tableIndex=0;for($i=1;$i -le $doc.Tables.Count;$i++){if($doc.Tables.Item($i).Range.Text.Contains($needle)){$tableIndex=$i;break}}`
|
||||||
|
: `$tableIndex=${tableIndex}`;
|
||||||
|
const body = [
|
||||||
|
selector,
|
||||||
|
"if($tableIndex -lt 1 -or $tableIndex -gt $doc.Tables.Count){Write-Output '__WORD_TABLE_NOT_FOUND__';exit 0}",
|
||||||
|
"$table=$doc.Tables.Item($tableIndex);$rows=@()",
|
||||||
|
"for($r=1;$r -le $table.Rows.Count;$r++){ $row=@(); for($c=1;$c -le $table.Columns.Count;$c++){ try { $text=$table.Cell($r,$c).Range.Text } catch { $text='' }; $clean=''; foreach($ch in $text.ToCharArray()){ $code=[int]$ch; if($code -ne 13 -and $code -ne 7){$clean += $ch} }; $row += $clean }; $rows += ,$row }",
|
||||||
|
"$json=$rows|ConvertTo-Json -Compress -Depth 5",
|
||||||
|
"[Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($json))|Write-Output"
|
||||||
|
].join(";");
|
||||||
|
const output = await powershell(`$word=$null;$doc=$null;try{$word=New-Object -ComObject Word.Application;$word.Visible=$false;$word.DisplayAlerts=0;$doc=$word.Documents.Open(${psString(filePath)});${body}}finally{if($doc -ne $null){try{$doc.Close()}catch{};try{[void][Runtime.InteropServices.Marshal]::ReleaseComObject($doc)}catch{}};if($word -ne $null){try{$word.Quit()}catch{};try{[void][Runtime.InteropServices.Marshal]::ReleaseComObject($word)}catch{}};[GC]::Collect();[GC]::WaitForPendingFinalizers()}`);
|
||||||
|
if (output.includes("__WORD_TABLE_NOT_FOUND__")) throw new RuntimeError("word_table_not_found", "Word 表格不存在", { detail: `path=${filePath};tableIndex=${tableIndex};content=${content}` });
|
||||||
|
const parsed = JSON.parse(Buffer.from(output.trim(), "base64").toString("utf8"));
|
||||||
|
const rows = Array.isArray(parsed) ? parsed : [parsed];
|
||||||
|
return rows.map((row) => {
|
||||||
|
const value = row && typeof row === "object" && Array.isArray((row as { value?: unknown }).value) ? (row as { value: unknown[] }).value : row;
|
||||||
|
return Array.isArray(value) ? value : [value];
|
||||||
|
});
|
||||||
|
});
|
||||||
@ -1,6 +1,7 @@
|
|||||||
import path from "node:path";
|
import path from "node:path";
|
||||||
|
import { access, mkdir } from "node:fs/promises";
|
||||||
import { register } from "../engine/registry";
|
import { register } from "../engine/registry";
|
||||||
import { powershell, psString } from "./desktop";
|
import { powershell, psString } from "../driver/windows/powershell";
|
||||||
|
|
||||||
interface WordHandle { path: string; }
|
interface WordHandle { path: string; }
|
||||||
|
|
||||||
@ -41,4 +42,31 @@ register("Word_Write", async (_ctx, _node, args) => {
|
|||||||
return { path: filePath } satisfies WordHandle;
|
return { path: filePath } satisfies WordHandle;
|
||||||
});
|
});
|
||||||
|
|
||||||
register(["Word_Save", "Word_SaveAs"], async (_ctx, _node, args) => ({ path: handlePath(args) } satisfies WordHandle));
|
async function ensureFile(value: string): Promise<string> {
|
||||||
|
try {
|
||||||
|
await access(value);
|
||||||
|
} catch {
|
||||||
|
throw new Error(`Word文件不存在: ${value}`);
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
register("Word_Save", async (_ctx, _node, args) => {
|
||||||
|
const filePath = await ensureFile(ensure(handlePath(args), "Word文件路径"));
|
||||||
|
// 打开即存:架构上每次操作独立开合 Word 实例,Save 负责兜底刷新并证明文件真实可写
|
||||||
|
await powershell(wordScript(`$doc = $word.Documents.Open(${psString(filePath)}); $doc.Save()`));
|
||||||
|
return { path: filePath } satisfies WordHandle;
|
||||||
|
});
|
||||||
|
|
||||||
|
register("Word_SaveAs", async (_ctx, _node, args) => {
|
||||||
|
const previousPath = await ensureFile(ensure(handlePath(args), "Word文件路径"));
|
||||||
|
const rawDir = text(args, ["fileAddr", "path", "folder", "savePath"]);
|
||||||
|
const rawName = text(args, ["filename", "name"]);
|
||||||
|
// 目录空缺沿用原目录;目录相对时基于原文档目录解析;文件名空缺沿用原名(与 catalog 语义一致)
|
||||||
|
const dir = rawDir ? path.resolve(path.dirname(previousPath), rawDir) : path.dirname(previousPath);
|
||||||
|
const target = path.resolve(dir, rawName || path.basename(previousPath));
|
||||||
|
const withExt = path.extname(target) ? target : `${target}.docx`;
|
||||||
|
await mkdir(path.dirname(withExt), { recursive: true });
|
||||||
|
await powershell(wordScript(`$doc = $word.Documents.Open(${psString(previousPath)}); $doc.SaveAs2(${psString(withExt)}); ${psString(withExt)}`));
|
||||||
|
return { path: withExt } satisfies WordHandle;
|
||||||
|
});
|
||||||
9
packages/rpa-core/src/driver/platform.ts
Normal file
9
packages/rpa-core/src/driver/platform.ts
Normal file
@ -0,0 +1,9 @@
|
|||||||
|
/**
|
||||||
|
* 平台判断唯一入口(dev-conventions 规则 4 的例外白名单)。
|
||||||
|
*
|
||||||
|
* 规范:平台差异只允许出现在两处——`driver/<平台>/` 目录下的驱动实现,以及本文件;
|
||||||
|
* 能力文件不得直接调用 `process.platform`,需要平台分支时从这里取值。
|
||||||
|
*/
|
||||||
|
export const isWindows = process.platform === "win32";
|
||||||
|
export const isMacOS = process.platform === "darwin";
|
||||||
|
export const isLinux = process.platform === "linux";
|
||||||
165
packages/rpa-core/src/driver/types.ts
Normal file
165
packages/rpa-core/src/driver/types.ts
Normal file
@ -0,0 +1,165 @@
|
|||||||
|
/**
|
||||||
|
* AutomationDriver 驱动抽象(任务 3.2,决策点 6:五域接口 + 机械包一层)。
|
||||||
|
*
|
||||||
|
* 接口签名以现有 PowerShell 能力的输入输出为准,不重新发明:
|
||||||
|
* WindowsDriver(driver/windows/windows-driver.ts)是 PowerShell 实现,行为与
|
||||||
|
* 改造前完全一致;引擎创建时注入 driver(createEngine 可注入自定义实现)。
|
||||||
|
*
|
||||||
|
* 约定:driver 只承载「原始注入原语」(单个 SendKeys/一次 SetCursorPos/一次
|
||||||
|
* 窗口操作);带拟真抖动、坐标换算、多步组合的「策略」仍留在能力文件,
|
||||||
|
* 保证机械替换不改任何时序行为。
|
||||||
|
*/
|
||||||
|
|
||||||
|
/** UIA 选择器(element 域类型,能力与 driver 共用) */
|
||||||
|
export interface UiaSelector {
|
||||||
|
window?: string;
|
||||||
|
name?: string;
|
||||||
|
id?: string;
|
||||||
|
/** .NET ControlType 字段名(已映射,如 ComboBox/ListItem/DataGrid) */
|
||||||
|
type?: string;
|
||||||
|
/** 用户输入的原始类型字符串 */
|
||||||
|
typeRaw?: string;
|
||||||
|
className?: string;
|
||||||
|
screen?: { x: number; y: number; w: number; h: number };
|
||||||
|
/** Relative physical rect inside the top-level window. */
|
||||||
|
rel?: { x: number; y: number; w: number; h: number };
|
||||||
|
process?: string;
|
||||||
|
winClass?: string;
|
||||||
|
enriched?: boolean;
|
||||||
|
aiType?: string;
|
||||||
|
aiText?: string;
|
||||||
|
ai?: boolean;
|
||||||
|
/** 从窗口(或桌面根)到目标元素的祖先路径,逐段用稳定属性定位 */
|
||||||
|
path?: Array<{ type?: string; name?: string; id?: string; className?: string; index?: number }>;
|
||||||
|
index: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DriverPoint {
|
||||||
|
x: number;
|
||||||
|
y: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DriverRect {
|
||||||
|
x: number;
|
||||||
|
y: number;
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface WindowInfo extends DriverRect {
|
||||||
|
pid: number;
|
||||||
|
title: string;
|
||||||
|
hwnd: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface WindowSummary {
|
||||||
|
pid: number;
|
||||||
|
title: string;
|
||||||
|
hwnd: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type MouseButton = "left" | "right";
|
||||||
|
|
||||||
|
export interface MouseClickOptions {
|
||||||
|
button?: MouseButton;
|
||||||
|
double?: boolean;
|
||||||
|
/** 每次 down-up 后的停顿毫秒(0 = 不停顿,保持旧桌面能力的紧凑时序) */
|
||||||
|
settleMs?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MouseDragOptions {
|
||||||
|
button?: MouseButton;
|
||||||
|
/** 按下后的按住毫秒 */
|
||||||
|
holdMs?: number;
|
||||||
|
/** 拖到终点后、松开前的停顿毫秒 */
|
||||||
|
releaseDelayMs?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MouseSmoothOptions {
|
||||||
|
/** true=缓入缓出+轻微抖动(拟真),false=匀速直线 */
|
||||||
|
ease?: boolean;
|
||||||
|
/** 1~100,越大越快 */
|
||||||
|
speed?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** mouse 域:坐标一律为物理像素(DIP 换算属于能力层策略) */
|
||||||
|
export interface MouseDriver {
|
||||||
|
moveTo(point: DriverPoint): Promise<void>;
|
||||||
|
click(point: DriverPoint, options?: MouseClickOptions): Promise<void>;
|
||||||
|
drag(from: DriverPoint, to: DriverPoint, options?: MouseDragOptions): Promise<void>;
|
||||||
|
scroll(amount: number): Promise<void>;
|
||||||
|
/** 从当前光标位置出发按速度插值移动到目标 */
|
||||||
|
moveSmooth(target: DriverPoint, options?: MouseSmoothOptions): Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface KeyboardSendOptions {
|
||||||
|
keys: string;
|
||||||
|
intervalMs: number;
|
||||||
|
/** api=后台接口整段发送 / manual=逐字符拟真 / clipboard=剪贴板粘贴 */
|
||||||
|
mode: "api" | "manual" | "clipboard";
|
||||||
|
/** 文本类组件做 SendKeys 字面转义;按键组合组件原样发送 */
|
||||||
|
textMode: boolean;
|
||||||
|
/** 目标窗口标题(非空时先多招式激活再输入,与旧行为一致) */
|
||||||
|
activateTitle?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** keyboard 域 */
|
||||||
|
export interface KeyboardDriver {
|
||||||
|
send(options: KeyboardSendOptions): Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface WindowDriver {
|
||||||
|
/** 按标题模糊匹配窗口并返回矩形信息(GetWindowRect,物理像素) */
|
||||||
|
findByTitle(title: string): Promise<WindowInfo>;
|
||||||
|
/** 简单激活(SetForegroundWindow + 可选最大化),返回窗口信息 */
|
||||||
|
activate(title: string, options?: { maximize?: boolean }): Promise<WindowInfo>;
|
||||||
|
/** 多招式前台激活(失败抛错),返回窗口主标题 */
|
||||||
|
focus(title: string): Promise<string>;
|
||||||
|
/** 前台窗口标题 */
|
||||||
|
foregroundTitle(): Promise<string>;
|
||||||
|
/** 前台窗口矩形左上角(物理像素) */
|
||||||
|
foregroundPosition(): Promise<DriverPoint>;
|
||||||
|
exists(title: string): Promise<boolean>;
|
||||||
|
/** CloseMainWindow 结果("True"/"False") */
|
||||||
|
close(title: string): Promise<string>;
|
||||||
|
/** ShowWindow 命令(6=最小化 3=最大化 9=恢复 5=显示 0=隐藏) */
|
||||||
|
setState(title: string, showCommand: number): Promise<void>;
|
||||||
|
/** 只改尺寸,保持左上角不动 */
|
||||||
|
setSize(title: string, width: number, height: number): Promise<void>;
|
||||||
|
setPos(title: string, rect: DriverRect): Promise<WindowInfo>;
|
||||||
|
list(filter?: { title?: string; regex?: boolean }): Promise<WindowSummary[]>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** screen 域(截图等 System.Drawing 组合操作不在接口面内,能力层可继续用 powershell()) */
|
||||||
|
export interface ScreenDriver {
|
||||||
|
/** 主屏边界(System.Windows.Forms.Screen.PrimaryScreen.Bounds) */
|
||||||
|
size(): Promise<{ x: number; y: number; width: number; height: number }>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ElementFindOptions {
|
||||||
|
timeoutMs?: number;
|
||||||
|
gapMs?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* element 域(UIA)。PowerShell 实现承载在 capabilities/desktop-uia.ts
|
||||||
|
* (ground 原语集),WindowsDriver.element 委托该实现;能力 handler 层
|
||||||
|
* 到 driver 形式的收口在阶段 5/6 按需进行,避免行为回归。
|
||||||
|
*/
|
||||||
|
export interface ElementDriver {
|
||||||
|
parseSelector(raw: unknown): UiaSelector;
|
||||||
|
find(selector: UiaSelector, options?: ElementFindOptions): Promise<Record<string, unknown> | null>;
|
||||||
|
exists(selector: UiaSelector): Promise<boolean>;
|
||||||
|
click(selector: UiaSelector, options?: ElementFindOptions): Promise<void>;
|
||||||
|
setValue(selector: UiaSelector, value: string): Promise<void>;
|
||||||
|
getValue(selector: UiaSelector): Promise<string | Record<string, unknown> | null>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 五域聚合,注入点(RpaEngine.driver) */
|
||||||
|
export interface AutomationDriver {
|
||||||
|
readonly mouse: MouseDriver;
|
||||||
|
readonly keyboard: KeyboardDriver;
|
||||||
|
readonly window: WindowDriver;
|
||||||
|
readonly screen: ScreenDriver;
|
||||||
|
readonly element: ElementDriver;
|
||||||
|
}
|
||||||
85
packages/rpa-core/src/driver/windows/powershell.ts
Normal file
85
packages/rpa-core/src/driver/windows/powershell.ts
Normal file
@ -0,0 +1,85 @@
|
|||||||
|
import { execFile } from "node:child_process";
|
||||||
|
import { promisify } from "node:util";
|
||||||
|
import { RuntimeError } from "../../engine/types";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* PowerShell 进程助手(任务 3.1 从 capabilities/desktop.ts 拆出)。
|
||||||
|
*
|
||||||
|
* 包内所有 PowerShell 调用必须经此助手(开发规范 v1 第二节第 3 条),
|
||||||
|
* 禁止各能力文件自行 execFile("powershell.exe", ...);
|
||||||
|
* 阶段 3.5(可选)将统一切换为常驻 worker,接口签名不变。
|
||||||
|
*/
|
||||||
|
|
||||||
|
const execFileAsync = promisify(execFile);
|
||||||
|
|
||||||
|
function encoded(command: string): string {
|
||||||
|
return Buffer.from(command, "utf16le").toString("base64");
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 从 PowerShell 错误输出里提取 CLIXML 段的真实错误文本(否则显示为乱码转义序列) */
|
||||||
|
function decodeCliXml(detail: string): string {
|
||||||
|
const lines = [...detail.matchAll(/<S S="Error">(.*?)<\/S>/gs)].map((m) => m[1]);
|
||||||
|
if (lines.length === 0) return detail;
|
||||||
|
// _xHHHH_ 是 CLIXML 转义
|
||||||
|
const decoded = lines.join("\n").replace(/_x([0-9A-Fa-f]{4})_/g, (_, h) => String.fromCharCode(parseInt(h, 16)));
|
||||||
|
const parts = decoded.split("\n").map((l) => l.trim());
|
||||||
|
const position = parts.filter((l) => l.startsWith("所在位置")).slice(0, 1);
|
||||||
|
const message = parts.filter(
|
||||||
|
(l) => l && !l.startsWith("所在位置") && !l.startsWith("+") && !l.startsWith("~~") && !l.includes("CategoryInfo") && !l.includes("FullyQualifiedErrorId") && !l.startsWith("<Objs") && !l.startsWith("#<")
|
||||||
|
);
|
||||||
|
const main = message[0] || parts[0] || detail;
|
||||||
|
return position.length ? `${main}(${position[0]})` : main;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 管道字节流解码:优先 UTF-8;出现替换符说明是系统代码页(GBK)写入的(如解析类错误不走 UTF-8 前缀),回退 GB18030 */
|
||||||
|
function decodePipe(buf: Buffer): string {
|
||||||
|
const utf8 = buf.toString("utf8");
|
||||||
|
if (!utf8.includes("\uFFFD")) return utf8;
|
||||||
|
try {
|
||||||
|
return new TextDecoder("gb18030").decode(buf);
|
||||||
|
} catch {
|
||||||
|
return utf8;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function powershell(command: string, timeout = 30000): Promise<string> {
|
||||||
|
try {
|
||||||
|
// Windows PowerShell 的默认输出编码依机器代码页变化;Node 按 UTF-8 解码管道。
|
||||||
|
const utf8 = "$ProgressPreference='SilentlyContinue'; [Console]::OutputEncoding=[System.Text.UTF8Encoding]::new($false); $OutputEncoding=[Console]::OutputEncoding; ";
|
||||||
|
const result = await execFileAsync("powershell.exe", ["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-EncodedCommand", encoded(utf8 + command)], {
|
||||||
|
windowsHide: true,
|
||||||
|
timeout,
|
||||||
|
maxBuffer: 4 * 1024 * 1024,
|
||||||
|
encoding: "buffer"
|
||||||
|
} as never);
|
||||||
|
return decodePipe(result.stdout as unknown as Buffer).trim();
|
||||||
|
} catch (error) {
|
||||||
|
const err = error as Error & { stdout?: unknown; stderr?: unknown };
|
||||||
|
// 脚本主动失败(RpaFail)把完整错误写在 stdout 的 RPA_ERROR:: 标记后,不经 CLIXML 渲染,无截断
|
||||||
|
const stdoutText = Buffer.isBuffer(err.stdout) ? decodePipe(err.stdout) : String(err.stdout ?? "");
|
||||||
|
const marked = stdoutText.match(/RPA_ERROR::([\s\S]*)/);
|
||||||
|
if (marked) {
|
||||||
|
const detail = marked[1].trim();
|
||||||
|
throw new RuntimeError("desktop_command_failed", `桌面操作失败:${detail}`, { detail });
|
||||||
|
}
|
||||||
|
const stderrBuf = Buffer.isBuffer(err.stderr) ? err.stderr : Buffer.from(String(err.stderr ?? ""), "utf8");
|
||||||
|
const stderrText = stderrBuf.length > 0 ? decodePipe(stderrBuf) : err.message;
|
||||||
|
const detail = decodeCliXml(stderrText);
|
||||||
|
throw new RuntimeError("desktop_command_failed", `桌面操作失败:${detail}`, { detail });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function psString(value: unknown): string {
|
||||||
|
return `'${String(value ?? "").replace(/'/g, "''")}'`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 把文本包装成 PowerShell 安全表达式。
|
||||||
|
* ASCII 直接用单引号字符串(撇号内部 '' 转义);含非 ASCII 用 [char]0xXXXX 数值构造——
|
||||||
|
* Windows PowerShell 5.1 会把 ' ' " " 等智能引号当作引号变体解析,直接放进字符串字面量会报「字符串缺少终止符」。
|
||||||
|
* 所有内嵌用户文本的脚本生成点(Set-Clipboard/SendWait/throw 消息等)都应使用本函数而非 psString。
|
||||||
|
*/
|
||||||
|
export function psText(text: string): string {
|
||||||
|
if (/^[\x20-\x7E]*$/.test(text)) return psString(text);
|
||||||
|
return `(-join @(${Array.from(text).map((c) => `[char]0x${c.codePointAt(0)!.toString(16).padStart(4, "0")}`).join(",")}))`;
|
||||||
|
}
|
||||||
269
packages/rpa-core/src/driver/windows/windows-driver.ts
Normal file
269
packages/rpa-core/src/driver/windows/windows-driver.ts
Normal file
@ -0,0 +1,269 @@
|
|||||||
|
import type {
|
||||||
|
AutomationDriver,
|
||||||
|
DriverPoint,
|
||||||
|
DriverRect,
|
||||||
|
ElementDriver,
|
||||||
|
KeyboardSendOptions,
|
||||||
|
MouseClickOptions,
|
||||||
|
MouseDragOptions,
|
||||||
|
MouseSmoothOptions,
|
||||||
|
MouseDriver,
|
||||||
|
KeyboardDriver,
|
||||||
|
ScreenDriver,
|
||||||
|
WindowDriver,
|
||||||
|
WindowInfo,
|
||||||
|
WindowSummary
|
||||||
|
} from "../types";
|
||||||
|
import { powershell, psString, psText } from "./powershell";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* WindowsDriver(任务 3.3):AutomationDriver 的 PowerShell 实现。
|
||||||
|
* 脚本与改造前各能力文件内的片段逐字一致(仅 Add-Type 块收拢到本文件),
|
||||||
|
* 行为不变;坐标一律物理像素,DIP 换算等策略留在能力层。
|
||||||
|
*/
|
||||||
|
|
||||||
|
/** 基础注入原语(原 capabilities/desktop.ts 的 USER32 块,补 SetProcessDpiAwarenessContext:
|
||||||
|
* 原 window-extended.ts 的 WINDOW_API 在同一进程内已设置 per-monitor DPI 感知,
|
||||||
|
* 驱动统一携带该声明,保证 SetCursorPos 全程物理像素语义一致) */
|
||||||
|
const USER32 = `Add-Type @'
|
||||||
|
using System;
|
||||||
|
using System.Runtime.InteropServices;
|
||||||
|
public static class RpaUser32 {
|
||||||
|
[DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y);
|
||||||
|
[DllImport("user32.dll")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra);
|
||||||
|
[DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow();
|
||||||
|
[DllImport("user32.dll", CharSet=CharSet.Unicode)] public static extern int GetWindowText(IntPtr hWnd, System.Text.StringBuilder text, int count);
|
||||||
|
[DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd);
|
||||||
|
[DllImport("user32.dll")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value);
|
||||||
|
}
|
||||||
|
'@; try { [void][RpaUser32]::SetProcessDpiAwarenessContext([IntPtr](-4)) } catch {}`;
|
||||||
|
|
||||||
|
/** 窗口矩形/鼠标原语(原 capabilities/window-extended.ts 的 WINDOW_API 块) */
|
||||||
|
const WINDOW_API = `Add-Type @'
|
||||||
|
using System;
|
||||||
|
using System.Runtime.InteropServices;
|
||||||
|
public static class RpaWindowExtended {
|
||||||
|
[StructLayout(LayoutKind.Sequential)] public struct RECT { public int Left; public int Top; public int Right; public int Bottom; }
|
||||||
|
[StructLayout(LayoutKind.Sequential)] public struct POINT { public int X; public int Y; }
|
||||||
|
[DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr hWnd, out RECT rect);
|
||||||
|
[DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd);
|
||||||
|
[DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr hWnd, int command);
|
||||||
|
[DllImport("user32.dll")] public static extern bool SetWindowPos(IntPtr hWnd, IntPtr after, int x, int y, int width, int height, uint flags);
|
||||||
|
[DllImport("user32.dll")] public static extern bool SetCursorPos(int x, int y);
|
||||||
|
[DllImport("user32.dll")] public static extern bool GetCursorPos(out POINT point);
|
||||||
|
[DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow();
|
||||||
|
[DllImport("user32.dll")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra);
|
||||||
|
[DllImport("user32.dll")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value);
|
||||||
|
}
|
||||||
|
'@; try { [void][RpaWindowExtended]::SetProcessDpiAwarenessContext([IntPtr](-4)) } catch {}`;
|
||||||
|
|
||||||
|
function findWindowScript(title: string): string {
|
||||||
|
// throw 消息内嵌用户文本走 psText(智能引号等非 ASCII 会让 PS5.1 字符串字面量未闭合)
|
||||||
|
return `$p = Get-Process | Where-Object { $_.MainWindowHandle -ne 0 -and $_.MainWindowTitle -like ${psString(`*${title}*`)} } | Select-Object -First 1; if (-not $p) { throw (${psText(`找不到窗口:${title}`)}) }`;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function windowObject(title: string): Promise<WindowInfo> {
|
||||||
|
const output = await powershell(`${WINDOW_API}; ${findWindowScript(title)}; $r = New-Object RpaWindowExtended+RECT; [RpaWindowExtended]::GetWindowRect($p.MainWindowHandle,[ref]$r) | Out-Null; @{pid=$p.Id;title=$p.MainWindowTitle;hwnd=$p.MainWindowHandle.ToInt64();x=$r.Left;y=$r.Top;width=$r.Right-$r.Left;height=$r.Bottom-$r.Top} | ConvertTo-Json -Compress`);
|
||||||
|
return JSON.parse(output);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 把匹配标题的窗口带到前台;多招式依次兜底(应用点击「运行」后自身是前台,
|
||||||
|
* Windows 前台锁定策略会拒绝普通 SetForegroundWindow,最小化再恢复可绕过)。
|
||||||
|
* 失败用 RPA_ERROR 标记写 stdout 后 exit 1:不走 throw/CLIXML,避免控制台宽度截断和编码歧义 */
|
||||||
|
function activateWindowScript(selector: string): string {
|
||||||
|
// 错误消息内嵌用户文本走 psText(窗口标题可能含智能引号/中文标点,直接放进单引号字面量会被 PS5.1 解析炸)
|
||||||
|
const failMsg = psText(`找不到窗口:${selector}。当前可见窗口标题: `);
|
||||||
|
return `function RpaFail($m) { [Console]::Out.WriteLine('RPA_ERROR::' + $m); exit 1 }
|
||||||
|
$p = Get-Process | Where-Object { $_.MainWindowTitle -and $_.MainWindowTitle -like ${psString(`*${selector}*`)} } | Select-Object -First 1
|
||||||
|
if (-not $p) { $vis = (Get-Process | Where-Object { $_.MainWindowTitle } | Select-Object -First 12 | ForEach-Object { '『' + $_.MainWindowTitle + '』' }) -join ' '; RpaFail (${failMsg} + $vis) }
|
||||||
|
Add-Type -MemberDefinition '[DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); [DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr h, int cmd); [DllImport("user32.dll")] public static extern bool IsIconic(IntPtr h); [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr h); [DllImport("user32.dll")] public static extern void keybd_event(byte bVk, byte bScan, uint dwFlags, UIntPtr dwExtraInfo); [DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr hWnd, IntPtr pid); [DllImport("user32.dll")] public static extern bool AttachThreadInput(uint a, uint b, bool attach); [DllImport("user32.dll")] public static extern void SwitchToThisWindow(IntPtr hWnd, bool fAltTab); [DllImport("kernel32.dll")] public static extern uint GetCurrentThreadId();' -Name RpaFocus2 -Namespace WinRpaFocus2 | Out-Null
|
||||||
|
$h = $p.MainWindowHandle
|
||||||
|
if ([WinRpaFocus2.RpaFocus2]::IsIconic($h)) { [void][WinRpaFocus2.RpaFocus2]::ShowWindow($h, 9); Start-Sleep -Milliseconds 150 }
|
||||||
|
if ([WinRpaFocus2.RpaFocus2]::GetForegroundWindow() -ne $h) {
|
||||||
|
[void][WinRpaFocus2.RpaFocus2]::SwitchToThisWindow($h, $true)
|
||||||
|
Start-Sleep -Milliseconds 150
|
||||||
|
}
|
||||||
|
if ([WinRpaFocus2.RpaFocus2]::GetForegroundWindow() -ne $h) {
|
||||||
|
[void][WinRpaFocus2.RpaFocus2]::keybd_event(0x12, 0, 0, [UIntPtr]::Zero)
|
||||||
|
[void][WinRpaFocus2.RpaFocus2]::SetForegroundWindow($h)
|
||||||
|
[void][WinRpaFocus2.RpaFocus2]::keybd_event(0x12, 0, 2, [UIntPtr]::Zero)
|
||||||
|
Start-Sleep -Milliseconds 200
|
||||||
|
}
|
||||||
|
if ([WinRpaFocus2.RpaFocus2]::GetForegroundWindow() -ne $h) {
|
||||||
|
$fgThread = [WinRpaFocus2.RpaFocus2]::GetWindowThreadProcessId([WinRpaFocus2.RpaFocus2]::GetForegroundWindow(), [IntPtr]::Zero)
|
||||||
|
$thisThread = [WinRpaFocus2.RpaFocus2]::GetCurrentThreadId()
|
||||||
|
$attached = [WinRpaFocus2.RpaFocus2]::AttachThreadInput($thisThread, $fgThread, $true)
|
||||||
|
try { [void][WinRpaFocus2.RpaFocus2]::SetForegroundWindow($h) } finally { if ($attached) { [void][WinRpaFocus2.RpaFocus2]::AttachThreadInput($thisThread, $fgThread, $false) } }
|
||||||
|
Start-Sleep -Milliseconds 200
|
||||||
|
}
|
||||||
|
if ([WinRpaFocus2.RpaFocus2]::GetForegroundWindow() -ne $h) {
|
||||||
|
[void][WinRpaFocus2.RpaFocus2]::ShowWindow($h, 6)
|
||||||
|
Start-Sleep -Milliseconds 150
|
||||||
|
[void][WinRpaFocus2.RpaFocus2]::ShowWindow($h, 9)
|
||||||
|
Start-Sleep -Milliseconds 300
|
||||||
|
}
|
||||||
|
if ([WinRpaFocus2.RpaFocus2]::GetForegroundWindow() -ne $h) { RpaFail '无法把窗口带到前台,请确认窗口未被关闭或全屏占用' }`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 把 SendKeys 特殊字符按字面转义(+^%~(){}[] → {+} 等);SendKeys 用花括号表示字面量 */
|
||||||
|
function sendKeysEscape(ch: string): string {
|
||||||
|
return "+^%~(){}[]".includes(ch) ? `{${ch}}` : ch;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 单字符/短字面量别名(语义标注用),实现同 psText(历史调用点保留) */
|
||||||
|
function psCharLiteral(text: string): string {
|
||||||
|
return psText(text);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 生成从当前位置到目标的插值路径点(JS 侧算好,PS 侧只执行循环)(原 window-extended.ts 的 mousePath) */
|
||||||
|
function mousePath(from: DriverPoint, to: DriverPoint, mode: string, speed: number): Array<[number, number]> {
|
||||||
|
const dx = to.x - from.x;
|
||||||
|
const dy = to.y - from.y;
|
||||||
|
const distance = Math.hypot(dx, dy);
|
||||||
|
if (distance < 2) return [[to.x, to.y]];
|
||||||
|
const stepCount = Math.max(6, Math.min(50, Math.round(distance / 12)));
|
||||||
|
const points: Array<[number, number]> = [];
|
||||||
|
for (let i = 1; i <= stepCount; i++) {
|
||||||
|
const t = i / stepCount;
|
||||||
|
// 模拟人工:缓入缓出 + 轻微垂直抖动;匀速直线:线性
|
||||||
|
const ease = mode === "0" ? (t < 0.5 ? 2 * t * t : 1 - Math.pow(-2 * t + 2, 2) / 2) : t;
|
||||||
|
const jitter = mode === "0" ? Math.sin(t * Math.PI) * (Math.sin(i * 12.9898) * 43758.5453 % 1) * 3 : 0;
|
||||||
|
points.push([
|
||||||
|
Math.round(from.x + dx * ease - (dy / distance) * jitter),
|
||||||
|
Math.round(from.y + dy * ease + (dx / distance) * jitter)
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
points.push([to.x, to.y]);
|
||||||
|
return points;
|
||||||
|
}
|
||||||
|
|
||||||
|
const windowsMouse: MouseDriver = {
|
||||||
|
async moveTo(point) {
|
||||||
|
await powershell(`${USER32}; [RpaUser32]::SetCursorPos(${point.x},${point.y})`);
|
||||||
|
},
|
||||||
|
async click(point, options: MouseClickOptions = {}) {
|
||||||
|
const right = options.button === "right";
|
||||||
|
const down = right ? 0x0008 : 0x0002;
|
||||||
|
const up = right ? 0x0010 : 0x0004;
|
||||||
|
const repeats = options.double ? 2 : 1;
|
||||||
|
const settle = options.settleMs && options.settleMs > 0 ? `; Start-Sleep -Milliseconds ${Math.round(options.settleMs)}` : "";
|
||||||
|
await powershell(`${USER32}; [RpaUser32]::SetCursorPos(${point.x},${point.y});${Array.from({ length: repeats }, () => ` [RpaUser32]::mouse_event(${down},0,0,0,[UIntPtr]::Zero); [RpaUser32]::mouse_event(${up},0,0,0,[UIntPtr]::Zero)${settle}`).join(";")}`);
|
||||||
|
},
|
||||||
|
async drag(from, to, options: MouseDragOptions = {}) {
|
||||||
|
const right = options.button === "right";
|
||||||
|
const down = right ? 0x0008 : 0x0002;
|
||||||
|
const up = right ? 0x0010 : 0x0004;
|
||||||
|
const hold = Math.max(0, Math.round(options.holdMs ?? 80));
|
||||||
|
const release = Math.max(0, Math.round(options.releaseDelayMs ?? 80));
|
||||||
|
await powershell(`${USER32}; [RpaUser32]::SetCursorPos(${from.x},${from.y}); [RpaUser32]::mouse_event(${down},0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds ${hold}; [RpaUser32]::SetCursorPos(${to.x},${to.y});${release > 0 ? ` Start-Sleep -Milliseconds ${release};` : ""} [RpaUser32]::mouse_event(${up},0,0,0,[UIntPtr]::Zero)`);
|
||||||
|
},
|
||||||
|
async scroll(amount) {
|
||||||
|
await powershell(`${USER32}; [RpaUser32]::mouse_event(0x0800,0,0,${amount},[UIntPtr]::Zero)`);
|
||||||
|
},
|
||||||
|
async moveSmooth(target, options: MouseSmoothOptions = {}) {
|
||||||
|
const ease = options.ease !== false;
|
||||||
|
const speed = Math.min(100, Math.max(1, options.speed ?? 35));
|
||||||
|
const delay = Math.min(80, Math.max(4, Math.round(1500 / speed)));
|
||||||
|
// 从当前光标位置出发,按速度插值(路径全程物理像素,与目标同一坐标系)
|
||||||
|
const output = await powershell(`${WINDOW_API}; $cur = [RpaWindowExtended+POINT]::new(); [void][RpaWindowExtended]::GetCursorPos([ref]$cur); "$($cur.X),$($cur.Y)"`);
|
||||||
|
const [cx, cy] = output.split(",").map(Number);
|
||||||
|
const from: DriverPoint = Number.isFinite(cx) && Number.isFinite(cy) ? { x: cx, y: cy } : { x: 0, y: 0 };
|
||||||
|
const path = mousePath(from, target, ease ? "0" : "1", speed);
|
||||||
|
const ptsLiteral = path.map(([px, py]) => `@(${px},${py})`).join(",");
|
||||||
|
await powershell(`${WINDOW_API}; $pts = @(${ptsLiteral}); foreach ($pt in $pts) { [void][RpaWindowExtended]::SetCursorPos($pt[0], $pt[1]); Start-Sleep -Milliseconds ${delay} }`);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const windowsKeyboard: KeyboardDriver = {
|
||||||
|
async send(options: KeyboardSendOptions) {
|
||||||
|
// 三种输入方式:脚本尽量简单,转义逻辑放 JS 侧,避免 PowerShell 字符串拼接的语法陷阱
|
||||||
|
let send: string;
|
||||||
|
if (options.mode === "clipboard") {
|
||||||
|
// 剪贴板粘贴:原文进剪贴板 + Ctrl+V,不做 SendKeys 转义
|
||||||
|
send = `Set-Clipboard -Value ${psText(options.keys)}; Start-Sleep -Milliseconds 120; [System.Windows.Forms.SendKeys]::SendWait('^v')`;
|
||||||
|
} else if (options.mode === "manual" && options.textMode) {
|
||||||
|
// 人工模拟:逐字符;JS 侧把每个字符转义成 SendKeys 字面量,PS 侧只做数组循环
|
||||||
|
const escapedChars = Array.from(options.keys).map((ch) => sendKeysEscape(ch));
|
||||||
|
const arr = escapedChars.map((c) => psCharLiteral(c)).join(",");
|
||||||
|
send = `$chars = @(${arr}); foreach ($c in $chars) { [System.Windows.Forms.SendKeys]::SendWait([string]$c); Start-Sleep -Milliseconds ${options.intervalMs} }`;
|
||||||
|
} else {
|
||||||
|
// 后台接口 / 按键组合 / 人工模拟(按键组合组件):整段发送,文本特殊字符转义
|
||||||
|
const escaped = options.textMode ? Array.from(options.keys).map(sendKeysEscape).join("") : options.keys;
|
||||||
|
send = `[System.Windows.Forms.SendKeys]::SendWait(${psText(escaped)})`;
|
||||||
|
}
|
||||||
|
const activate = options.activateTitle ? `${activateWindowScript(options.activateTitle)}; Start-Sleep -Milliseconds 300; ` : "";
|
||||||
|
await powershell(`${USER32}; ${activate}Add-Type -AssemblyName System.Windows.Forms; ${send}`);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const windowsWindow: WindowDriver = {
|
||||||
|
findByTitle: (title) => windowObject(title),
|
||||||
|
async activate(title, options: { maximize?: boolean } = {}) {
|
||||||
|
const maximize = options.maximize === true;
|
||||||
|
await powershell(`${WINDOW_API}; ${findWindowScript(title)}; [RpaWindowExtended]::SetForegroundWindow($p.MainWindowHandle) | Out-Null; ${maximize ? "[RpaWindowExtended]::ShowWindow($p.MainWindowHandle,3) | Out-Null" : ""}`);
|
||||||
|
return windowObject(title);
|
||||||
|
},
|
||||||
|
async focus(title) {
|
||||||
|
return powershell(`${USER32}\n${activateWindowScript(title)}\n$p.MainWindowTitle`);
|
||||||
|
},
|
||||||
|
async foregroundTitle() {
|
||||||
|
return powershell(`${USER32}; $sb = New-Object System.Text.StringBuilder 512; [RpaUser32]::GetWindowText([RpaUser32]::GetForegroundWindow(),$sb,$sb.Capacity) | Out-Null; $sb.ToString()`);
|
||||||
|
},
|
||||||
|
async foregroundPosition(): Promise<DriverPoint> {
|
||||||
|
const output = await powershell(`${WINDOW_API}; $fw = [RpaWindowExtended]::GetForegroundWindow(); $r = New-Object RpaWindowExtended+RECT; [void][RpaWindowExtended]::GetWindowRect($fw, [ref]$r); "$($r.Left),$($r.Top)"`);
|
||||||
|
const [x, y] = output.split(",").map(Number);
|
||||||
|
return { x, y };
|
||||||
|
},
|
||||||
|
async exists(title) {
|
||||||
|
return (await powershell(`$p = Get-Process | Where-Object { $_.MainWindowTitle -and $_.MainWindowTitle -like ${psString(`*${title}*`)} } | Select-Object -First 1; [bool]$p`)).toLowerCase() === "true";
|
||||||
|
},
|
||||||
|
async close(title): Promise<string> {
|
||||||
|
return powershell(`$p = Get-Process | Where-Object { $_.MainWindowTitle -and $_.MainWindowTitle -like ${psString(`*${title}*`)} } | Select-Object -First 1; if (-not $p) { throw '找不到窗口' }; $p.CloseMainWindow() | Out-Null; $true`);
|
||||||
|
},
|
||||||
|
async setState(title, showCommand): Promise<void> {
|
||||||
|
await powershell(`${WINDOW_API}; ${findWindowScript(title)}; [RpaWindowExtended]::ShowWindow($p.MainWindowHandle,${showCommand}) | Out-Null`);
|
||||||
|
},
|
||||||
|
async setSize(title, width, height) {
|
||||||
|
await powershell(`${WINDOW_API}; ${findWindowScript(title)}; $r = New-Object RpaWindowExtended+RECT; [RpaWindowExtended]::GetWindowRect($p.MainWindowHandle,[ref]$r) | Out-Null; [RpaWindowExtended]::SetWindowPos($p.MainWindowHandle,[IntPtr]::Zero,$r.Left,$r.Top,${width},${height},4) | Out-Null`);
|
||||||
|
},
|
||||||
|
async setPos(title, rect: DriverRect): Promise<WindowInfo> {
|
||||||
|
if (!(rect.width > 0 && rect.height > 0)) throw new Error("窗口尺寸必须大于 0");
|
||||||
|
await powershell(`${WINDOW_API}; ${findWindowScript(title)}; [RpaWindowExtended]::SetWindowPos($p.MainWindowHandle,[IntPtr]::Zero,${rect.x},${rect.y},${rect.width},${rect.height},4) | Out-Null`);
|
||||||
|
return { ...(await windowObject(title)), x: rect.x, y: rect.y, width: rect.width, height: rect.height };
|
||||||
|
},
|
||||||
|
async list(filter: { title?: string; regex?: boolean } = {}) {
|
||||||
|
const title = filter.title ?? "";
|
||||||
|
const regex = filter.regex === true;
|
||||||
|
const output = await powershell(`Get-Process | Where-Object { $_.MainWindowHandle -ne 0 ${title ? (regex ? `-and $_.MainWindowTitle -match ${psString(title)}` : `-and $_.MainWindowTitle -like ${psString(`*${title}*`)}`) : ""} } | ForEach-Object { @{pid=$_.Id;title=$_.MainWindowTitle;hwnd=$_.MainWindowHandle.ToInt64()} } | ConvertTo-Json -Compress`);
|
||||||
|
if (!output) return [] as WindowSummary[];
|
||||||
|
const parsed = JSON.parse(output);
|
||||||
|
return (Array.isArray(parsed) ? parsed : [parsed]) as WindowSummary[];
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const windowsScreen: ScreenDriver = {
|
||||||
|
async size() {
|
||||||
|
const output = await powershell("Add-Type -AssemblyName System.Windows.Forms; $b=[System.Windows.Forms.Screen]::PrimaryScreen.Bounds; @{width=$b.Width;height=$b.Height;x=$b.X;y=$b.Y} | ConvertTo-Json -Compress");
|
||||||
|
return JSON.parse(output);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
/** element 域实现承载在 capabilities/desktop-uia.ts(阶段 3.3 说明),此处运行时注入,避免 driver → capabilities 的模块依赖 */
|
||||||
|
let elementImplementation: ElementDriver | undefined;
|
||||||
|
export function setElementDriverImplementation(impl: ElementDriver): void {
|
||||||
|
elementImplementation = impl;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class WindowsDriver implements AutomationDriver {
|
||||||
|
readonly mouse = windowsMouse;
|
||||||
|
readonly keyboard = windowsKeyboard;
|
||||||
|
readonly window = windowsWindow;
|
||||||
|
readonly screen = windowsScreen;
|
||||||
|
|
||||||
|
get element(): ElementDriver {
|
||||||
|
if (!elementImplementation) {
|
||||||
|
throw new Error("ElementDriver 实现未注册:请 import @rpa/core/capabilities(desktop-uia 模块自动注册)");
|
||||||
|
}
|
||||||
|
return elementImplementation;
|
||||||
|
}
|
||||||
|
}
|
||||||
101
packages/rpa-core/src/engine/engine.ts
Normal file
101
packages/rpa-core/src/engine/engine.ts
Normal file
@ -0,0 +1,101 @@
|
|||||||
|
import type { Client } from "basic-ftp";
|
||||||
|
import type { Workbook as ExcelJSWorkbook } from "exceljs";
|
||||||
|
import { WindowsDriver } from "../driver/windows/windows-driver";
|
||||||
|
import type { AutomationDriver } from "../driver/types";
|
||||||
|
import type { CatalogComponent, FlowNode } from "../types";
|
||||||
|
import { getHandler } from "./registry";
|
||||||
|
import { RuntimeError, type RunContext } from "./types";
|
||||||
|
|
||||||
|
/** 人为拟真操作的开关与随机参数(desktop 能力的进程级设置)。 */
|
||||||
|
export interface ManualOperationSettings {
|
||||||
|
enabled: boolean;
|
||||||
|
randomMove: boolean;
|
||||||
|
randomClick: boolean;
|
||||||
|
randomSleep: boolean;
|
||||||
|
minMs: number;
|
||||||
|
maxMs: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 引擎实例状态容器(任务 1.3,R4:模块级全局状态收拢)。
|
||||||
|
*
|
||||||
|
* 折中方案(决策记录 2026-11-04):能力文件的 register() 调用与全局注册表保持不变,
|
||||||
|
* 但 Excel 缓存、数据库连接、FTP 客户端等进程级可变状态统一挂到引擎实例上。
|
||||||
|
* 包内各能力模块通过 defaultEngine 访问;宿主/CLI 可用 createEngine() 产生隔离实例,
|
||||||
|
* 或拿到 defaultEngine 做测试间重置。
|
||||||
|
*/
|
||||||
|
export class RpaEngine {
|
||||||
|
/** excel-store:路径 → ExcelJS 工作簿共享缓存 */
|
||||||
|
readonly excelCache = new Map<string, ExcelJSWorkbook>();
|
||||||
|
/** office.ts:Excel 句柄 id → 工作簿 */
|
||||||
|
readonly excelHandles = new Map<string, ExcelJSWorkbook>();
|
||||||
|
/** remaining-core:连接 id → 数据库连接(mysql2/pg/mssql) */
|
||||||
|
readonly dbConnections = new Map<string, unknown>();
|
||||||
|
/** ftp.ts:当前 FTP 连接 */
|
||||||
|
ftpClient?: Client;
|
||||||
|
/** ocr-caps.ts:懒加载的 OCR 实例 */
|
||||||
|
ocrInstance?: unknown;
|
||||||
|
/** desktop.ts:人为拟真操作设置 */
|
||||||
|
manualOperation: ManualOperationSettings = {
|
||||||
|
enabled: false,
|
||||||
|
randomMove: true,
|
||||||
|
randomClick: true,
|
||||||
|
randomSleep: true,
|
||||||
|
minMs: 1000,
|
||||||
|
maxMs: 3000
|
||||||
|
};
|
||||||
|
/** 桌面自动化驱动:按系统自动选(迁移指南五件小事 #2)。win32 用 PowerShell 版 WindowsDriver;
|
||||||
|
* 其他平台尚未实现,访问时抛 platform_unsupported 明确报错(惰性创建,避免 import 即炸) */
|
||||||
|
private _driver?: AutomationDriver;
|
||||||
|
get driver(): AutomationDriver {
|
||||||
|
if (!this._driver) {
|
||||||
|
if (process.platform === "win32") {
|
||||||
|
this._driver = new WindowsDriver();
|
||||||
|
} else {
|
||||||
|
throw new RuntimeError(
|
||||||
|
"platform_unsupported",
|
||||||
|
`该平台暂不支持桌面自动化驱动(当前:${process.platform})`,
|
||||||
|
{ recommendation: "桌面/键鼠/窗口能力目前仅支持 Windows;macOS/Linux 驱动在迁移指南三端计划中规划" }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return this._driver;
|
||||||
|
}
|
||||||
|
/** 注入驱动实现(测试替身或宿主适配器);注入后不再走平台自动选择 */
|
||||||
|
setDriver(driver: AutomationDriver): void {
|
||||||
|
this._driver = driver;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 单能力直接调用(ADR-R1 语义真实化):CLI/宿主脚本不再裸摸全局 getHandler,
|
||||||
|
* 而是经引擎入口调用。注意 handler 仍解析自进程注册表(ADR-R1 短期形态)。
|
||||||
|
*/
|
||||||
|
async execute(
|
||||||
|
componentId: string,
|
||||||
|
args: Record<string, unknown>,
|
||||||
|
ctx: RunContext,
|
||||||
|
node?: FlowNode,
|
||||||
|
component?: CatalogComponent
|
||||||
|
): Promise<unknown> {
|
||||||
|
const handler = getHandler(componentId);
|
||||||
|
if (!handler) {
|
||||||
|
throw new RuntimeError("capability_not_implemented", `能力 ${componentId} 未注册`, { componentId });
|
||||||
|
}
|
||||||
|
const resolvedNode: FlowNode = node ?? { id: "direct", componentId, position: { x: 0, y: 0 }, values: {}, options: {} };
|
||||||
|
return handler(ctx, resolvedNode, args, component as CatalogComponent);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface EngineOptions {
|
||||||
|
/** 注入桌面自动化驱动(测试替身或宿主适配器);缺省按 process.platform 自动选择 */
|
||||||
|
driver?: AutomationDriver;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createEngine(options?: EngineOptions): RpaEngine {
|
||||||
|
const engine = new RpaEngine();
|
||||||
|
if (options?.driver) engine.setDriver(options.driver);
|
||||||
|
return engine;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 能力模块内部使用的默认引擎实例(与全局注册表配套的进程级单例)。 */
|
||||||
|
export const defaultEngine = createEngine();
|
||||||
53
packages/rpa-core/src/engine/error-codes.ts
Normal file
53
packages/rpa-core/src/engine/error-codes.ts
Normal file
@ -0,0 +1,53 @@
|
|||||||
|
/**
|
||||||
|
* 稳定错误码枚举(任务 4.3,R9)。
|
||||||
|
*
|
||||||
|
* 契约:RuntimeError 的 code 必须取自本清单;新增能力需要新错误码时先在此登记。
|
||||||
|
* message 保持中文(给人看),机器消费走 code 与 options 里的结构化字段
|
||||||
|
* (recommendation / nodeId / componentId / arg / detail)。
|
||||||
|
* 宿主侧另有自己的扩展码(browser 会话 / 拾取 / vision-lab 等),不进本清单。
|
||||||
|
*/
|
||||||
|
|
||||||
|
export const RUNTIME_ERROR_CODES = [
|
||||||
|
// 通用
|
||||||
|
"validation_error",
|
||||||
|
"timeout",
|
||||||
|
"cancelled",
|
||||||
|
"capability_not_implemented",
|
||||||
|
"expression_error",
|
||||||
|
"user_exception",
|
||||||
|
"not_found",
|
||||||
|
"network_error",
|
||||||
|
"http_error",
|
||||||
|
"external_service_failed",
|
||||||
|
// 桌面驱动
|
||||||
|
"desktop_command_failed",
|
||||||
|
"desktop_element_not_found",
|
||||||
|
"desktop_error",
|
||||||
|
// 浏览器
|
||||||
|
"browser_error",
|
||||||
|
"browser_disconnected",
|
||||||
|
"tab_unavailable",
|
||||||
|
// AI 服务
|
||||||
|
"ai_not_configured",
|
||||||
|
"ai_request_timeout",
|
||||||
|
"ai_rate_limited",
|
||||||
|
"ai_request_failed",
|
||||||
|
// 视觉
|
||||||
|
"cv_not_found",
|
||||||
|
// Office
|
||||||
|
"excel_com_failed",
|
||||||
|
"word_table_not_found",
|
||||||
|
"word_text_not_found",
|
||||||
|
"outlook_unavailable",
|
||||||
|
// 其他
|
||||||
|
"platform_unsupported",
|
||||||
|
"decryption_failed",
|
||||||
|
"host_unavailable"
|
||||||
|
] as const;
|
||||||
|
|
||||||
|
export type ErrorCode = (typeof RUNTIME_ERROR_CODES)[number];
|
||||||
|
|
||||||
|
/** 运行期校验(audit 测试用):code 必须在清单内 */
|
||||||
|
export function isKnownErrorCode(code: string): boolean {
|
||||||
|
return (RUNTIME_ERROR_CODES as readonly string[]).includes(code);
|
||||||
|
}
|
||||||
117
packages/rpa-core/src/engine/graph-validator.ts
Normal file
117
packages/rpa-core/src/engine/graph-validator.ts
Normal file
@ -0,0 +1,117 @@
|
|||||||
|
import type { FlowDefinition } from "../types";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 流程图静态校验(任务:审查报告 P0-03)。
|
||||||
|
* 在执行前拦截结构性错误,避免运行期以诡异方式失败。
|
||||||
|
* 只做纯结构检查,不读目录/注册表(组件是否存在由运行时校验并给出友好错误)。
|
||||||
|
*/
|
||||||
|
|
||||||
|
export interface FlowIssue {
|
||||||
|
severity: "error" | "warning";
|
||||||
|
code: string;
|
||||||
|
nodeId?: string;
|
||||||
|
edgeId?: string;
|
||||||
|
message: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
const CONTROL_FLOW_COMPONENTS = new Set([
|
||||||
|
"logic_control_if_else",
|
||||||
|
"logic_control_count_for",
|
||||||
|
"logic_control_array_for_v3",
|
||||||
|
"logic_control_dictionary_for_v1",
|
||||||
|
"logic_control_while",
|
||||||
|
"infinite_loop",
|
||||||
|
"logic_control_try_catch"
|
||||||
|
]);
|
||||||
|
|
||||||
|
export function validateFlowGraph(flow: FlowDefinition): FlowIssue[] {
|
||||||
|
const issues: FlowIssue[] = [];
|
||||||
|
const nodeIds = new Set<string>();
|
||||||
|
|
||||||
|
for (const node of flow.nodes) {
|
||||||
|
if (nodeIds.has(node.id)) {
|
||||||
|
issues.push({ severity: "error", code: "duplicate_node_id", nodeId: node.id, message: `节点 ID 重复:${node.id}` });
|
||||||
|
}
|
||||||
|
nodeIds.add(node.id);
|
||||||
|
if (node.parentId && !flow.nodes.some((candidate) => candidate.id === node.parentId)) {
|
||||||
|
issues.push({
|
||||||
|
severity: "error",
|
||||||
|
code: "orphan_child",
|
||||||
|
nodeId: node.id,
|
||||||
|
message: `节点「${node.label || node.id}」挂在不存在的父节点 ${node.parentId} 下`
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const edge of flow.edges) {
|
||||||
|
if (!nodeIds.has(edge.source)) {
|
||||||
|
issues.push({ severity: "error", code: "dangling_edge", edgeId: edge.id, message: `连线 ${edge.id} 的起点 ${edge.source} 不存在于画布` });
|
||||||
|
}
|
||||||
|
if (!nodeIds.has(edge.target)) {
|
||||||
|
issues.push({ severity: "error", code: "dangling_edge", edgeId: edge.id, message: `连线 ${edge.id} 的终点 ${edge.target} 不存在于画布` });
|
||||||
|
}
|
||||||
|
if (edge.source === edge.target && nodeIds.has(edge.source)) {
|
||||||
|
issues.push({
|
||||||
|
severity: "error",
|
||||||
|
code: "self_loop",
|
||||||
|
edgeId: edge.id,
|
||||||
|
nodeId: edge.source,
|
||||||
|
message: `节点「${edge.source}」连接到自己,执行时会立即跳过`
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 顶层图环检测:执行引擎从起点沿边游走且不重访,环意味着存在永远不会执行的分支或意图不明的回边
|
||||||
|
const top = flow.nodes.filter((node) => !node.parentId);
|
||||||
|
const topIds = new Set(top.map((node) => node.id));
|
||||||
|
const edges = flow.edges.filter((edge) => topIds.has(edge.source) && topIds.has(edge.target));
|
||||||
|
const adjacency = new Map<string, string[]>();
|
||||||
|
for (const edge of edges) {
|
||||||
|
if (edge.source === edge.target) continue;
|
||||||
|
adjacency.set(edge.source, [...(adjacency.get(edge.source) || []), edge.target]);
|
||||||
|
}
|
||||||
|
const incoming = new Set(edges.map((edge) => edge.target));
|
||||||
|
const start =
|
||||||
|
top.find((node) => !incoming.has(node.id) && node.componentId.startsWith("Trigger.")) ||
|
||||||
|
top.find((node) => !incoming.has(node.id)) ||
|
||||||
|
top[0];
|
||||||
|
|
||||||
|
if (start) {
|
||||||
|
const state = new Map<string, "visiting" | "done">();
|
||||||
|
const reachable = new Set<string>();
|
||||||
|
const visit = (nodeId: string, path: string[]): void => {
|
||||||
|
if (state.get(nodeId) === "done") return;
|
||||||
|
if (state.get(nodeId) === "visiting") {
|
||||||
|
const cycleStart = path.indexOf(nodeId);
|
||||||
|
const cycle = [...path.slice(cycleStart), nodeId].join(" → ");
|
||||||
|
issues.push({ severity: "error", code: "cycle", nodeId, message: `顶层流程存在环,环上的节点执行顺序不明确:${cycle}` });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
state.set(nodeId, "visiting");
|
||||||
|
reachable.add(nodeId);
|
||||||
|
for (const next of adjacency.get(nodeId) || []) visit(next, [...path, nodeId]);
|
||||||
|
state.set(nodeId, "done");
|
||||||
|
};
|
||||||
|
visit(start.id, [start.id]);
|
||||||
|
|
||||||
|
for (const node of top) {
|
||||||
|
if (!reachable.has(node.id)) {
|
||||||
|
const knownContainer = CONTROL_FLOW_COMPONENTS.has(node.componentId);
|
||||||
|
if (!knownContainer && !flow.edges.some((edge) => edge.target === node.id)) {
|
||||||
|
issues.push({
|
||||||
|
severity: "warning",
|
||||||
|
code: "unreachable_node",
|
||||||
|
nodeId: node.id,
|
||||||
|
message: `节点「${node.label || node.id}」没有从起点可达的连线,执行时不会运行`
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return issues;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function hasBlockingIssues(issues: FlowIssue[]): boolean {
|
||||||
|
return issues.some((issue) => issue.severity === "error");
|
||||||
|
}
|
||||||
110
packages/rpa-core/src/engine/host-bridge.ts
Normal file
110
packages/rpa-core/src/engine/host-bridge.ts
Normal file
@ -0,0 +1,110 @@
|
|||||||
|
import type { FlowDefinition } from "../types";
|
||||||
|
import { RuntimeError } from "./types";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 宿主桥(任务 2.1 正式化)。
|
||||||
|
*
|
||||||
|
* 包内能力需要的宿主能力(子流程仓库 / Electron 窗口 / DPI 换算 / 应用配置)
|
||||||
|
* 通过本接口注入,包本身禁止 import "electron" 或宿主模块(见包内依赖检查脚本)。
|
||||||
|
* 宿主启动时调用 setSubflowSource() / setHostBridge()。
|
||||||
|
*/
|
||||||
|
|
||||||
|
export interface SubflowSource {
|
||||||
|
listFlows(): Promise<Array<{ id: string; name: string }>>;
|
||||||
|
getFlow(id: string): Promise<FlowDefinition>;
|
||||||
|
}
|
||||||
|
|
||||||
|
let subflowSource: SubflowSource | undefined;
|
||||||
|
|
||||||
|
export function setSubflowSource(source: SubflowSource): void {
|
||||||
|
subflowSource = source;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getSubflowSource(): SubflowSource | undefined {
|
||||||
|
return subflowSource;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 人机交互表单窗口的最小面(能力只用到这些成员,宿主用 BrowserWindow 适配) */
|
||||||
|
export interface HostDialogWindow {
|
||||||
|
isDestroyed(): boolean;
|
||||||
|
destroy(): void;
|
||||||
|
loadURL(url: string): Promise<void>;
|
||||||
|
on(event: "closed", listener: () => void): void;
|
||||||
|
/** 在窗口页面里执行 JS 并取回结果(宿主侧以 userGesture=true 执行) */
|
||||||
|
executeJavaScript(script: string): Promise<unknown>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HostPoint {
|
||||||
|
x: number;
|
||||||
|
y: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** AI 服务配置(宿主 settings.aiConfig) */
|
||||||
|
export interface HostAiConfig {
|
||||||
|
baseUrl: string;
|
||||||
|
apiKey: string;
|
||||||
|
model: string;
|
||||||
|
visionModel: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 显示器信息(视觉能力的屏幕枚举用;id 为宿主侧稳定标识字符串) */
|
||||||
|
export interface HostDisplay {
|
||||||
|
id: string;
|
||||||
|
bounds: { x: number; y: number; width: number; height: number };
|
||||||
|
scaleFactor: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 屏幕服务(迁移阶段1-A9):视觉能力原本经动态 import("electron") 拿
|
||||||
|
* desktopCapturer/screen,现改为宿主注入,Core 回归零 Electron 依赖。
|
||||||
|
*/
|
||||||
|
export interface HostScreenService {
|
||||||
|
/** 全部显示器 */
|
||||||
|
listDisplays(): HostDisplay[];
|
||||||
|
/** 主显示器 */
|
||||||
|
primaryDisplay(): HostDisplay;
|
||||||
|
/** DIP 屏幕坐标 → 物理像素 */
|
||||||
|
dipToScreenPoint(point: HostPoint): HostPoint;
|
||||||
|
/** 与该 DIP 矩形相交面积最大的显示器 */
|
||||||
|
displayMatching(rect: { x: number; y: number; width: number; height: number }): HostDisplay;
|
||||||
|
/**
|
||||||
|
* 抓取指定显示器当前画面,像素尺寸尽量接近 want 值,返回 PNG 字节。
|
||||||
|
* 无画面/抓取失败时抛错(调用方决定回退策略)。
|
||||||
|
*/
|
||||||
|
captureDisplay(displayId: string, wantWidth: number, wantHeight: number): Promise<Uint8Array>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 宿主能力桥最小面(决策点 4):dialog 窗口工厂 / DPI 坐标换算 / 配置读取 / 屏幕服务 */
|
||||||
|
export interface HostBridge {
|
||||||
|
/** 创建人机交互表单窗口(human-dialog 用) */
|
||||||
|
createFormWindow(options: { title: string }): HostDialogWindow;
|
||||||
|
/** DIP 逻辑坐标 → 物理像素(window-extended 的鼠标/拖拽坐标换算) */
|
||||||
|
dipToScreenPoint(point: HostPoint): HostPoint;
|
||||||
|
/** 物理像素 → DIP 逻辑坐标 */
|
||||||
|
screenToDipPoint(point: HostPoint): HostPoint;
|
||||||
|
/** 读取 AI 服务配置(ai-service 用) */
|
||||||
|
loadAiConfig(): HostAiConfig;
|
||||||
|
/** 屏幕服务(vision/cv-vision 截屏与多屏几何);未注入时视觉能力报 host_unavailable */
|
||||||
|
screen?: HostScreenService;
|
||||||
|
}
|
||||||
|
|
||||||
|
let hostBridge: HostBridge | undefined;
|
||||||
|
|
||||||
|
export function setHostBridge(bridge: HostBridge): void {
|
||||||
|
hostBridge = bridge;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getHostBridge(): HostBridge | undefined {
|
||||||
|
return hostBridge;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 视觉能力取屏幕服务的统一入口:未注入宿主桥/屏幕服务时抛稳定错误码 */
|
||||||
|
export function requireScreenService(): HostScreenService {
|
||||||
|
const bridge = getHostBridge();
|
||||||
|
if (!bridge?.screen) {
|
||||||
|
throw new RuntimeError("host_unavailable", "宿主未注入屏幕服务(HostBridge.screen),截屏/视觉能力不可用", {
|
||||||
|
recommendation: "宿主启动时需调用 setHostBridge 并提供 screen 服务(参考 demo 的 src/main/index.ts)"
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return bridge.screen;
|
||||||
|
}
|
||||||
89
packages/rpa-core/src/engine/registry.ts
Normal file
89
packages/rpa-core/src/engine/registry.ts
Normal file
@ -0,0 +1,89 @@
|
|||||||
|
import type { CapabilityHandler } from "./types";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 能力结构化元数据(任务 4.1,决策点 8:meta 可选 + 新能力强制)。
|
||||||
|
* 老能力逐步补齐;规范 v1 验收 checklist 对新能力要求必填。
|
||||||
|
*/
|
||||||
|
export interface CapabilityParamMeta {
|
||||||
|
key: string;
|
||||||
|
label?: string;
|
||||||
|
type: "string" | "number" | "boolean" | "object" | "array";
|
||||||
|
required?: boolean;
|
||||||
|
defaultValue?: string | number | boolean | null;
|
||||||
|
description?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 平台类型;现阶段只有 win32 有实现,darwin/linux 仅类型占位(dev-conventions 规则 4) */
|
||||||
|
export type CapabilityPlatform = "win32" | "darwin" | "linux";
|
||||||
|
|
||||||
|
export interface CapabilityMeta {
|
||||||
|
/** 一句话能力摘要(中文) */
|
||||||
|
summary?: string;
|
||||||
|
/** 参数 schema(按字段名列出) */
|
||||||
|
params?: CapabilityParamMeta[];
|
||||||
|
/** 返回值说明 */
|
||||||
|
returns?: string;
|
||||||
|
/** 默认超时毫秒(无外部交互语义的能力省略) */
|
||||||
|
timeoutMs?: number;
|
||||||
|
/** 可能抛出的稳定错误码(见 engine/error-codes.ts) */
|
||||||
|
errorCodes?: string[];
|
||||||
|
/** 执行中是否需要回传截图(Agent 侧视觉判断用) */
|
||||||
|
needsScreenshot?: boolean;
|
||||||
|
/**
|
||||||
|
* 能力是否可安全重试(无副作用或副作用可重复)。
|
||||||
|
* false 时运行时强制忽略节点的 retryCount(重试门控,防重复副作用);
|
||||||
|
* 缺省表示未知,遵循流程显式配置。
|
||||||
|
*/
|
||||||
|
idempotent?: boolean;
|
||||||
|
/** 平台限制;省略表示跨平台 */
|
||||||
|
platforms?: CapabilityPlatform[];
|
||||||
|
}
|
||||||
|
|
||||||
|
const handlers = new Map<string, CapabilityHandler>();
|
||||||
|
const metas = new Map<string, CapabilityMeta>();
|
||||||
|
|
||||||
|
export function register(ids: string | string[], handler: CapabilityHandler, meta?: CapabilityMeta): void {
|
||||||
|
const list = Array.isArray(ids) ? ids : [ids];
|
||||||
|
for (const id of list) {
|
||||||
|
handlers.set(id, handler);
|
||||||
|
if (meta) metas.set(id, meta);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getHandler(id: string): CapabilityHandler | undefined {
|
||||||
|
return handlers.get(id);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getCapabilityMeta(id: string): CapabilityMeta | undefined {
|
||||||
|
return metas.get(id);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function hasCapability(id: string): boolean {
|
||||||
|
return handlers.has(id);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function capabilityIds(): string[] {
|
||||||
|
return [...handlers.keys()];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function capabilityCount(): number {
|
||||||
|
return handlers.size;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 给一批能力落平台默认值(来自 register-all.ts 的 PLATFORM_GROUPS 单一权威表)。
|
||||||
|
* - 空数组 = 平台无限制,不落 meta;
|
||||||
|
* - 已有 meta 且已声明 platforms 的不覆盖(能力可显式声明更窄的平台限制);
|
||||||
|
* - 无 meta 的补一个只含 platforms 的 meta。
|
||||||
|
*/
|
||||||
|
export function applyPlatformDefaults(ids: string[], platforms: CapabilityPlatform[]): void {
|
||||||
|
if (!platforms.length) return;
|
||||||
|
for (const id of ids) {
|
||||||
|
const meta = metas.get(id);
|
||||||
|
if (meta) {
|
||||||
|
if (!meta.platforms) metas.set(id, { ...meta, platforms: [...platforms] });
|
||||||
|
} else {
|
||||||
|
metas.set(id, { platforms: [...platforms] });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -1,6 +1,7 @@
|
|||||||
import type { FlowDefinition, FlowNode, RunError } from "../../shared/types";
|
import type { FlowDefinition, FlowNode, RunError } from "../types";
|
||||||
import { evaluate, evaluateCondition, interpolate, toNumber } from "./expression";
|
import { evaluate, evaluateCondition, interpolate, toNumber } from "./expression";
|
||||||
import { getHandler } from "./registry";
|
import { hasBlockingIssues, validateFlowGraph } from "./graph-validator";
|
||||||
|
import { getCapabilityMeta, getHandler } from "./registry";
|
||||||
import {
|
import {
|
||||||
asRuntimeError,
|
asRuntimeError,
|
||||||
BreakSignal,
|
BreakSignal,
|
||||||
@ -138,6 +139,17 @@ export async function runFlow(
|
|||||||
options.emit({ type: "run.started", startedAt });
|
options.emit({ type: "run.started", startedAt });
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
const issues = validateFlowGraph(flow);
|
||||||
|
for (const issue of issues.filter((item) => item.severity === "warning")) {
|
||||||
|
options.log(issue.message, "warn");
|
||||||
|
}
|
||||||
|
if (hasBlockingIssues(issues)) {
|
||||||
|
const summary = issues.filter((issue) => issue.severity === "error").map((item) => item.message).join(";");
|
||||||
|
throw new RuntimeError("validation_error", `流程图校验未通过:${summary}`, {
|
||||||
|
recommendation: "修复流程图结构后再运行",
|
||||||
|
detail: JSON.stringify(issues.filter((issue) => issue.severity === "error"))
|
||||||
|
});
|
||||||
|
}
|
||||||
await executeTopLevelGraph(flow, ctx, states);
|
await executeTopLevelGraph(flow, ctx, states);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (error instanceof StopSignal) {
|
if (error instanceof StopSignal) {
|
||||||
@ -202,6 +214,9 @@ async function executeTopLevelGraph(flow: FlowDefinition, ctx: RunContext, state
|
|||||||
try {
|
try {
|
||||||
await executeNode(node, ctx, states);
|
await executeNode(node, ctx, states);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
if (error instanceof BreakSignal || error instanceof ContinueSignal || error instanceof StopSignal) throw error;
|
||||||
|
// 取消不进异常分支:中止运行时直接上抛,由 runFlow 统一收口为 cancelled
|
||||||
|
if (asRuntimeError(error).code === "cancelled") throw error;
|
||||||
const errorEdge = edges.find((edge) => edge.source === node.id && edge.sourceHandle === "error");
|
const errorEdge = edges.find((edge) => edge.source === node.id && edge.sourceHandle === "error");
|
||||||
const target = errorEdge ? topById.get(errorEdge.target) : undefined;
|
const target = errorEdge ? topById.get(errorEdge.target) : undefined;
|
||||||
if (!target) throw error;
|
if (!target) throw error;
|
||||||
@ -240,14 +255,14 @@ async function executeNode(node: FlowNode, ctx: RunContext, states: Map<string,
|
|||||||
state.status = "failed";
|
state.status = "failed";
|
||||||
state.error = error.toRunError();
|
state.error = error.toRunError();
|
||||||
state.finishedAt = new Date().toISOString();
|
state.finishedAt = new Date().toISOString();
|
||||||
ctx.emit({ type: "node.finished", nodeId: node.id, status: "failed", error: state.error, durationMs: 0, evidence: [] });
|
ctx.emit({ type: "node.finished", nodeId: node.id, status: "failed", error: state.error, durationMs: 0, evidence: [], attempt: state.attempt });
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (node.enabled === false) {
|
if (node.enabled === false) {
|
||||||
state.status = "skipped";
|
state.status = "skipped";
|
||||||
state.finishedAt = new Date().toISOString();
|
state.finishedAt = new Date().toISOString();
|
||||||
ctx.emit({ type: "node.finished", nodeId: node.id, status: "skipped", durationMs: 0, evidence: [] });
|
ctx.emit({ type: "node.finished", nodeId: node.id, status: "skipped", durationMs: 0, evidence: [], attempt: state.attempt });
|
||||||
return undefined;
|
return undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -265,7 +280,7 @@ async function executeNode(node: FlowNode, ctx: RunContext, states: Map<string,
|
|||||||
{
|
{
|
||||||
nodeId: node.id,
|
nodeId: node.id,
|
||||||
componentId: node.componentId,
|
componentId: node.componentId,
|
||||||
recommendation: "换用已接入组件;该能力已列入后续接入清单"
|
recommendation: "请检查组件配置或重新添加节点"
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@ -273,23 +288,48 @@ async function executeNode(node: FlowNode, ctx: RunContext, states: Map<string,
|
|||||||
const delayBefore = Math.max(0, Number(node.options?.delayBeforeMs || 0));
|
const delayBefore = Math.max(0, Number(node.options?.delayBeforeMs || 0));
|
||||||
const delayAfter = Math.max(0, Number(node.options?.delayAfterMs || 0));
|
const delayAfter = Math.max(0, Number(node.options?.delayAfterMs || 0));
|
||||||
const timeoutMs = Math.max(0, Number(node.options?.timeoutMs || 30000));
|
const timeoutMs = Math.max(0, Number(node.options?.timeoutMs || 30000));
|
||||||
const retries = Math.max(0, Math.min(100, Number(node.options?.retryCount || 0)));
|
const effectiveTimeout = timeoutMs > 0 ? timeoutMs : 30000;
|
||||||
|
const meta = getCapabilityMeta(node.componentId) || getCapabilityMeta(component.functionName);
|
||||||
|
let retries = Math.max(0, Math.min(100, Number(node.options?.retryCount || 0)));
|
||||||
|
// 重试门控:实现方声明非幂等的能力,无视流程里的重试配置(防重复副作用)
|
||||||
|
if (retries > 0 && meta?.idempotent === false) {
|
||||||
|
retries = 0;
|
||||||
|
ctx.log(`能力「${component.name}」为非幂等操作,已忽略重试设置(retryCount)`, "warn", node.id);
|
||||||
|
}
|
||||||
const interval = Math.max(1000, Number(node.options?.retryIntervalMs || 1000));
|
const interval = Math.max(1000, Number(node.options?.retryIntervalMs || 1000));
|
||||||
|
|
||||||
await sleep(delayBefore, ctx.signal);
|
await sleep(delayBefore, ctx.signal);
|
||||||
for (let attempt = 1; attempt <= retries + 1; attempt += 1) {
|
for (let attempt = 1; attempt <= retries + 1; attempt += 1) {
|
||||||
state.attempt = attempt;
|
state.attempt = attempt;
|
||||||
|
// 尝试级取消源:run 取消与节点超时都汇聚到它,真正中止底层操作
|
||||||
|
const attemptCtl = new AbortController();
|
||||||
|
const onRunAbort = () => attemptCtl.abort();
|
||||||
|
if (ctx.signal.aborted) attemptCtl.abort();
|
||||||
|
else ctx.signal.addEventListener("abort", onRunAbort, { once: true });
|
||||||
|
let timeoutError: RuntimeError | undefined;
|
||||||
|
const timeoutTimer = setTimeout(() => {
|
||||||
|
timeoutError = new RuntimeError("timeout", `${component.name} 执行超时(${effectiveTimeout}ms)`, {
|
||||||
|
recommendation: "检查目标是否可达,或调大该节点的超时时间",
|
||||||
|
nodeId: node.id,
|
||||||
|
componentId: component.id
|
||||||
|
});
|
||||||
|
attemptCtl.abort(timeoutError);
|
||||||
|
}, effectiveTimeout);
|
||||||
|
// handler 收到的 ctx.signal 是尝试级信号:它内部的 sleep/HTTP/子进程监听它即可被中止
|
||||||
|
const attemptCtx: RunContext = { ...ctx, signal: attemptCtl.signal };
|
||||||
try {
|
try {
|
||||||
result = await withTimeout(
|
result = await withTimeout(
|
||||||
Promise.resolve(handler(ctx, node, args, component)),
|
Promise.resolve(handler(attemptCtx, node, args, component)),
|
||||||
timeoutMs,
|
effectiveTimeout,
|
||||||
ctx.signal,
|
attemptCtl.signal,
|
||||||
component.name
|
component.name
|
||||||
);
|
);
|
||||||
break;
|
break;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (error instanceof BreakSignal || error instanceof ContinueSignal || error instanceof StopSignal) throw error;
|
if (error instanceof BreakSignal || error instanceof ContinueSignal || error instanceof StopSignal) throw error;
|
||||||
const runtimeError = asRuntimeError(error, node, component.id);
|
const runtimeError = asRuntimeError(error, node, component.id);
|
||||||
|
// 超时中止底层后,handler 可能先抛出 cancelled;若中止原因是超时则归一为 timeout
|
||||||
|
if (attemptCtl.signal.aborted && timeoutError && runtimeError.code === "cancelled") throw timeoutError;
|
||||||
if (attempt > retries || !isRetryable(runtimeError.code)) {
|
if (attempt > retries || !isRetryable(runtimeError.code)) {
|
||||||
throw new RuntimeError(runtimeError.code, runtimeError.reason, {
|
throw new RuntimeError(runtimeError.code, runtimeError.reason, {
|
||||||
recommendation: runtimeError.recommendation,
|
recommendation: runtimeError.recommendation,
|
||||||
@ -304,6 +344,9 @@ async function executeNode(node: FlowNode, ctx: RunContext, states: Map<string,
|
|||||||
node.id
|
node.id
|
||||||
);
|
);
|
||||||
await sleep(interval, ctx.signal);
|
await sleep(interval, ctx.signal);
|
||||||
|
} finally {
|
||||||
|
clearTimeout(timeoutTimer);
|
||||||
|
ctx.signal.removeEventListener("abort", onRunAbort);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (node.options?.outputVar) ctx.vars[node.options.outputVar] = result;
|
if (node.options?.outputVar) ctx.vars[node.options.outputVar] = result;
|
||||||
@ -314,13 +357,22 @@ async function executeNode(node: FlowNode, ctx: RunContext, states: Map<string,
|
|||||||
state.output = result;
|
state.output = result;
|
||||||
state.finishedAt = new Date().toISOString();
|
state.finishedAt = new Date().toISOString();
|
||||||
state.durationMs = Date.now() - startMs;
|
state.durationMs = Date.now() - startMs;
|
||||||
|
// A++ 检查点协议:成功节点即续跑锚点;checkpointId 取节点 id(跨 attempt 稳定);
|
||||||
|
// 幂等键同样跨 attempt 稳定,供带 resumeToken 重入时去重;非幂等能力不作为锚点
|
||||||
|
const checkpointMeta = getCapabilityMeta(node.componentId) || getCapabilityMeta(component.functionName);
|
||||||
|
const resumable = checkpointMeta?.idempotent !== false;
|
||||||
ctx.emit({
|
ctx.emit({
|
||||||
type: "node.finished",
|
type: "node.finished",
|
||||||
nodeId: node.id,
|
nodeId: node.id,
|
||||||
status: "succeeded",
|
status: "succeeded",
|
||||||
output: result,
|
output: result,
|
||||||
durationMs: state.durationMs,
|
durationMs: state.durationMs,
|
||||||
evidence: state.evidence
|
evidence: state.evidence,
|
||||||
|
checkpointId: node.id,
|
||||||
|
resumable,
|
||||||
|
executor: "local",
|
||||||
|
attempt: state.attempt,
|
||||||
|
...(resumable ? { idempotencyKey: `${ctx.runId}:${node.id}` } : {})
|
||||||
});
|
});
|
||||||
return result;
|
return result;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@ -328,10 +380,27 @@ async function executeNode(node: FlowNode, ctx: RunContext, states: Map<string,
|
|||||||
state.status = "skipped";
|
state.status = "skipped";
|
||||||
state.finishedAt = new Date().toISOString();
|
state.finishedAt = new Date().toISOString();
|
||||||
state.durationMs = Date.now() - startMs;
|
state.durationMs = Date.now() - startMs;
|
||||||
ctx.emit({ type: "node.finished", nodeId: node.id, status: "skipped", durationMs: state.durationMs, evidence: [] });
|
ctx.emit({ type: "node.finished", nodeId: node.id, status: "skipped", durationMs: state.durationMs, evidence: [], attempt: state.attempt });
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
const runtimeError = asRuntimeError(error, node, component.id);
|
const runtimeError = asRuntimeError(error, node, component.id);
|
||||||
|
// 取消语义:用户中止运行时节点状态为 cancelled,不适用 skipOnError,也不进异常分支
|
||||||
|
if (runtimeError.code === "cancelled" || ctx.signal.aborted) {
|
||||||
|
state.error = runtimeError;
|
||||||
|
state.status = "cancelled";
|
||||||
|
state.finishedAt = new Date().toISOString();
|
||||||
|
state.durationMs = Date.now() - startMs;
|
||||||
|
ctx.emit({
|
||||||
|
type: "node.finished",
|
||||||
|
nodeId: node.id,
|
||||||
|
status: "cancelled",
|
||||||
|
error: runtimeError,
|
||||||
|
durationMs: state.durationMs,
|
||||||
|
evidence: [],
|
||||||
|
attempt: state.attempt
|
||||||
|
});
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
state.error = runtimeError;
|
state.error = runtimeError;
|
||||||
state.finishedAt = new Date().toISOString();
|
state.finishedAt = new Date().toISOString();
|
||||||
state.durationMs = Date.now() - startMs;
|
state.durationMs = Date.now() - startMs;
|
||||||
@ -345,7 +414,8 @@ async function executeNode(node: FlowNode, ctx: RunContext, states: Map<string,
|
|||||||
status: state.status,
|
status: state.status,
|
||||||
error: runtimeError,
|
error: runtimeError,
|
||||||
durationMs: state.durationMs,
|
durationMs: state.durationMs,
|
||||||
evidence: []
|
evidence: [],
|
||||||
|
attempt: state.attempt
|
||||||
});
|
});
|
||||||
if (skip) return undefined;
|
if (skip) return undefined;
|
||||||
throw error;
|
throw error;
|
||||||
@ -463,6 +533,9 @@ async function executeContainer(
|
|||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (error instanceof BreakSignal || error instanceof ContinueSignal || error instanceof StopSignal) throw error;
|
if (error instanceof BreakSignal || error instanceof ContinueSignal || error instanceof StopSignal) throw error;
|
||||||
ctx.vars.error = asRuntimeError(error, node, componentId);
|
ctx.vars.error = asRuntimeError(error, node, componentId);
|
||||||
|
// 明确告知:捕获 ≠ 运行失败,属于"可选步骤"的正常跳过路径
|
||||||
|
const reason = error instanceof Error ? error.message : String(error);
|
||||||
|
await ctx.log(`「${node.label || node.id}」子步骤未成功,已捕获并继续执行:${reason}`, "warn", node.id);
|
||||||
await runSlot("except_fn");
|
await runSlot("except_fn");
|
||||||
return "catch";
|
return "catch";
|
||||||
}
|
}
|
||||||
@ -1,4 +1,4 @@
|
|||||||
import type { CatalogComponent, FlowDefinition, FlowNode, RunError, RunEvent } from "../../shared/types";
|
import type { CatalogComponent, FlowDefinition, FlowNode, RunError, RunEvent } from "../types";
|
||||||
|
|
||||||
export interface BrowserLike {
|
export interface BrowserLike {
|
||||||
navigate(url: string): Promise<{ url: string; title: string }>;
|
navigate(url: string): Promise<{ url: string; title: string }>;
|
||||||
@ -67,6 +67,8 @@ export interface BrowserElementReference {
|
|||||||
tag: string;
|
tag: string;
|
||||||
text: string;
|
text: string;
|
||||||
elementValue: string;
|
elementValue: string;
|
||||||
|
/** 相似元素列表场景:该元素在选择器全部匹配结果中的序号(动态 id 泛化后引用选择器相同,需靠序号区分) */
|
||||||
|
index?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface BrowserElementInfo extends BrowserElementReference {
|
export interface BrowserElementInfo extends BrowserElementReference {
|
||||||
@ -137,7 +139,8 @@ export type CapabilityHandler = (
|
|||||||
export interface NodeState {
|
export interface NodeState {
|
||||||
nodeId: string;
|
nodeId: string;
|
||||||
componentId: string;
|
componentId: string;
|
||||||
status: "idle" | "running" | "succeeded" | "failed" | "skipped" | "cancelled";
|
// waiting_user:节点挂起等待人审(对面 approval 流);unknown:崩溃后结果不可知(对面 workflow-recovery 对齐)
|
||||||
|
status: "idle" | "running" | "succeeded" | "failed" | "skipped" | "cancelled" | "waiting_user" | "unknown";
|
||||||
startedAt?: string;
|
startedAt?: string;
|
||||||
finishedAt?: string;
|
finishedAt?: string;
|
||||||
durationMs?: number;
|
durationMs?: number;
|
||||||
@ -147,6 +150,24 @@ export interface NodeState {
|
|||||||
attempt: number;
|
attempt: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 与对面 workflow-engine StepStatus(8 态)同构;唯一非同名映射是 idle↔pending
|
||||||
|
export type WorkflowStepStatus = "pending" | "running" | "waiting_user" | "succeeded" | "failed" | "cancelled" | "unknown" | "skipped";
|
||||||
|
|
||||||
|
export const NODE_STATUS_TO_STEP_STATUS: Record<NodeState["status"], WorkflowStepStatus> = {
|
||||||
|
idle: "pending",
|
||||||
|
running: "running",
|
||||||
|
waiting_user: "waiting_user",
|
||||||
|
succeeded: "succeeded",
|
||||||
|
failed: "failed",
|
||||||
|
cancelled: "cancelled",
|
||||||
|
unknown: "unknown",
|
||||||
|
skipped: "skipped"
|
||||||
|
};
|
||||||
|
|
||||||
|
export function toStepStatus(status: NodeState["status"]): WorkflowStepStatus {
|
||||||
|
return NODE_STATUS_TO_STEP_STATUS[status];
|
||||||
|
}
|
||||||
|
|
||||||
export interface RunResult {
|
export interface RunResult {
|
||||||
runId: string;
|
runId: string;
|
||||||
status: "succeeded" | "failed" | "cancelled";
|
status: "succeeded" | "failed" | "cancelled";
|
||||||
104
packages/rpa-core/src/index.ts
Normal file
104
packages/rpa-core/src/index.ts
Normal file
@ -0,0 +1,104 @@
|
|||||||
|
/**
|
||||||
|
* @rpa/core 包入口。
|
||||||
|
*
|
||||||
|
* 只导出引擎 API 与类型,不做能力注册副作用;
|
||||||
|
* 纯 Node 能力通过 `@rpa/core/capabilities` 子路径按需拉起(保持与 demo 宿主
|
||||||
|
* src/main/capabilities/index.ts 相同的副作用注册模型,见任务 4.2 再显式化)。
|
||||||
|
*/
|
||||||
|
export { createEngine, defaultEngine, RpaEngine, type ManualOperationSettings } from "./engine/engine";
|
||||||
|
export { register, getHandler, hasCapability, capabilityIds, capabilityCount } from "./engine/registry";
|
||||||
|
export { runFlow } from "./engine/runtime";
|
||||||
|
export { RuntimeError, BreakSignal, ContinueSignal, StopSignal, asRuntimeError } from "./engine/types";
|
||||||
|
export { setSubflowSource, getSubflowSource, type SubflowSource } from "./engine/host-bridge";
|
||||||
|
// 错误契约(任务 4.3):稳定错误码枚举 + 机器可读结构(RuntimeError.code / recommendation / detail 等)
|
||||||
|
export { RUNTIME_ERROR_CODES, isKnownErrorCode, type ErrorCode } from "./engine/error-codes";
|
||||||
|
// 能力元数据(任务 4.1)与显式注册入口(任务 4.2,双入口过渡)
|
||||||
|
export { getCapabilityMeta, type CapabilityMeta, type CapabilityParamMeta } from "./engine/registry";
|
||||||
|
export { registerAll, capabilityDomains } from "./capabilities/register-all";
|
||||||
|
export {
|
||||||
|
setHostBridge,
|
||||||
|
getHostBridge,
|
||||||
|
type HostBridge,
|
||||||
|
type HostDialogWindow,
|
||||||
|
type HostPoint,
|
||||||
|
type HostAiConfig
|
||||||
|
} from "./engine/host-bridge";
|
||||||
|
// AI 服务(阶段 2 经 HostBridge 迁入包内):宿主注入 loadAiConfig,消费方走此导出
|
||||||
|
export {
|
||||||
|
chatCompletion,
|
||||||
|
getAiConfig,
|
||||||
|
requireAiConfig,
|
||||||
|
type ChatCompletionOptions,
|
||||||
|
type ChatContentPart,
|
||||||
|
type ChatMessage,
|
||||||
|
type ResolvedAiConfig
|
||||||
|
} from "./capabilities/ai-service";
|
||||||
|
// 浏览器 CDP 外部路径(阶段 2 迁入):ExternalTransport + 外部浏览器进程管理
|
||||||
|
export { ExternalTransport, mapCdpError, type CdpTransport, type CdpEventHandler } from "./browser/transport";
|
||||||
|
export {
|
||||||
|
listPageTargets,
|
||||||
|
openExternalBrowser,
|
||||||
|
openNewPageTarget,
|
||||||
|
sessionKeyOf,
|
||||||
|
detectBrowserPath,
|
||||||
|
connectRemoteBrowser,
|
||||||
|
launchExternalBrowser,
|
||||||
|
type ExternalBrowserOptions,
|
||||||
|
type ExternalBrowserType,
|
||||||
|
type ExternalLaunchOptions,
|
||||||
|
type ExternalRemoteOptions,
|
||||||
|
type ExternalConnection
|
||||||
|
} from "./browser/external";
|
||||||
|
// 桌面自动化驱动(阶段 3):五域接口 + PowerShell 实现,引擎创建时注入
|
||||||
|
export type {
|
||||||
|
AutomationDriver,
|
||||||
|
DriverPoint,
|
||||||
|
DriverRect,
|
||||||
|
ElementDriver,
|
||||||
|
ElementFindOptions,
|
||||||
|
KeyboardDriver,
|
||||||
|
KeyboardSendOptions,
|
||||||
|
MouseClickOptions,
|
||||||
|
MouseDragOptions,
|
||||||
|
MouseDriver,
|
||||||
|
MouseSmoothOptions,
|
||||||
|
MouseButton,
|
||||||
|
ScreenDriver,
|
||||||
|
UiaSelector,
|
||||||
|
WindowDriver,
|
||||||
|
WindowInfo,
|
||||||
|
WindowSummary
|
||||||
|
} from "./driver/types";
|
||||||
|
export { WindowsDriver, setElementDriverImplementation } from "./driver/windows/windows-driver";
|
||||||
|
|
||||||
|
// PowerShell 助手(任务 3.1 已移入 driver/windows/)
|
||||||
|
export { powershell, psString } from "./driver/windows/powershell";
|
||||||
|
export type {
|
||||||
|
BrowserLike,
|
||||||
|
BrowserTabInfo,
|
||||||
|
BrowserElementReference,
|
||||||
|
BrowserElementInfo,
|
||||||
|
BrowserSelectOption,
|
||||||
|
BrowserSelectState,
|
||||||
|
BrowserSimilarElementOptions,
|
||||||
|
BrowserElementRegion,
|
||||||
|
RunContext,
|
||||||
|
CapabilityHandler,
|
||||||
|
NodeState,
|
||||||
|
RunResult
|
||||||
|
} from "./engine/types";
|
||||||
|
export type {
|
||||||
|
CatalogWidget,
|
||||||
|
CatalogField,
|
||||||
|
CatalogOutput,
|
||||||
|
CatalogComponent,
|
||||||
|
CatalogGroup,
|
||||||
|
Catalog,
|
||||||
|
FlowVariable,
|
||||||
|
FlowNodeOptions,
|
||||||
|
FlowNode,
|
||||||
|
FlowEdge,
|
||||||
|
FlowDefinition,
|
||||||
|
RunError,
|
||||||
|
RunEvent
|
||||||
|
} from "./types";
|
||||||
193
packages/rpa-core/src/types.ts
Normal file
193
packages/rpa-core/src/types.ts
Normal file
@ -0,0 +1,193 @@
|
|||||||
|
/**
|
||||||
|
* rpa-core 领域类型(任务 1.2,类型单一源)
|
||||||
|
*
|
||||||
|
* 从 src/shared/types 拆出:与流程/目录/运行相关、引擎必须依赖的模型归包所有。
|
||||||
|
* 宿主专属类型(AppInfo / AppSettings / FlowSummary / VisionLab* 等)留在 src/shared/types,
|
||||||
|
* 该文件通过 type-only re-export 引用本文件,保证全仓不出现两份定义。
|
||||||
|
*/
|
||||||
|
|
||||||
|
export type CatalogWidget =
|
||||||
|
| "text"
|
||||||
|
| "textarea"
|
||||||
|
| "number"
|
||||||
|
| "select"
|
||||||
|
| "radio"
|
||||||
|
| "boolean"
|
||||||
|
| "file"
|
||||||
|
| "folder"
|
||||||
|
| "element"
|
||||||
|
| "code"
|
||||||
|
| "secret"
|
||||||
|
| "region"
|
||||||
|
| "hidden";
|
||||||
|
|
||||||
|
export interface CatalogField {
|
||||||
|
key: string;
|
||||||
|
label: string;
|
||||||
|
widget: CatalogWidget;
|
||||||
|
type: "string" | "number" | "boolean" | "object";
|
||||||
|
displayType: string;
|
||||||
|
required: boolean;
|
||||||
|
defaultValue: string;
|
||||||
|
placeholder: string;
|
||||||
|
description: string;
|
||||||
|
advanced: boolean;
|
||||||
|
valueTypes: string[];
|
||||||
|
/** 下拉/单选选项:字符串(显示=存储值)或 { label, value }(显示与存储分离) */
|
||||||
|
options?: Array<string | { label: string; value: string }>;
|
||||||
|
unit?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CatalogOutput {
|
||||||
|
label: string;
|
||||||
|
key: string;
|
||||||
|
suggestedName: string;
|
||||||
|
valueTypes: string[];
|
||||||
|
fields: Array<{ key: string; label: string; defaultValue: string; valueTypes: string[] }>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CatalogComponent {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
kind: "vendor" | "native" | "trigger";
|
||||||
|
group: string;
|
||||||
|
categories: string[];
|
||||||
|
categorySignature: string;
|
||||||
|
description: string;
|
||||||
|
messageTemplate: string;
|
||||||
|
baseType: string;
|
||||||
|
functionName: string;
|
||||||
|
functionCalls: Array<{ function: string; args: string[]; assignTo: string }>;
|
||||||
|
version: string;
|
||||||
|
tags: string[];
|
||||||
|
platformTag: string | null;
|
||||||
|
icon: string;
|
||||||
|
risk: "low" | "high";
|
||||||
|
fields: CatalogField[];
|
||||||
|
output: CatalogOutput;
|
||||||
|
slots: string[];
|
||||||
|
nodeKind?: string;
|
||||||
|
/** 同名旧版本 / 重复别名指向的保留能力 ID。 */
|
||||||
|
duplicateOf?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CatalogGroup {
|
||||||
|
id: string;
|
||||||
|
label: string;
|
||||||
|
icon: string;
|
||||||
|
count: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Catalog {
|
||||||
|
schemaVersion: number;
|
||||||
|
generatedAt: string;
|
||||||
|
counts: {
|
||||||
|
total: number;
|
||||||
|
vendor: number;
|
||||||
|
native: number;
|
||||||
|
triggers: number;
|
||||||
|
fields: number;
|
||||||
|
};
|
||||||
|
groups: CatalogGroup[];
|
||||||
|
components: CatalogComponent[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface FlowVariable {
|
||||||
|
name: string;
|
||||||
|
type: string;
|
||||||
|
defaultValue?: string;
|
||||||
|
description?: string;
|
||||||
|
scope?: "flow" | "local";
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface FlowNodeOptions {
|
||||||
|
outputVar?: string;
|
||||||
|
delayBeforeMs?: number;
|
||||||
|
delayAfterMs?: number;
|
||||||
|
timeoutMs?: number;
|
||||||
|
retryCount?: number;
|
||||||
|
retryIntervalMs?: number;
|
||||||
|
skipOnError?: boolean;
|
||||||
|
logException?: boolean;
|
||||||
|
errorRouteEnabled?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface FlowNode {
|
||||||
|
id: string;
|
||||||
|
componentId: string;
|
||||||
|
label?: string;
|
||||||
|
position: { x: number; y: number };
|
||||||
|
values: Record<string, unknown>;
|
||||||
|
options: FlowNodeOptions;
|
||||||
|
parentId?: string | null;
|
||||||
|
slot?: string;
|
||||||
|
enabled?: boolean;
|
||||||
|
width?: number;
|
||||||
|
height?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface FlowEdge {
|
||||||
|
id: string;
|
||||||
|
source: string;
|
||||||
|
target: string;
|
||||||
|
sourceHandle?: string | null;
|
||||||
|
targetHandle?: string | null;
|
||||||
|
label?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface FlowDefinition {
|
||||||
|
schemaVersion: 1;
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
description?: string;
|
||||||
|
enabled?: boolean;
|
||||||
|
trigger?: {
|
||||||
|
type: "manual" | "schedule" | "file";
|
||||||
|
label: string;
|
||||||
|
};
|
||||||
|
variables: FlowVariable[];
|
||||||
|
nodes: FlowNode[];
|
||||||
|
edges: FlowEdge[];
|
||||||
|
meta: {
|
||||||
|
createdAt: string;
|
||||||
|
updatedAt: string;
|
||||||
|
appVersion: string;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RunError {
|
||||||
|
code: string;
|
||||||
|
reason: string;
|
||||||
|
recommendation?: string;
|
||||||
|
location?: { flowId?: string; nodeId?: string; componentId?: string; arg?: string };
|
||||||
|
detail?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RunEvent {
|
||||||
|
type:
|
||||||
|
| "run.started"
|
||||||
|
| "node.started"
|
||||||
|
| "node.log"
|
||||||
|
| "node.finished"
|
||||||
|
| "run.finished"
|
||||||
|
| "run.total";
|
||||||
|
runId: string;
|
||||||
|
nodeId?: string;
|
||||||
|
level?: "info" | "warn" | "error";
|
||||||
|
message?: string;
|
||||||
|
status?: string;
|
||||||
|
output?: unknown;
|
||||||
|
error?: RunError;
|
||||||
|
durationMs?: number;
|
||||||
|
evidence?: string[];
|
||||||
|
attempt?: number;
|
||||||
|
// A++ 检查点协议负载(ADR-R2 补记):node.finished 成功时由 runtime 填充
|
||||||
|
checkpointId?: string;
|
||||||
|
idempotencyKey?: string;
|
||||||
|
resumable?: boolean;
|
||||||
|
// 双端透明调度(ADR-R6 补记):本次执行驻地;用户界面不展示,审计/排障必须可见
|
||||||
|
executor?: "local" | "backend";
|
||||||
|
startedAt?: string;
|
||||||
|
nodeLabel?: string;
|
||||||
|
total?: number;
|
||||||
|
}
|
||||||
24
packages/rpa-core/src/types/sm-crypto.d.ts
vendored
Normal file
24
packages/rpa-core/src/types/sm-crypto.d.ts
vendored
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
// sm-crypto 无官方类型声明;仅声明本仓库用到的 SM4 解密 API
|
||||||
|
declare module "sm-crypto" {
|
||||||
|
export interface Sm4Options {
|
||||||
|
padding?: "pkcs#7" | "none";
|
||||||
|
mode?: "cbc";
|
||||||
|
iv?: string;
|
||||||
|
}
|
||||||
|
export const sm4: {
|
||||||
|
decrypt(
|
||||||
|
cipherText: string,
|
||||||
|
key: string,
|
||||||
|
options?: Sm4Options
|
||||||
|
): string;
|
||||||
|
encrypt(plainText: string, key: string, options?: Sm4Options): string;
|
||||||
|
};
|
||||||
|
export const sm2: {
|
||||||
|
doEncrypt(msgString: string, publicKey: string, cipherMode?: 0 | 1): string;
|
||||||
|
doDecrypt(
|
||||||
|
cipherText: string,
|
||||||
|
privateKey: string,
|
||||||
|
cipherMode?: 0 | 1
|
||||||
|
): string;
|
||||||
|
};
|
||||||
|
}
|
||||||
Binary file not shown.
|
After Width: | Height: | Size: 5.4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 5.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 6.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 7.3 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 6.1 KiB |
@ -1,15 +1,15 @@
|
|||||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
import type { CatalogComponent, FlowNode } from "../../shared/types";
|
import type { CatalogComponent, FlowNode } from "../../src/types";
|
||||||
import type { RunContext } from "../engine/types";
|
import type { RunContext } from "../../src/engine/types";
|
||||||
import { getHandler } from "../engine/registry";
|
import { getHandler } from "../../src/engine/registry";
|
||||||
|
|
||||||
const mocks = vi.hoisted(() => ({ powershell: vi.fn(), execFile: vi.fn() }));
|
const mocks = vi.hoisted(() => ({ powershell: vi.fn(), execFile: vi.fn() }));
|
||||||
vi.mock("./desktop", () => ({ powershell: mocks.powershell, psString: (value: unknown) => `'${String(value ?? "").replace(/'/g, "''")}'` }));
|
vi.mock("../driver/windows/powershell", () => ({ powershell: mocks.powershell, psString: (value: unknown) => `'${String(value ?? "").replace(/'/g, "''")}'` }));
|
||||||
vi.mock("node:child_process", () => ({ execFile: mocks.execFile }));
|
vi.mock("node:child_process", () => ({ execFile: mocks.execFile }));
|
||||||
import "./data";
|
import "../../src/capabilities/data";
|
||||||
import "./control";
|
import "../../src/capabilities/control";
|
||||||
import "./system-extended";
|
import "../../src/capabilities/system-extended";
|
||||||
import "./windows-system";
|
import "../../src/capabilities/windows-system";
|
||||||
|
|
||||||
function run(id: string, args: Record<string, unknown> = {}, signal = new AbortController().signal) {
|
function run(id: string, args: Record<string, unknown> = {}, signal = new AbortController().signal) {
|
||||||
const node: FlowNode = { id: "regression", componentId: id, values: args, options: {}, position: { x: 0, y: 0 } };
|
const node: FlowNode = { id: "regression", componentId: id, values: args, options: {}, position: { x: 0, y: 0 } };
|
||||||
@ -18,7 +18,7 @@ function run(id: string, args: Record<string, unknown> = {}, signal = new AbortC
|
|||||||
|
|
||||||
afterEach(() => { vi.useRealTimers(); vi.resetAllMocks(); });
|
afterEach(() => { vi.useRealTimers(); vi.resetAllMocks(); });
|
||||||
|
|
||||||
describe("应用与系统组件缺陷回归", () => {
|
describe.skipIf(process.platform !== "win32")("应用与系统组件缺陷回归", () => {
|
||||||
it("没有匹配进程时返回 [] / null,忽略空行和非法 PID", async () => {
|
it("没有匹配进程时返回 [] / null,忽略空行和非法 PID", async () => {
|
||||||
mocks.powershell.mockResolvedValue("");
|
mocks.powershell.mockResolvedValue("");
|
||||||
await expect(run("App_GetPids", { pattern: "missing.exe" })).resolves.toEqual([]);
|
await expect(run("App_GetPids", { pattern: "missing.exe" })).resolves.toEqual([]);
|
||||||
157
packages/rpa-core/test/capabilities/app-system-smoke.test.ts
Normal file
157
packages/rpa-core/test/capabilities/app-system-smoke.test.ts
Normal file
@ -0,0 +1,157 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { getHandler } from "../../src/engine/registry";
|
||||||
|
import type { CatalogComponent, FlowNode } from "../../src/types";
|
||||||
|
import type { RunContext } from "../../src/engine/types";
|
||||||
|
import "../../src/capabilities/data";
|
||||||
|
import "../../src/capabilities/desktop";
|
||||||
|
import "../../src/capabilities/system-extended";
|
||||||
|
import "../../src/capabilities/utility-extended";
|
||||||
|
import "../../src/capabilities/common-final";
|
||||||
|
import "../../src/capabilities/control";
|
||||||
|
import "../../src/capabilities/windows-system";
|
||||||
|
import "../../src/capabilities/basic-encryption";
|
||||||
|
|
||||||
|
const ctx = {
|
||||||
|
signal: new AbortController().signal,
|
||||||
|
log: () => {},
|
||||||
|
vars: {},
|
||||||
|
evidenceDir: "D:/office-test2",
|
||||||
|
runId: "smoke-run",
|
||||||
|
flow: {
|
||||||
|
id: "smoke-flow", name: "冒烟测试流程", description: "", enabled: true,
|
||||||
|
trigger: null, meta: { appVersion: "dev", createdAt: "", updatedAt: "" },
|
||||||
|
nodes: []
|
||||||
|
}
|
||||||
|
} as unknown as RunContext;
|
||||||
|
|
||||||
|
const node = { id: "smoke-app", componentId: "smoke", position: { x: 0, y: 0 }, values: {}, options: {} } as unknown as FlowNode;
|
||||||
|
|
||||||
|
async function run(componentId: string, args: Record<string, unknown>, ctxOverride?: RunContext): Promise<unknown> {
|
||||||
|
const handler = getHandler(componentId);
|
||||||
|
if (!handler) throw new Error(`组件 ${componentId} 未注册运行时`);
|
||||||
|
const n = { ...node, componentId } as unknown as FlowNode;
|
||||||
|
return handler(ctxOverride ?? ctx, n, args, {} as CatalogComponent);
|
||||||
|
}
|
||||||
|
|
||||||
|
describe.skipIf(process.platform !== "win32")("应用与系统组件冒烟", () => {
|
||||||
|
it("数据类型判断族", async () => {
|
||||||
|
expect(await run("Basic_IsNumber", { val: "12.5" })).toBe(true);
|
||||||
|
expect(await run("Basic_IsNumber", { val: "abc" })).toBe(false);
|
||||||
|
expect(await run("Basic_IsNull", { val: "" })).toBe(true);
|
||||||
|
expect(await run("Basic_IsNull", { val: "x" })).toBe(false);
|
||||||
|
expect(await run("Basic_IsArray", { val: [1, 2] })).toBe(true);
|
||||||
|
expect(await run("Basic_IsDictionary", { val: { a: 1 } })).toBe(true);
|
||||||
|
expect(await run("Basic_Type", { val: [1] })).toBe("list");
|
||||||
|
expect(await run("Basic_Type", { val: "s" })).toBe("string");
|
||||||
|
expect(await run("Basic_JudgeDataType", { val: 5, into_type: "int" })).toBe(true);
|
||||||
|
expect(await run("Basic_JudgeDataType", { val: 5.5, into_type: "int" })).toBe(false);
|
||||||
|
console.log("[数据判断族] 通过");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("数据转换族", async () => {
|
||||||
|
expect(await run("Basic_ConvertString", { val: 123 })).toBe("123");
|
||||||
|
expect(await run("Basic_ConvertInt", { val: "12.9" })).toBe(12);
|
||||||
|
expect(await run("Basic_ConvertBool", { val: 1 })).toBe(true);
|
||||||
|
expect(await run("Basic_ConvertNumber", { val: "3.14" })).toBe(3.14);
|
||||||
|
expect(await run("Basic_DataTypeConversion", { val: "是", into_type: "bool" })).toBe(true);
|
||||||
|
expect(await run("Basic_DataTypeConversion", { val: 8, into_type: "string" })).toBe("8");
|
||||||
|
console.log("[数据转换族] 通过");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("复制/随机/密文", async () => {
|
||||||
|
const src = { a: [1, { b: 2 }] };
|
||||||
|
const cloned = (await run("Basic_Clone", { val: src })) as typeof src;
|
||||||
|
expect(cloned).toEqual(src);
|
||||||
|
expect(cloned).not.toBe(src);
|
||||||
|
const ri = (await run("Basic_RandomInt", { min: 5, max: 5 })) as number;
|
||||||
|
expect(ri).toBe(5);
|
||||||
|
const rnd = (await run("Basic_Rnd", {})) as number;
|
||||||
|
expect(rnd).toBeGreaterThanOrEqual(0);
|
||||||
|
expect(rnd).toBeLessThan(1);
|
||||||
|
expect(await run("Basic_Dodecryption", { words: Buffer.from("hello-世界", "utf8").toString("base64") })).toBe("hello-世界");
|
||||||
|
console.log("[复制/随机/密文] 通过");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("延迟/调试输出/设置变量", async () => {
|
||||||
|
const logs: string[] = [];
|
||||||
|
const ctxLog = { ...ctx, log: (m: string) => logs.push(m), vars: {} } as unknown as RunContext;
|
||||||
|
const t0 = Date.now();
|
||||||
|
expect(await run("Basic_Delay", { mileSecond: 100 })).toBe(0.1);
|
||||||
|
expect(Date.now() - t0).toBeGreaterThanOrEqual(95);
|
||||||
|
await run("Basic_DebugOutput", { msg: "冒烟输出" }, ctxLog);
|
||||||
|
expect(logs.join()).toContain("冒烟输出");
|
||||||
|
const setNode = { ...node, options: { outputVar: "v_test" } } as unknown as FlowNode;
|
||||||
|
const handler = getHandler("Basic_SetVariable_V1")!;
|
||||||
|
expect(await handler(ctxLog, setNode, { value: "42", variableType: "rpa_int" }, {} as CatalogComponent)).toBe(42);
|
||||||
|
expect((ctxLog as unknown as { vars: Record<string, unknown> }).vars.v_test).toBe(42);
|
||||||
|
console.log("[延迟/输出/变量] 通过");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("自定义配置与业务计数", async () => {
|
||||||
|
const cfg = (await run("CustomConfig_GetConfig", {
|
||||||
|
jsonlist: JSON.stringify([{ name: "host", value: "127.0.0.1" }]),
|
||||||
|
configName: "host"
|
||||||
|
})) as string;
|
||||||
|
expect(cfg).toBe("127.0.0.1");
|
||||||
|
const ctxCount = { ...ctx, vars: {} } as unknown as RunContext;
|
||||||
|
expect(await run("Basic_BusinessOrderCount", { start_count: 100, count: 3 }, ctxCount)).toBe(103);
|
||||||
|
expect(await run("Basic_BusinessOrderCount", { start_count: 100, count: 3 }, ctxCount)).toBe(106);
|
||||||
|
console.log("[配置/计数] 通过");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("系统路径与环境变量", async () => {
|
||||||
|
expect(String(await run("System_GetTempPath", {}))).toMatch(/Temp|tmp/i);
|
||||||
|
expect(String(await run("System_GetHomePath", {}))).toMatch(/Users|home/i);
|
||||||
|
expect(String(await run("System_GetSystemPath", { path_name: "desktop" }))).toMatch(/Desktop/);
|
||||||
|
expect(await run("System_GetEnv", { name: "PATH" })).toBeTruthy();
|
||||||
|
expect(await run("System_SetEnv", { name: "RPA_SMOKE", value: "ok" })).toBe("ok");
|
||||||
|
expect(await run("System_GetEnv", { name: "RPA_SMOKE" })).toBe("ok");
|
||||||
|
expect(await run("Basic_GetProcessInfo", {})).toBeTruthy();
|
||||||
|
console.log("[系统路径/环境变量] 通过");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("输入法状态获取", async () => {
|
||||||
|
const st = (await run("System_GetInputMethodState", {})) as { culture: string; layout: string };
|
||||||
|
console.log("[输入法]", JSON.stringify(st));
|
||||||
|
expect(st.culture).toBeTruthy();
|
||||||
|
}, 60000);
|
||||||
|
|
||||||
|
it("执行命令行两种组件", async () => {
|
||||||
|
const out1 = String(await run("System_ExecuteCommand", { command: "echo rpa-smoke-123" }));
|
||||||
|
expect(out1).toContain("rpa-smoke-123");
|
||||||
|
const out2 = String(await run("System_ExecuteCommandV1", { command_type: "1", dos_command: "echo dos-ok-456" }));
|
||||||
|
expect(out2).toContain("dos-ok-456");
|
||||||
|
const out3 = String(await run("System_ExecuteCommandV1", { command_type: "3", ps_command: "Write-Output 'ps-ok-789'" }));
|
||||||
|
expect(out3).toContain("ps-ok-789");
|
||||||
|
console.log("[执行命令] 通过");
|
||||||
|
}, 120000);
|
||||||
|
|
||||||
|
it("应用生命周期: 启动→查PID→关闭", async () => {
|
||||||
|
const pid = (await run("App_Run", { executeable: "notepad.exe" })) as number;
|
||||||
|
expect(Number.isInteger(pid)).toBe(true);
|
||||||
|
await new Promise((r) => setTimeout(r, 1200));
|
||||||
|
// Win11 notepad.exe 为应用别名:Start-Process 返回的启动器 PID 会立即退出,真实进程另获新 PID,
|
||||||
|
// 因此不比较具体 PID,只验证「能查到运行中的 notepad 进程」
|
||||||
|
const single = (await run("App_GetPid", { pattern: "notepad" })) as number | null;
|
||||||
|
expect(single).toBeTruthy();
|
||||||
|
const all = (await run("App_GetPids", { p_name: "notepad" })) as number[];
|
||||||
|
expect(all.length).toBeGreaterThan(0);
|
||||||
|
const killed = await run("App_KillV1", { close_type: "name", p_name: "notepad" });
|
||||||
|
console.log("[应用生命周期] 启动PID:", pid, "查询到:", all, "关闭:", JSON.stringify(killed));
|
||||||
|
await new Promise((r) => setTimeout(r, 800));
|
||||||
|
const after = (await run("App_GetPids", { pattern: "notepad" })) as number[];
|
||||||
|
expect(after.length).toBe(0);
|
||||||
|
}, 120000);
|
||||||
|
|
||||||
|
it("Windows凭据: 添加→读取→删除", async () => {
|
||||||
|
const target = "RPA_SMOKETEST_GENERIC";
|
||||||
|
expect((await run("System_AddCredential", { application: target, name: "smoke-user", password: "p@ss" })) as { added: boolean }).toEqual({ added: true, target });
|
||||||
|
const read = (await run("System_ReadCredential", { application: target })) as { exists: boolean; detail: string };
|
||||||
|
expect(read.exists).toBe(true);
|
||||||
|
expect(read.detail).toContain("smoke-user");
|
||||||
|
expect((await run("System_DeleteCredential", { application: target })) as { deleted: boolean }).toEqual({ deleted: true, target });
|
||||||
|
const readAfter = (await run("System_ReadCredential", { application: target })) as { exists: boolean };
|
||||||
|
expect(readAfter.exists).toBe(false);
|
||||||
|
console.log("[凭据闭环] 通过");
|
||||||
|
}, 120000);
|
||||||
|
});
|
||||||
31
packages/rpa-core/test/capabilities/basic-manual.test.ts
Normal file
31
packages/rpa-core/test/capabilities/basic-manual.test.ts
Normal file
@ -0,0 +1,31 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { getHandler } from "../../src/engine/registry";
|
||||||
|
import type { CatalogComponent, FlowNode } from "../../src/types";
|
||||||
|
import type { RunContext } from "../../src/engine/types";
|
||||||
|
import "../../src/capabilities/desktop";
|
||||||
|
import "../../src/capabilities/basic-encryption";
|
||||||
|
|
||||||
|
const context = {} as RunContext;
|
||||||
|
const component = {} as CatalogComponent;
|
||||||
|
const node = (id: string): FlowNode => ({ id, componentId: id, position: { x: 0, y: 0 }, values: {}, options: {} });
|
||||||
|
async function run(id: string, args: Record<string, unknown>) {
|
||||||
|
const handler = getHandler(id);
|
||||||
|
expect(handler, id).toBeTruthy();
|
||||||
|
return handler!(context, node(id), args, component);
|
||||||
|
}
|
||||||
|
|
||||||
|
describe.skipIf(process.platform !== "win32")("manual keyboard and encrypted variables", () => {
|
||||||
|
it("registers manual keyboard switches", async () => {
|
||||||
|
await expect(run("Keyboard_EnableManualOperation", { random_move: true, random_click: true, random_sleep: false, min_time: 0, max_time: 0 })).resolves.toMatchObject({ enabled: true });
|
||||||
|
await expect(run("Keyboard_DisableManualOperation", {})).resolves.toEqual({ enabled: false });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("decodes base64 and SM4 ciphertext", async () => {
|
||||||
|
await expect(run("Basic_Dodecryption", { words: Buffer.from("hello", "utf8").toString("base64") })).resolves.toBe("hello");
|
||||||
|
const { sm4 } = await import("sm-crypto");
|
||||||
|
const key = Buffer.from("INPUT_PARAM_KEY4", "utf8").toString("hex");
|
||||||
|
const cipher = sm4.encrypt("secret", key, { padding: "pkcs#7" });
|
||||||
|
await expect(run("Basic_Dodecryption", { words: cipher })).resolves.toBe("secret");
|
||||||
|
await expect(run("Basic_Dodecryption", {})).rejects.toThrow();
|
||||||
|
});
|
||||||
|
});
|
||||||
@ -5,15 +5,15 @@ import net from "node:net";
|
|||||||
import { describe, expect, it, afterEach } from "vitest";
|
import { describe, expect, it, afterEach } from "vitest";
|
||||||
import { PDFDocument } from "pdf-lib";
|
import { PDFDocument } from "pdf-lib";
|
||||||
import { PNG } from "pngjs";
|
import { PNG } from "pngjs";
|
||||||
import { getHandler } from "../engine/registry";
|
import { getHandler } from "../../src/engine/registry";
|
||||||
import type { RunContext } from "../engine/types";
|
import type { RunContext } from "../../src/engine/types";
|
||||||
import type { CatalogComponent, FlowNode } from "../../shared/types";
|
import type { CatalogComponent, FlowNode } from "../../src/types";
|
||||||
import "./desktop";
|
import "../../src/capabilities/desktop";
|
||||||
import "./office";
|
import "../../src/capabilities/office";
|
||||||
import "./pdf";
|
import "../../src/capabilities/pdf";
|
||||||
import "./message";
|
import "../../src/capabilities/message";
|
||||||
import "./word";
|
import "../../src/capabilities/word";
|
||||||
import { findImage } from "./vision";
|
import { findImage } from "../../src/capabilities/vision";
|
||||||
|
|
||||||
const tempDirs: string[] = [];
|
const tempDirs: string[] = [];
|
||||||
const node = (componentId: string): FlowNode => ({
|
const node = (componentId: string): FlowNode => ({
|
||||||
60
packages/rpa-core/test/capabilities/captcha-caps.test.ts
Normal file
60
packages/rpa-core/test/capabilities/captcha-caps.test.ts
Normal file
@ -0,0 +1,60 @@
|
|||||||
|
import { existsSync } from "node:fs";
|
||||||
|
import path from "node:path";
|
||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { getHandler } from "../../src/engine/registry";
|
||||||
|
import type { CatalogComponent, FlowNode } from "../../src/types";
|
||||||
|
import type { RunContext } from "../../src/engine/types";
|
||||||
|
import "../../src/capabilities/remaining-core";
|
||||||
|
import "../../src/capabilities/captcha-caps"; // last-wins:覆盖 remaining-core 的通用实现
|
||||||
|
import { cleanCodeText, parseCalculation } from "../../src/capabilities/captcha-caps";
|
||||||
|
|
||||||
|
// 测试图随仓库走(__fixtures__/captcha,PIL 生成),真实 OCR 模型来自 npm 依赖
|
||||||
|
const fixturesDir = path.join(import.meta.dirname, "__fixtures__", "captcha");
|
||||||
|
const fixture = (name: string) => path.join(fixturesDir, `${name}.png`);
|
||||||
|
const ctx = { signal: new AbortController().signal, log: () => {}, vars: {}, evidenceDir: fixturesDir } as unknown as RunContext;
|
||||||
|
const node: FlowNode = { id: "smoke", componentId: "smoke", position: { x: 0, y: 0 }, values: {}, options: {} };
|
||||||
|
|
||||||
|
describe("parseCalculation 解析层(纯逻辑)", () => {
|
||||||
|
it("基础四则与等号问号", () => {
|
||||||
|
expect(parseCalculation("3+5=?")).toEqual({ expression: "3+5", result: 8 });
|
||||||
|
});
|
||||||
|
it("x/X/* 归一为乘", () => {
|
||||||
|
expect(parseCalculation("6x4=?")!.result).toBe(24);
|
||||||
|
expect(parseCalculation("9X9=?")!.result).toBe(81);
|
||||||
|
});
|
||||||
|
it("全角与杂字符", () => {
|
||||||
|
expect(parseCalculation("12+7=?")!.result).toBe(19);
|
||||||
|
});
|
||||||
|
it("长破折号/Unicode 减号", () => {
|
||||||
|
expect(parseCalculation("12—7")!.result).toBe(5);
|
||||||
|
});
|
||||||
|
it("除法整除与小数", () => {
|
||||||
|
expect(parseCalculation("8/2")!.result).toBe(4);
|
||||||
|
expect(parseCalculation("7/2")!.result).toBe(3.5);
|
||||||
|
});
|
||||||
|
it("无算式返回 null", () => {
|
||||||
|
expect(parseCalculation("ab12cd")).toBeNull();
|
||||||
|
});
|
||||||
|
it("字符验证码清洗", () => {
|
||||||
|
expect(cleanCodeText("a b 7 X ? = ")).toBe("ab7X");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe.skipIf(!existsSync(fixture("captcha_0_3p5_q")))("OCR_CalculationCode 端到端(真实模型)", () => {
|
||||||
|
const cases: Array<[string, string]> = [
|
||||||
|
["captcha_0_3p5_q", "8"],
|
||||||
|
["captcha_1_12m7_q", "5"],
|
||||||
|
["captcha_2_6t4_q", "24"],
|
||||||
|
["captcha_3_15p28_q", "43"],
|
||||||
|
["captcha_4_9t9_q", "81"],
|
||||||
|
];
|
||||||
|
for (const [file, expected] of cases) {
|
||||||
|
it(`${file} -> ${expected}`, async () => {
|
||||||
|
const handler = getHandler("OCR_CalculationCode");
|
||||||
|
expect(handler).toBeTruthy();
|
||||||
|
const out = await handler!(ctx, node, { path: fixture(file) }, {} as CatalogComponent);
|
||||||
|
console.log(`[${file}]`, String(out));
|
||||||
|
expect(String(out)).toBe(expected);
|
||||||
|
}, 300000);
|
||||||
|
}
|
||||||
|
});
|
||||||
20
packages/rpa-core/test/capabilities/captcha-recon.test.ts
Normal file
20
packages/rpa-core/test/capabilities/captcha-recon.test.ts
Normal file
@ -0,0 +1,20 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { getHandler } from "../../src/engine/registry";
|
||||||
|
import type { CatalogComponent, FlowNode } from "../../src/types";
|
||||||
|
import type { RunContext } from "../../src/engine/types";
|
||||||
|
import "../../src/capabilities/remaining-core";
|
||||||
|
|
||||||
|
const ctx = { signal: new AbortController().signal, log: () => {}, vars: {}, evidenceDir: "D:/office-test2/ocr-test" } as unknown as RunContext;
|
||||||
|
const node: FlowNode = { id: "smoke", componentId: "smoke", position: { x: 0, y: 0 }, values: {}, options: {} };
|
||||||
|
|
||||||
|
describe("算术验证码现状侦察", () => {
|
||||||
|
const files = ["captcha_0_3p5_q", "captcha_1_12m7_q", "captcha_2_6t4_q", "captcha_3_15p28_q", "captcha_4_9t9_q"];
|
||||||
|
it("当前 OCR 原始输出", async () => {
|
||||||
|
for (const f of files) {
|
||||||
|
const handler = getHandler("OCR_CalculationCode");
|
||||||
|
const out = await handler!(ctx, node, { path: `D:/office-test2/ocr-test/${f}.png` }, {} as CatalogComponent);
|
||||||
|
console.log(`[${f}]`, JSON.stringify(out));
|
||||||
|
}
|
||||||
|
expect(true).toBe(true);
|
||||||
|
}, 600000);
|
||||||
|
});
|
||||||
@ -4,10 +4,10 @@ import os from "node:os";
|
|||||||
import path from "node:path";
|
import path from "node:path";
|
||||||
import { PDFDocument } from "pdf-lib";
|
import { PDFDocument } from "pdf-lib";
|
||||||
import { afterEach, describe, expect, it } from "vitest";
|
import { afterEach, describe, expect, it } from "vitest";
|
||||||
import type { CatalogComponent, FlowNode } from "../../shared/types";
|
import type { CatalogComponent, FlowNode } from "../../src/types";
|
||||||
import type { RunContext } from "../engine/types";
|
import type { RunContext } from "../../src/engine/types";
|
||||||
import { getHandler } from "../engine/registry";
|
import { getHandler } from "../../src/engine/registry";
|
||||||
import "./desktop"; import "./pdf"; import "./system-extended"; import "./window-extended"; import "./common-final";
|
import "../../src/capabilities/index";
|
||||||
|
|
||||||
const dirs: string[] = []; const context = { vars: {} } as RunContext; const component = {} as CatalogComponent;
|
const dirs: string[] = []; const context = { vars: {} } as RunContext; const component = {} as CatalogComponent;
|
||||||
const node = (id: string): FlowNode => ({ id, componentId: id, position: { x: 0, y: 0 }, values: {}, options: {} });
|
const node = (id: string): FlowNode => ({ id, componentId: id, position: { x: 0, y: 0 }, values: {}, options: {} });
|
||||||
77
packages/rpa-core/test/capabilities/compat-audit.test.ts
Normal file
77
packages/rpa-core/test/capabilities/compat-audit.test.ts
Normal file
@ -0,0 +1,77 @@
|
|||||||
|
import { describe, it } from "vitest";
|
||||||
|
import { capabilityIds } from "../../src/engine/registry";
|
||||||
|
import { writeFileSync, readFileSync } from "node:fs";
|
||||||
|
import os from "node:os";
|
||||||
|
import path from "node:path";
|
||||||
|
import "../../src/capabilities/index"; // 汇总导入全部能力模块(与运行时同构)
|
||||||
|
|
||||||
|
interface CatalogComp {
|
||||||
|
id: string;
|
||||||
|
name?: string;
|
||||||
|
fields?: { key: string; label?: string; widget?: string; options?: { label: string; value: string }[]; defaultValue?: unknown }[];
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("系统组件兼容性全量审查", () => {
|
||||||
|
it("注册覆盖 + 面板可见性 + 字段契约核对", async () => {
|
||||||
|
const catalog: CatalogComp[] = JSON.parse(readFileSync("resources/catalog.json", "utf-8")).components ?? [];
|
||||||
|
const dup = JSON.parse(readFileSync("resources/capability-duplicates.json", "utf-8"));
|
||||||
|
const dupMap = new Map<string, { status: string; canonical?: string }>(
|
||||||
|
(Array.isArray(dup) ? dup : []).map((d: { id: string; status: string; canonical?: string }) => [d.id, { status: d.status, canonical: d.canonical }])
|
||||||
|
);
|
||||||
|
const registered = new Set(capabilityIds());
|
||||||
|
|
||||||
|
// 1) catalog 组件 → 运行时注册
|
||||||
|
const unregistered: { id: string; name?: string }[] = [];
|
||||||
|
for (const c of catalog) {
|
||||||
|
if (!registered.has(c.id)) unregistered.push({ id: c.id, name: c.name });
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2) 幽灵注册:handler 有但 catalog 没有(可能是内部别名,不一定是问题,但要列出来)
|
||||||
|
const catalogIds = new Set(catalog.map((c) => c.id));
|
||||||
|
const ghosts: string[] = [];
|
||||||
|
for (const id of registered) {
|
||||||
|
if (!catalogIds.has(id)) ghosts.push(id);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3) duplicates 一致性:hide 必须有 canonical 且 canonical 已注册;keep/隐藏组件的执行器必须在
|
||||||
|
const dupIssues: string[] = [];
|
||||||
|
for (const [id, info] of dupMap) {
|
||||||
|
if (info.status === "hide" && info.canonical && !registered.has(info.canonical)) {
|
||||||
|
dupIssues.push(`hide 项 ${id} 的 canonical ${info.canonical} 未注册`);
|
||||||
|
}
|
||||||
|
if (info.status === "keep" && !registered.has(id)) {
|
||||||
|
dupIssues.push(`keep 项 ${id} 未注册(面板可见但运行会报未实现)`);
|
||||||
|
}
|
||||||
|
if (info.status === "hide" && !info.canonical && !registered.has(id)) {
|
||||||
|
dupIssues.push(`hide 项 ${id} 无 canonical 且未注册(旧流程节点将无法运行)`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4) 字段契约抽查:select 枚举 defaultValue 必须在 options 里;hidden 字段不能 required
|
||||||
|
const fieldIssues: string[] = [];
|
||||||
|
for (const c of catalog) {
|
||||||
|
for (const f of c.fields ?? []) {
|
||||||
|
if (f.widget === "select" && Array.isArray(f.options) && f.options.length > 0) {
|
||||||
|
const values = f.options.map((o) => String(o.value));
|
||||||
|
if (f.defaultValue !== undefined && f.defaultValue !== "" && !values.includes(String(f.defaultValue))) {
|
||||||
|
fieldIssues.push(`${c.id}.${f.key} select 默认值 "${f.defaultValue}" 不在选项值 [${values.join(",")}] 中`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const report = {
|
||||||
|
catalogTotal: catalog.length,
|
||||||
|
registeredTotal: registered.size,
|
||||||
|
unregisteredCount: unregistered.length,
|
||||||
|
unregistered: unregistered.slice(0, 50),
|
||||||
|
ghostCount: ghosts.length,
|
||||||
|
ghosts: ghosts.slice(0, 40),
|
||||||
|
dupIssues,
|
||||||
|
fieldIssues: fieldIssues.slice(0, 40),
|
||||||
|
fieldIssueCount: fieldIssues.length
|
||||||
|
};
|
||||||
|
// 报告写系统临时目录:不得污染仓库工作树(对面 CI 门禁要求干净树)
|
||||||
|
writeFileSync(path.join(os.tmpdir(), ".compat-report.txt"), JSON.stringify(report, null, 1), "utf8");
|
||||||
|
}, 120000);
|
||||||
|
});
|
||||||
146
packages/rpa-core/test/capabilities/cv-vision.test.ts
Normal file
146
packages/rpa-core/test/capabilities/cv-vision.test.ts
Normal file
@ -0,0 +1,146 @@
|
|||||||
|
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import fs from "node:fs";
|
||||||
|
import os from "node:os";
|
||||||
|
import path from "node:path";
|
||||||
|
|
||||||
|
const powershellMock = vi.hoisted(() => vi.fn(async () => '{"ok":true}'));
|
||||||
|
const detectMock = vi.hoisted(() => vi.fn(async () => ({
|
||||||
|
texts: [
|
||||||
|
{ text: "确认保存", box: [[0, 0], [20, 0], [20, 10], [0, 10]] },
|
||||||
|
{ text: "确认", box: [[100, 200], [130, 200], [130, 210], [100, 210]] }
|
||||||
|
]
|
||||||
|
})));
|
||||||
|
// 屏幕服务模拟: 默认无画面(走 GDI 回退), 多屏用例按显示器 id/请求宽度提供对应画面
|
||||||
|
const electronState = vi.hoisted(() => ({ sources: [] as Array<{ display_id: string; thumbnail: any }> }));
|
||||||
|
|
||||||
|
vi.mock("../driver/windows/powershell", () => ({ powershell: powershellMock }));
|
||||||
|
vi.mock("@gutenye/ocr-node", () => ({
|
||||||
|
default: { create: vi.fn(async () => ({ detect: detectMock })) }
|
||||||
|
}));
|
||||||
|
|
||||||
|
import { getHandler } from "../../src/engine/registry";
|
||||||
|
import { setHostBridge, type HostDisplay } from "../../src/engine/host-bridge";
|
||||||
|
import "../../src/capabilities/cv-vision";
|
||||||
|
|
||||||
|
const displayOf = (id: string, x: number, width: number, height: number, scaleFactor: number): HostDisplay => ({
|
||||||
|
id,
|
||||||
|
bounds: { x, y: 0, width, height },
|
||||||
|
scaleFactor
|
||||||
|
});
|
||||||
|
const displays: HostDisplay[] = [
|
||||||
|
displayOf("1", 0, 1920, 1080, 1),
|
||||||
|
displayOf("2", 1920, 1280, 720, 2)
|
||||||
|
];
|
||||||
|
|
||||||
|
setHostBridge({
|
||||||
|
createFormWindow: () => { throw new Error("not used in cv-vision tests"); },
|
||||||
|
dipToScreenPoint: (point) => ({ x: point.x, y: point.y }),
|
||||||
|
screenToDipPoint: (point) => ({ x: point.x, y: point.y }),
|
||||||
|
loadAiConfig: () => ({ baseUrl: "", apiKey: "", model: "", visionModel: "" }),
|
||||||
|
screen: {
|
||||||
|
listDisplays: () => displays,
|
||||||
|
primaryDisplay: () => displays[0],
|
||||||
|
dipToScreenPoint: (point) => ({ x: point.x, y: point.y }),
|
||||||
|
displayMatching: () => displays[0],
|
||||||
|
captureDisplay: async (displayId: string, wantWidth: number, wantHeight: number) => {
|
||||||
|
// 模拟真实 captureDisplay 的对位规则,但不做 sources[0] 兜底:
|
||||||
|
// 保持"主屏无画面 → 跳过该屏"的旧多屏语义
|
||||||
|
const source = electronState.sources.find((item) => item.display_id === displayId && item.thumbnail.getSize().width === wantWidth)
|
||||||
|
|| electronState.sources.find((item) => item.thumbnail.getSize().width === wantWidth);
|
||||||
|
if (!source || source.thumbnail.isEmpty()) throw new Error(`无法获取显示器 ${displayId} 的屏幕截图`);
|
||||||
|
void wantHeight;
|
||||||
|
return source.thumbnail.toPNG();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("CV vision capabilities", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
powershellMock.mockClear();
|
||||||
|
detectMock.mockClear();
|
||||||
|
electronState.sources = [];
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each(["CV_GetText", "CV_SetText", "CV_MouseHover", "CV_MouseClick"])("registers %s", (id) => {
|
||||||
|
expect(getHandler(id), id).toBeTruthy();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects an empty CV selector", async () => {
|
||||||
|
await expect(getHandler("CV_GetText")!({ log: vi.fn() } as any, { id: "n1", componentId: "CV_GetText" } as any, {}, {} as any))
|
||||||
|
.rejects.toThrow("CV 目标为空");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("locates OCR text and clicks its center with offset", async () => {
|
||||||
|
const handler = getHandler("CV_MouseClick")!;
|
||||||
|
await expect(handler({ log: vi.fn() } as any, { id: "n1", componentId: "CV_MouseClick" } as any, {
|
||||||
|
cvSelector: "确认",
|
||||||
|
offsetX: 5,
|
||||||
|
offsetY: 2
|
||||||
|
}, {} as any)).resolves.toBe(true);
|
||||||
|
expect(detectMock).toHaveBeenCalledTimes(1);
|
||||||
|
const script = String((powershellMock.mock.calls as unknown as Array<[string]>)[1]?.[0] ?? "");
|
||||||
|
expect(script).toContain("SetCursorPos(15, 7)");
|
||||||
|
expect(script).toContain("mouse_event(0x0002");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns matched OCR text", async () => {
|
||||||
|
await expect(getHandler("CV_GetText")!({ log: vi.fn() } as any, { id: "n2", componentId: "CV_GetText" } as any, {
|
||||||
|
cvSelector: "保存"
|
||||||
|
}, {} as any)).resolves.toBe("确认保存");
|
||||||
|
expect(powershellMock).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("restricts matching to the search region when provided", async () => {
|
||||||
|
// 屏幕上有两处「确认」: 区域(0,0,50,50)内只应命中 (5,5) 那处, 而不是第一个全屏命中 (115,205)
|
||||||
|
await expect(getHandler("CV_MouseClick")!({ log: vi.fn() } as any, { id: "n3", componentId: "CV_MouseClick" } as any, {
|
||||||
|
cvSelector: "确认",
|
||||||
|
searchRegion: JSON.stringify({ x: 0, y: 0, width: 50, height: 50 })
|
||||||
|
}, {} as any)).resolves.toBe(true);
|
||||||
|
const script = String((powershellMock.mock.calls as unknown as Array<[string]>)[1]?.[0] ?? "");
|
||||||
|
expect(script).toContain("SetCursorPos(10, 5)");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects when target is outside the search region", async () => {
|
||||||
|
await expect(getHandler("CV_MouseClick")!({ log: vi.fn() } as any, { id: "n4", componentId: "CV_MouseClick" } as any, {
|
||||||
|
cvSelector: "确认",
|
||||||
|
searchRegion: JSON.stringify({ x: 500, y: 500, width: 50, height: 50 })
|
||||||
|
}, {} as any)).rejects.toThrow("未找到目标文本");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("locates targets on a secondary display and maps DIP coordinates", async () => {
|
||||||
|
// 主屏无画面, 副屏(bounds 1920,0 1280x720 缩放2)物理中心(450,210) → DIP(2145,105)
|
||||||
|
const fakeThumb = (w: number, h: number): any => ({
|
||||||
|
isEmpty: () => false,
|
||||||
|
getSize: () => ({ width: w, height: h }),
|
||||||
|
toPNG: () => Buffer.from([137, 80, 78, 71])
|
||||||
|
});
|
||||||
|
electronState.sources = [{ display_id: "2", thumbnail: fakeThumb(2560, 1440) }];
|
||||||
|
detectMock.mockImplementationOnce(async () => ({
|
||||||
|
texts: [{ text: "副屏按钮", box: [[400, 200], [500, 200], [500, 220], [400, 220]] }]
|
||||||
|
}));
|
||||||
|
await expect(getHandler("CV_MouseClick")!({ log: vi.fn() } as any, { id: "n5", componentId: "CV_MouseClick" } as any, {
|
||||||
|
cvSelector: "副屏按钮"
|
||||||
|
}, {} as any)).resolves.toBe(true);
|
||||||
|
const script = String((powershellMock.mock.calls as unknown as Array<[string]>).at(-1)?.[0] ?? "");
|
||||||
|
expect(script).toContain("SetCursorPos(2145, 105)");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("maps region-image hits through the display scale", async () => {
|
||||||
|
// 框选图来自缩放2的副屏: 图内物理中心(120,90) + 框选DIP原点(1920,0) → DIP(1980,45)
|
||||||
|
const regionShot = path.join(os.tmpdir(), "rpa-cv-region-test-" + Date.now() + ".png");
|
||||||
|
fs.writeFileSync(regionShot, Buffer.from([137, 80, 78, 71]));
|
||||||
|
try {
|
||||||
|
detectMock.mockImplementationOnce(async () => ({
|
||||||
|
texts: [{ text: "确认", box: [[100, 80], [140, 80], [140, 100], [100, 100]] }]
|
||||||
|
}));
|
||||||
|
await expect(getHandler("CV_MouseHover")!({ log: vi.fn() } as any, { id: "n6", componentId: "CV_MouseHover" } as any, {
|
||||||
|
cvSelector: "确认",
|
||||||
|
searchRegion: JSON.stringify({ x: 1920, y: 0, width: 640, height: 360, scale: 2, imagePath: regionShot })
|
||||||
|
}, {} as any)).resolves.toBe(true);
|
||||||
|
const script = String((powershellMock.mock.calls as unknown as Array<[string]>).at(-1)?.[0] ?? "");
|
||||||
|
expect(script).toContain("SetCursorPos(1980, 45)");
|
||||||
|
} finally {
|
||||||
|
try { fs.unlinkSync(regionShot); } catch {}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
@ -2,10 +2,10 @@ import { mkdtemp, readFile, rm } from "node:fs/promises";
|
|||||||
import os from "node:os";
|
import os from "node:os";
|
||||||
import path from "node:path";
|
import path from "node:path";
|
||||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
import type { Catalog, CatalogComponent, FlowNode } from "../../shared/types";
|
import type { Catalog, CatalogComponent, FlowNode } from "../../src/types";
|
||||||
import type { RunContext } from "../engine/types";
|
import type { RunContext } from "../../src/engine/types";
|
||||||
import { getHandler } from "../engine/registry";
|
import { getHandler } from "../../src/engine/registry";
|
||||||
import "./data-extended";
|
import "../../src/capabilities/data-extended";
|
||||||
|
|
||||||
const ids = [
|
const ids = [
|
||||||
"Collection_Create", "Collection_HasItem", "Collection_Count", "Collection_Remove", "Collection_Add",
|
"Collection_Create", "Collection_HasItem", "Collection_Count", "Collection_Remove", "Collection_Add",
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user