阶段2-断Electron依赖: human-dialog/window-extended/ai-service经HostBridge迁入包, external+transport迁包(session留宿主), 新增check:deps依赖护栏与包README留守清单
This commit is contained in:
yu authored and trae-solo-agent [origin=hook_stop] committed
1 parent
caa8e19332
commit
3decced268
21 files changed
+505
-236
No files matched your search
@@ -0,0 +1,106 @@
|
||||
import { getHostBridge } from "../engine/host-bridge";
|
||||
import { RuntimeError } from "../engine/types";
|
||||
|
||||
export type ChatContentPart =
|
||||
| { type: "text"; text: string }
|
||||
| { type: "image_url"; image_url: { url: string } };
|
||||
|
||||
export interface ChatMessage {
|
||||
role: "system" | "user" | "assistant";
|
||||
content: string | ChatContentPart[];
|
||||
}
|
||||
|
||||
export interface ResolvedAiConfig {
|
||||
baseUrl: string;
|
||||
apiKey: string;
|
||||
model: string;
|
||||
visionModel: string;
|
||||
}
|
||||
|
||||
export function getAiConfig(): ResolvedAiConfig {
|
||||
const bridge = getHostBridge();
|
||||
// 宿主桥未注入（vitest / CLI 未配置）时返回空配置，requireAiConfig 统一抛 ai_not_configured
|
||||
const config = bridge ? bridge.loadAiConfig() : { baseUrl: "", apiKey: "", model: "", visionModel: "" };
|
||||
return { ...config, visionModel: config.visionModel || config.model };
|
||||
}
|
||||
|
||||
export function requireAiConfig(options: { vision?: boolean } = {}): ResolvedAiConfig {
|
||||
const config = getAiConfig();
|
||||
if (!config.baseUrl || !config.model || (options.vision && !config.visionModel)) {
|
||||
throw new RuntimeError("ai_not_configured", "AI 服务未配置", {
|
||||
recommendation: options.vision
|
||||
? "到 设置 → AI 服务 填写 Base URL 与多模态视觉模型（兼容 OpenAI 协议）"
|
||||
: "到 设置 → AI 服务 填写 Base URL 与模型名（兼容 OpenAI 协议）"
|
||||
});
|
||||
}
|
||||
return config;
|
||||
}
|
||||
|
||||
export interface ChatCompletionOptions {
|
||||
temperature?: number;
|
||||
vision?: boolean;
|
||||
/** 交互式请求（用户正在等待，如 AI 拾取）：默认短超时 45s、最多 2 次，超时不重试 */
|
||||
interactive?: boolean;
|
||||
/** 单次请求超时毫秒数；显式指定后覆盖交互式/后台默认值 */
|
||||
timeoutMs?: number;
|
||||
/** 最大尝试次数；显式指定后覆盖交互式/后台默认值 */
|
||||
maxAttempts?: number;
|
||||
}
|
||||
|
||||
export async function chatCompletion(messages: ChatMessage[], options: ChatCompletionOptions = {}): Promise<string> {
|
||||
const config = requireAiConfig({ vision: options.vision });
|
||||
const base = config.baseUrl.replace(/\/+$/, "");
|
||||
const interactive = options.interactive === true;
|
||||
const timeoutMs = Math.max(1000, Math.round(options.timeoutMs ?? (interactive ? 45000 : 170000)));
|
||||
const maxAttempts = Math.max(1, Math.round(options.maxAttempts ?? (interactive ? 2 : 3)));
|
||||
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch(base + "/chat/completions", {
|
||||
signal: AbortSignal.timeout(timeoutMs),
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", Authorization: "Bearer " + config.apiKey },
|
||||
body: JSON.stringify({
|
||||
model: options.vision ? config.visionModel : config.model,
|
||||
messages,
|
||||
temperature: options.temperature ?? 0.3
|
||||
})
|
||||
});
|
||||
} catch (error) {
|
||||
// 超时（AbortSignal.timeout）不重试：重发只会成倍拉长等待
|
||||
if ((error as { name?: string }).name === "TimeoutError") {
|
||||
throw new RuntimeError("ai_request_timeout", `AI 请求超时（${timeoutMs}ms），请检查网络或稍后重试`);
|
||||
}
|
||||
if (attempt === maxAttempts) throw error;
|
||||
await new Promise(resolve => setTimeout(resolve, 1200 * attempt));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (response.ok) {
|
||||
const data = (await response.json()) as { choices?: Array<{ message?: { content?: string } }> };
|
||||
return data.choices?.[0]?.message?.content ?? "";
|
||||
}
|
||||
|
||||
const detail = (await response.text()).slice(0, 300);
|
||||
const retryable = response.status === 408 || response.status === 429 || response.status >= 500;
|
||||
if (retryable && attempt < maxAttempts) {
|
||||
const retryAfter = Number(response.headers.get("retry-after"));
|
||||
const fallbackDelay = 1400 * attempt;
|
||||
const delay = Number.isFinite(retryAfter) && retryAfter >= 0 && retryAfter <= 15
|
||||
? retryAfter * 1000
|
||||
: fallbackDelay;
|
||||
await new Promise(resolve => setTimeout(resolve, delay));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (response.status === 429) {
|
||||
const suffix = detail ? " 详情：" + detail.slice(0, 180) : "";
|
||||
throw new RuntimeError("ai_rate_limited", "AI 服务限流或配额不足（HTTP 429），请稍后重试或更换可用视觉模型。" + suffix);
|
||||
}
|
||||
|
||||
throw new RuntimeError("ai_request_failed", "AI 请求失败（HTTP " + response.status + "）：" + detail);
|
||||
}
|
||||
|
||||
throw new RuntimeError("ai_request_failed", "AI 请求失败：已重试仍未成功");
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
import { createServer } from "node:http";
|
||||
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { PDFDocument } from "pdf-lib";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import type { CatalogComponent, FlowNode } from "../types";
|
||||
import type { RunContext } from "../engine/types";
|
||||
import { getHandler } from "../engine/registry";
|
||||
import "./index";
|
||||
|
||||
const dirs: string[] = []; const context = { vars: {} } as RunContext; const component = {} as CatalogComponent;
|
||||
const node = (id: string): FlowNode => ({ id, componentId: id, position: { x: 0, y: 0 }, values: {}, options: {} });
|
||||
async function run(id: string, args: Record<string, unknown> = {}) { const handler = getHandler(id); expect(handler, id).toBeTruthy(); return handler!(context, node(id), args, component); }
|
||||
afterEach(async () => { context.vars = {}; await Promise.all(dirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); });
|
||||
|
||||
describe("final common capabilities", () => {
|
||||
it("registers window, keyboard, clipboard, system, PDF and webhook components", () => {
|
||||
for (const id of ["Window_ActiveWindow", "Window_CloseWindow_v1", "Window_GetScreenSize", "Window_WindowList", "Window_GetRect", "Window_MaximumSize", "Window_SetWindowShow", "Window_WaitWindow", "Keyboard_DriveSend", "Keyboard_DriveSend_V1", "Keyboard_DriveSend_V2", "Clipboard_SetImage", "Clipboard_SetImageV1", "Clipboard_SaveImage", "Clipboard_SaveImageV1", "System_PlaySound", "CustomConfig_GetConfig", "Basic_BusinessOrderCount", "PDF_ConcatPdfV1", "PDF_SetPdfWaterMark", "PDF_SetPdfWaterMarkV1", "MessageNotification_DingTalk_Message", "MessageNotification_Feishu_Message", "MessageNotification_WXWork_Message"]) expect(getHandler(id), id).toBeTruthy();
|
||||
});
|
||||
|
||||
it("reads custom config and increments business counters", async () => {
|
||||
await expect(run("CustomConfig_GetConfig", { jsonlist: [{ name: "timeout", value: 30 }], configName: "timeout" })).resolves.toBe(30);
|
||||
await expect(run("Basic_BusinessOrderCount", { start_count: 10, count: 2 })).resolves.toBe(12);
|
||||
await expect(run("Basic_BusinessOrderCount", { start_count: 10, count: 2 })).resolves.toBe(14);
|
||||
});
|
||||
|
||||
it("adds an ASCII PDF watermark", async () => {
|
||||
const directory = await mkdtemp(path.join(os.tmpdir(), "rpa-common-final-")); dirs.push(directory); const input = path.join(directory, "input.pdf"); const pdf = await PDFDocument.create(); pdf.addPage([300, 300]); await writeFile(input, await pdf.save());
|
||||
const output = await run("PDF_SetPdfWaterMark", { fileInPath: input, markTextInfo: "RPA", newPdfName: "watermark.pdf" }) as string;
|
||||
expect((await readFile(output)).byteLength).toBeGreaterThan(100);
|
||||
});
|
||||
|
||||
it("posts the three webhook formats to a local server", async () => {
|
||||
const bodies: unknown[] = []; const server = createServer((request, response) => { let raw = ""; request.on("data", (chunk) => raw += chunk); request.on("end", () => { bodies.push(JSON.parse(raw)); response.writeHead(200, { "Content-Type": "application/json" }); response.end('{"ok":true}'); }); });
|
||||
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve)); const address = server.address(); const webhook = `http://127.0.0.1:${typeof address === "object" && address ? address.port : 0}/hook`;
|
||||
try { await run("MessageNotification_DingTalk_Message", { webhook, content_text: "ding" }); await run("MessageNotification_Feishu_Message", { webhook, content_text: "fei" }); await run("MessageNotification_WXWork_Message", { webhook, content_text: "wx" }); expect(bodies).toHaveLength(3); } finally { await new Promise<void>((resolve) => server.close(() => resolve())); }
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,105 @@
|
||||
import { getHostBridge, type HostDialogWindow } from "../engine/host-bridge";
|
||||
import { register } from "../engine/registry";
|
||||
import { RuntimeError } from "../engine/types";
|
||||
|
||||
interface FormFieldSpec { key?: string; name?: string; label?: string; type?: string; value?: string; options?: unknown[] }
|
||||
|
||||
function parseFields(raw: unknown, nodeId: string): FormFieldSpec[] {
|
||||
if (raw === undefined || raw === null || raw === "") {
|
||||
throw new RuntimeError("validation_error", "表单配置 jsonlist 为空", { nodeId, componentId: nodeId, recommendation: "填写 JSON 数组，例如 [{\"key\":\"name\",\"label\":\"名称\",\"type\":\"text\"}]" });
|
||||
}
|
||||
if (Array.isArray(raw)) return raw as FormFieldSpec[];
|
||||
const text = String(raw).trim();
|
||||
if (text.startsWith("[")) {
|
||||
try {
|
||||
const parsed = JSON.parse(text) as unknown;
|
||||
if (Array.isArray(parsed)) return parsed as FormFieldSpec[];
|
||||
} catch { /* fall through */ }
|
||||
}
|
||||
throw new RuntimeError("validation_error", "表单配置 jsonlist 无效", { nodeId, componentId: nodeId, recommendation: "填写 JSON 数组，例如 [{\"key\":\"name\",\"label\":\"名称\",\"type\":\"text\"}]" });
|
||||
}
|
||||
|
||||
function fieldKey(f: FormFieldSpec, index: number): string {
|
||||
return String(f.key ?? f.name ?? f.label ?? "field" + (index + 1));
|
||||
}
|
||||
|
||||
function escapeHtml(s: string): string {
|
||||
return s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """);
|
||||
}
|
||||
|
||||
async function showForm(options: { title: string; fields: FormFieldSpec[]; timeoutSec: number }): Promise<Record<string, string>> {
|
||||
const bridge = getHostBridge();
|
||||
if (!bridge) {
|
||||
throw new RuntimeError("host_unavailable", "宿主桥未注入，无法打开人机交互窗口", {
|
||||
recommendation: "宿主（Electron 主进程）启动时需调用 setHostBridge() 注入窗口工厂"
|
||||
});
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
const rows = options.fields.map((f, i) => {
|
||||
const key = escapeHtml(fieldKey(f, i));
|
||||
const label = escapeHtml(f.label ?? fieldKey(f, i));
|
||||
const value = escapeHtml(String(f.value ?? ""));
|
||||
const type = String(f.type ?? "text").toLowerCase();
|
||||
if (type === "select" || (Array.isArray(f.options) && f.options.length)) {
|
||||
const opts = (f.options ?? []).map(o => {
|
||||
const ov = typeof o === "string" ? o : String((o as { value?: string }).value ?? (o as { label?: string }).label ?? "");
|
||||
const ol = typeof o === "string" ? o : String((o as { label?: string }).label ?? ov);
|
||||
return '<option value="' + escapeHtml(ov) + '"' + (ov === String(f.value ?? "") ? " selected" : "") + ">" + escapeHtml(ol) + "</option>";
|
||||
}).join("");
|
||||
return "<label>" + label + '<br/><select data-key="' + key + '">' + opts + "</select></label><br/>";
|
||||
}
|
||||
if (type === "textarea") {
|
||||
return "<label>" + label + '<br/><textarea data-key="' + key + '" rows="3">' + value + "</textarea></label><br/>";
|
||||
}
|
||||
if (type === "checkbox") {
|
||||
const checked = String(f.value ?? "") === "true" || String(f.value ?? "") === "1" ? " checked" : "";
|
||||
return '<label><input type="checkbox" data-key="' + key + '"' + checked + " /> " + label + "</label><br/>";
|
||||
}
|
||||
const inputType = type === "password" ? "password" : "text";
|
||||
return "<label>" + label + '<br/><input type="' + inputType + '" data-key="' + key + '" value="' + value + '" /></label><br/>';
|
||||
}).join("\n");
|
||||
const html = '<!DOCTYPE html><html><head><meta charset="utf-8"><style>body{font-family:Microsoft YaHei,sans-serif;margin:16px;font-size:14px;color:#1f2937}label{display:block;margin-bottom:10px}input,select,textarea{width:100%;box-sizing:border-box;padding:6px;margin-top:4px}#ok{margin-top:8px;padding:8px 24px;background:#2563eb;color:#fff;border:none;border-radius:6px;cursor:pointer}</style></head><body><h3>' + escapeHtml(options.title) + "</h3>" + rows + '<button id="ok">确认</button><script>document.getElementById("ok").onclick=function(){var out={};document.querySelectorAll("[data-key]").forEach(function(el){out[el.dataset.key]=el.type==="checkbox"?el.checked:el.value});window.__formValues=out;};</script></body></html>';
|
||||
const win: HostDialogWindow = bridge.createFormWindow({ title: options.title });
|
||||
let settled = false;
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
let timeout: NodeJS.Timeout | undefined;
|
||||
const finish = (value: Record<string, string>) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
if (timer) clearInterval(timer);
|
||||
if (timeout) clearTimeout(timeout);
|
||||
if (!win.isDestroyed()) win.destroy();
|
||||
resolve(value);
|
||||
};
|
||||
timer = setInterval(async () => {
|
||||
try {
|
||||
const values = await win.executeJavaScript("window.__formValues || null");
|
||||
if (values) finish(values as Record<string, string>);
|
||||
} catch {
|
||||
finish({});
|
||||
}
|
||||
}, 250);
|
||||
timeout = setTimeout(() => {
|
||||
try { void win.executeJavaScript("document.getElementById(\"ok\").click()"); } catch {
|
||||
finish({});
|
||||
}
|
||||
}, Math.max(5, options.timeoutSec) * 1000);
|
||||
win.on("closed", () => finish({}));
|
||||
void win.loadURL("data:text/html;charset=utf-8," + encodeURIComponent(html));
|
||||
});
|
||||
}
|
||||
|
||||
const DIALOG_IDS = [
|
||||
"Dialog_GetConfig", "Dialog_GetConfigV1", "Dialog_GetConfigV2", "Dialog_GetConfigV3",
|
||||
"Dialog_GetConfigV4", "Dialog_GetConfigV5", "Dialog_GetConfigV6", "Dialog_GetConfigV7", "Dialog_GetConfigV8"
|
||||
];
|
||||
|
||||
register(DIALOG_IDS, async (ctx, node, args) => {
|
||||
const fields = parseFields(args.jsonlist, node.id);
|
||||
const title = String(args.title ?? args.configName ?? "请输入");
|
||||
const timeoutSec = Number(args.wait_time ?? 30) || 30;
|
||||
const values = await showForm({ title, fields, timeoutSec });
|
||||
ctx.log("已获取人机交互表单输入：" + JSON.stringify(values), "info", node.id);
|
||||
return values;
|
||||
});
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
/**
|
||||
* rpa-core 纯 Node 能力注册入口。
|
||||
* rpa-core 能力注册入口。
|
||||
*
|
||||
* import 本模块即完成全部纯 Node 能力的副作用注册（与改造前
|
||||
* import 本模块即完成全部包内能力的副作用注册（与改造前
|
||||
* src/main/capabilities/index.ts 的加载顺序保持一致，别名/重复 id 的
|
||||
* last-wins 语义不变）。依赖 Electron 或宿主模块的能力（human-dialog /
|
||||
* window-extended / ai-service / ai-caps / browser* / web-batch2 / web-crawl）
|
||||
* 留在宿主 src/main/capabilities，阶段 2 通过 HostBridge 迁入。
|
||||
* last-wins 语义不变）。依赖 Electron 窗口的浏览器能力（ai-caps / browser* /
|
||||
* web-batch2 / web-crawl）留在宿主 src/main/capabilities；
|
||||
* human-dialog / ai-service / window-extended 已于阶段 2 经 HostBridge 迁入。
|
||||
*/
|
||||
import "./control";
|
||||
import "./files";
|
||||
@@ -37,3 +37,6 @@ import "./basic-encryption";
|
||||
import "./mail-remaining";
|
||||
import "./remaining-core";
|
||||
import "./common-final";
|
||||
import "./human-dialog";
|
||||
import "./ai-service";
|
||||
import "./window-extended";
|
||||
@@ -0,0 +1,94 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import type { CatalogComponent, FlowNode } from "../types";
|
||||
import { getHandler } from "../engine/registry";
|
||||
import type { RunContext } from "../engine/types";
|
||||
import "./window-extended";
|
||||
|
||||
const run = promisify(execFile);
|
||||
const component = {} as CatalogComponent;
|
||||
const node = (id: string): FlowNode => ({ id: "smoke-mouse", componentId: id, position: { x: 0, y: 0 }, values: {}, options: {} });
|
||||
|
||||
const ctx = {
|
||||
runId: "mouse-smoke", flowId: "mouse-smoke-flow", flow: {} as RunContext["flow"],
|
||||
vars: {}, secrets: {}, signal: new AbortController().signal, evidenceDir: "D:/office-test2",
|
||||
emit: () => {}, log: () => {}, componentOf: () => undefined
|
||||
} as unknown as RunContext;
|
||||
|
||||
const PS_API = `Add-Type @'
|
||||
using System; using System.Runtime.InteropServices;
|
||||
public static class RpaMouseSmoke {
|
||||
[StructLayout(LayoutKind.Sequential)] public struct POINT { public int X; public int Y; }
|
||||
[DllImport("user32.dll")] public static extern bool GetCursorPos(out POINT p);
|
||||
[DllImport("user32.dll")] public static extern bool SetProcessDpiAwarenessContext(IntPtr v);
|
||||
}
|
||||
'@; try { [void][RpaMouseSmoke]::SetProcessDpiAwarenessContext([IntPtr](-4)) } catch {}; $c = [RpaMouseSmoke+POINT]::new(); [void][RpaMouseSmoke]::GetCursorPos([ref]$c); "$($c.X),$($c.Y)"`;
|
||||
|
||||
async function cursorPos(): Promise<[number, number]> {
|
||||
const { stdout } = await run("powershell.exe", ["-NoProfile", "-NonInteractive", "-Command", PS_API], { windowsHide: true });
|
||||
return stdout.trim().split(",").map(Number) as [number, number];
|
||||
}
|
||||
|
||||
/** 共享桌面环境下光标可能被其他进程短暂挪动，轮询等待落位（容差 px），超时返回最后一次读数 */
|
||||
async function waitForCursor(target: [number, number], tolerance = 0, attempts = 10, gapMs = 100): Promise<[number, number]> {
|
||||
let pos = await cursorPos();
|
||||
for (let i = 0; i < attempts; i += 1) {
|
||||
if (Math.abs(pos[0] - target[0]) <= tolerance && Math.abs(pos[1] - target[1]) <= tolerance) return pos;
|
||||
await new Promise((r) => setTimeout(r, gapMs));
|
||||
pos = await cursorPos();
|
||||
}
|
||||
return pos;
|
||||
}
|
||||
|
||||
async function invoke(componentId: string, args: Record<string, unknown>): Promise<unknown> {
|
||||
const handler = getHandler(componentId);
|
||||
expect(handler).toBeTypeOf("function");
|
||||
return handler!(ctx, node(componentId), args, component);
|
||||
}
|
||||
|
||||
let original: [number, number];
|
||||
afterEach(async () => {
|
||||
if (original) await invoke("Mouse_MoveToV2", { location: { x: original[0], y: original[1] }, position: "0", move_type: "2", speed: 100 });
|
||||
});
|
||||
|
||||
describe("移动鼠标组件真实冒烟", () => {
|
||||
it("对象坐标 x/y JSON 不再报无效, 瞬间移动真实落位", async () => {
|
||||
original = await cursorPos();
|
||||
// 用户屏幕框选生成的浮点对象坐标（此前直接报「鼠标坐标无效」）
|
||||
const out = await invoke("Mouse_MoveToV2", { location: { x: 1397, y: 227.60000610351562 }, position: "0", move_type: "2", speed: 100 }) as { x: number; y: number };
|
||||
expect(out).toEqual({ x: 1397, y: 228 });
|
||||
const pos = await waitForCursor([1397, 228]);
|
||||
expect(pos[0]).toBe(1397);
|
||||
expect(pos[1]).toBe(228);
|
||||
console.log("[移动鼠标] 对象坐标解析 + 瞬间移动落位 ✓ 实际光标:", pos);
|
||||
}, 60000);
|
||||
|
||||
it("模拟人工移动(默认方式)平滑到达目标", async () => {
|
||||
original = await cursorPos();
|
||||
const out = await invoke("Mouse_MoveToV2", { location: "[300,300]", position: "0", move_type: "0", speed: 60 }) as { x: number; y: number };
|
||||
expect(out).toEqual({ x: 300, y: 300 });
|
||||
const pos = await waitForCursor([300, 300], 2);
|
||||
expect(Math.abs(pos[0] - 300)).toBeLessThanOrEqual(2);
|
||||
expect(Math.abs(pos[1] - 300)).toBeLessThanOrEqual(2);
|
||||
console.log("[移动鼠标] 模拟人工移动 ✓ 实际光标:", pos);
|
||||
}, 60000);
|
||||
|
||||
it("匀速直线移动与字符串数组坐标", async () => {
|
||||
original = await cursorPos();
|
||||
const out = await invoke("Mouse_MoveToV2", { location: "[500,400]", position: "0", move_type: "1", speed: 80 }) as { x: number; y: number };
|
||||
expect(out).toEqual({ x: 500, y: 400 });
|
||||
const pos = await waitForCursor([500, 400]);
|
||||
expect(pos[0]).toBe(500);
|
||||
expect(pos[1]).toBe(400);
|
||||
console.log("[移动鼠标] 匀速直线 + 字符串坐标 ✓ 实际光标:", pos);
|
||||
}, 60000);
|
||||
|
||||
it("相对激活窗口左上角模式返回有效坐标", async () => {
|
||||
original = await cursorPos();
|
||||
const out = await invoke("Mouse_MoveToV2", { location: { x: 10, y: 10 }, position: "1", move_type: "2", speed: 100 }) as { x: number; y: number };
|
||||
expect(Number.isFinite(out.x)).toBe(true);
|
||||
expect(Number.isFinite(out.y)).toBe(true);
|
||||
console.log("[移动鼠标] 相对激活窗口模式 ✓ 目标:", out);
|
||||
}, 60000);
|
||||
});
|
||||
@@ -0,0 +1,33 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { CatalogComponent, FlowNode } from "../types";
|
||||
import type { RunContext } from "../engine/types";
|
||||
import { getHandler } from "../engine/registry";
|
||||
import "./window-extended";
|
||||
|
||||
const context = {} as RunContext;
|
||||
const component = {} as CatalogComponent;
|
||||
const node = (id: string): FlowNode => ({ id, componentId: id, position: { x: 0, y: 0 }, values: {}, options: {} });
|
||||
const ids = [
|
||||
"Window_ActiveWindow_v1", "Window_Get", "Window_GetWindow", "Window_GetWindow_v1", "Window_GetWindowMessage",
|
||||
"Window_GetWindowMessage_v1", "Window_SetPos", "Window_MoveWindow", "Window_MoveWindow_v1", "Window_SizeWindow",
|
||||
"Mouse_MoveToV1", "Mouse_MoveToV2", "Mouse_DragToV1", "Mouse_DragToV2", "Mouse_DragToNew",
|
||||
"Mouse_ClickCombine", "Mouse_ClickCombine_v1", "Mouse_ClickCombine_v2", "Mouse_ClickCombine_v3", "Mouse_ClickCombine_v4"
|
||||
];
|
||||
|
||||
describe("window extended capabilities", () => {
|
||||
it("registers audited window and mouse variants", () => {
|
||||
for (const id of ids) expect(getHandler(id), id).toBeTruthy();
|
||||
});
|
||||
|
||||
it("rejects missing window selectors before touching the desktop", async () => {
|
||||
for (const id of ["Window_GetWindow", "Window_ActiveWindow_v1", "Window_MoveWindow", "Window_SizeWindow"]) {
|
||||
await expect(getHandler(id)!(context, node(id), {}, component)).rejects.toMatchObject({ code: "validation_error" });
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects malformed mouse coordinates before injecting input", async () => {
|
||||
await expect(getHandler("Mouse_MoveToV1")!(context, node("Mouse_MoveToV1"), { location: "bad" }, component)).rejects.toMatchObject({ code: "validation_error" });
|
||||
await expect(getHandler("Mouse_DragToV1")!(context, node("Mouse_DragToV1"), { Start: "bad", End: "[1,2]" }, component)).rejects.toMatchObject({ code: "validation_error" });
|
||||
await expect(getHandler("Mouse_ClickCombine")!(context, node("Mouse_ClickCombine"), { target: "1", location: "[1,2]" }, component)).rejects.toMatchObject({ code: "validation_error" });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,226 @@
|
||||
import { register } from "../engine/registry";
|
||||
import { RuntimeError, type CapabilityHandler } from "../engine/types";
|
||||
import { powershell, psString } from "./desktop";
|
||||
import { getHostBridge } from "../engine/host-bridge";
|
||||
|
||||
const WINDOW_API = `Add-Type @'
|
||||
using System;
|
||||
using System.Runtime.InteropServices;
|
||||
public static class RpaWindowExtended {
|
||||
[StructLayout(LayoutKind.Sequential)] public struct RECT { public int Left; public int Top; public int Right; public int Bottom; }
|
||||
[StructLayout(LayoutKind.Sequential)] public struct POINT { public int X; public int Y; }
|
||||
[DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr hWnd, out RECT rect);
|
||||
[DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd);
|
||||
[DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr hWnd, int command);
|
||||
[DllImport("user32.dll")] public static extern bool SetWindowPos(IntPtr hWnd, IntPtr after, int x, int y, int width, int height, uint flags);
|
||||
[DllImport("user32.dll")] public static extern bool SetCursorPos(int x, int y);
|
||||
[DllImport("user32.dll")] public static extern bool GetCursorPos(out POINT point);
|
||||
[DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow();
|
||||
[DllImport("user32.dll")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra);
|
||||
[DllImport("user32.dll")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value);
|
||||
}
|
||||
'@; try { [void][RpaWindowExtended]::SetProcessDpiAwarenessContext([IntPtr](-4)) } catch {}`;
|
||||
|
||||
function value(args: Record<string, unknown>, keys: string[], fallback = ""): string {
|
||||
for (const key of keys) {
|
||||
const current = args[key];
|
||||
if (current !== undefined && current !== null && current !== "") return String(current);
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function parseObject(candidate: unknown): Record<string, unknown> {
|
||||
if (candidate && typeof candidate === "object") return candidate as Record<string, unknown>;
|
||||
if (typeof candidate === "string" && candidate.trim().startsWith("{")) {
|
||||
try { return JSON.parse(candidate); } catch { return {}; }
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
function titleOf(args: Record<string, unknown>): string {
|
||||
const objectCandidate = args.window_obj ?? args["窗口对象"] ?? args.uiWindowSelector ?? args.target_window;
|
||||
const object = parseObject(objectCandidate);
|
||||
return String(object.title ?? object.windowTitle ?? args.title ?? args.match_title ?? args.select_title ?? objectCandidate ?? "").trim();
|
||||
}
|
||||
|
||||
function requiredTitle(args: Record<string, unknown>, nodeId: string, componentId: string): string {
|
||||
const title = titleOf(args);
|
||||
if (!title || title === "{}" || title === "窗口对象") throw new RuntimeError("validation_error", "窗口标题为空", { nodeId, componentId, recommendation: "填写窗口标题或传入窗口对象" });
|
||||
return title;
|
||||
}
|
||||
|
||||
function findWindow(title: string): string {
|
||||
return `$p = Get-Process | Where-Object { $_.MainWindowHandle -ne 0 -and $_.MainWindowTitle -like ${psString(`*${title}*`)} } | Select-Object -First 1; if (-not $p) { throw '找不到窗口：${title.replace(/'/g, "''")}' }`;
|
||||
}
|
||||
|
||||
function parsePoint(candidate: unknown, label: string, nodeId: string, componentId: string): [number, number] {
|
||||
let parsed: unknown = candidate;
|
||||
if (typeof parsed === "string") {
|
||||
const source = parsed;
|
||||
try { parsed = JSON.parse(source); } catch { parsed = source.split(/[,，xX ]+/).map(Number); }
|
||||
}
|
||||
// 屏幕框选/拾取坐标生成的是 {"x":..,"y":..} 对象，直接转换
|
||||
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
||||
const record = parsed as Record<string, unknown>;
|
||||
if (record.x !== undefined && record.y !== undefined) parsed = [record.x, record.y];
|
||||
}
|
||||
if (!Array.isArray(parsed) || parsed.length < 2 || parsed.slice(0, 2).some((item) => !Number.isFinite(Number(item)))) {
|
||||
throw new RuntimeError("validation_error", `${label}无效`, { nodeId, componentId });
|
||||
}
|
||||
return [Number(parsed[0]), Number(parsed[1])];
|
||||
}
|
||||
|
||||
async function windowObject(title: string): Promise<Record<string, unknown>> {
|
||||
const output = await powershell(`${WINDOW_API}; ${findWindow(title)}; $r = New-Object RpaWindowExtended+RECT; [RpaWindowExtended]::GetWindowRect($p.MainWindowHandle,[ref]$r) | Out-Null; @{pid=$p.Id;title=$p.MainWindowTitle;hwnd=$p.MainWindowHandle.ToInt64();x=$r.Left;y=$r.Top;width=$r.Right-$r.Left;height=$r.Bottom-$r.Top} | ConvertTo-Json -Compress`);
|
||||
return JSON.parse(output);
|
||||
}
|
||||
|
||||
const getWindow: CapabilityHandler = async (_ctx, node, args) => windowObject(requiredTitle(args, node.id, node.componentId));
|
||||
register(["Window_Get", "Window_GetWindow", "Window_GetWindow_v1"], getWindow);
|
||||
|
||||
register("Window_ActiveWindow_v1", async (_ctx, node, args) => {
|
||||
const title = requiredTitle(args, node.id, node.componentId);
|
||||
const maximize = [true, 1, "1", "true", "True"].includes(args.isMax as never);
|
||||
await powershell(`${WINDOW_API}; ${findWindow(title)}; [RpaWindowExtended]::SetForegroundWindow($p.MainWindowHandle) | Out-Null; ${maximize ? "[RpaWindowExtended]::ShowWindow($p.MainWindowHandle,3) | Out-Null" : ""}`);
|
||||
return windowObject(title);
|
||||
});
|
||||
|
||||
register(["Window_GetWindowMessage", "Window_GetWindowMessage_v1"], async (_ctx, node, args) => {
|
||||
const object = await windowObject(requiredTitle(args, node.id, node.componentId));
|
||||
const type = value(args, ["switchType"], "all").toLowerCase();
|
||||
if (type === "title") return object.title;
|
||||
if (type === "pid") return object.pid;
|
||||
if (type === "position" || type === "location") return [object.x, object.y];
|
||||
if (type === "size") return [object.width, object.height];
|
||||
return object;
|
||||
});
|
||||
|
||||
register(["Window_SetPos", "Window_MoveWindow", "Window_MoveWindow_v1", "Window_SizeWindow"], async (_ctx, node, args) => {
|
||||
const title = requiredTitle(args, node.id, node.componentId);
|
||||
const current = await windowObject(title);
|
||||
const moveOnly = node.componentId.includes("MoveWindow");
|
||||
const sizeOnly = node.componentId === "Window_SizeWindow";
|
||||
const position = args.location !== undefined ? parsePoint(args.location, "窗口位置", node.id, node.componentId) : [Number(args.x ?? current.x), Number(args.y ?? current.y)] as [number, number];
|
||||
const sizeCandidate = args.size ?? args.window_size;
|
||||
const size = sizeCandidate !== undefined ? parsePoint(sizeCandidate, "窗口尺寸", node.id, node.componentId) : [Number(args.width ?? current.width), Number(args.height ?? current.height)] as [number, number];
|
||||
const x = sizeOnly ? Number(current.x) : position[0];
|
||||
const y = sizeOnly ? Number(current.y) : position[1];
|
||||
const width = moveOnly ? Number(current.width) : size[0];
|
||||
const height = moveOnly ? Number(current.height) : size[1];
|
||||
if (!(width > 0 && height > 0)) throw new RuntimeError("validation_error", "窗口尺寸必须大于 0", { nodeId: node.id, componentId: node.componentId });
|
||||
await powershell(`${WINDOW_API}; ${findWindow(title)}; [RpaWindowExtended]::SetWindowPos($p.MainWindowHandle,[IntPtr]::Zero,${x},${y},${width},${height},4) | Out-Null`);
|
||||
return { ...(await windowObject(title)), x, y, width, height };
|
||||
});
|
||||
|
||||
/** 生成从当前位置到目标的插值路径点（JS 侧算好，PS 侧只执行循环） */
|
||||
function mousePath(from: [number, number], to: [number, number], mode: string, speed: number): Array<[number, number]> {
|
||||
const dx = to[0] - from[0];
|
||||
const dy = to[1] - from[1];
|
||||
const distance = Math.hypot(dx, dy);
|
||||
if (distance < 2) return [to];
|
||||
const stepCount = Math.max(6, Math.min(50, Math.round(distance / 12)));
|
||||
const points: Array<[number, number]> = [];
|
||||
for (let i = 1; i <= stepCount; i++) {
|
||||
const t = i / stepCount;
|
||||
// 模拟人工：缓入缓出 + 轻微垂直抖动；匀速直线：线性
|
||||
const ease = mode === "0" ? (t < 0.5 ? 2 * t * t : 1 - Math.pow(-2 * t + 2, 2) / 2) : t;
|
||||
const jitter = mode === "0" ? Math.sin(t * Math.PI) * (Math.sin(i * 12.9898) * 43758.5453 % 1) * 3 : 0;
|
||||
points.push([
|
||||
Math.round(from[0] + dx * ease - (dy / distance) * jitter),
|
||||
Math.round(from[1] + dy * ease + (dx / distance) * jitter)
|
||||
]);
|
||||
}
|
||||
points.push(to);
|
||||
return points;
|
||||
}
|
||||
|
||||
/** 界面(框选/元素拾取/截图)产出的是 DIP 逻辑坐标；DPI 感知进程的 SetCursorPos 用物理像素，
|
||||
* 缩放屏(125%/150%)下直接混用会成比例偏移(用户实测 125% 下差约 25% 距离)。
|
||||
* 统一在执行前换算；宿主桥未注入时(vitest)回退原值保持旧行为 */
|
||||
function dipToPhysical(x: number, y: number): [number, number] {
|
||||
try {
|
||||
const bridge = getHostBridge();
|
||||
if (bridge) {
|
||||
const physical = bridge.dipToScreenPoint({ x, y });
|
||||
return [Math.round(physical.x), Math.round(physical.y)];
|
||||
}
|
||||
} catch {}
|
||||
return [Math.round(x), Math.round(y)];
|
||||
}
|
||||
|
||||
/** 反向换算：物理像素(如 GetWindowRect 结果) → DIP，用于「相对激活窗口」偏移叠加 */
|
||||
function physicalToDip(x: number, y: number): [number, number] {
|
||||
try {
|
||||
const bridge = getHostBridge();
|
||||
if (bridge) {
|
||||
const dip = bridge.screenToDipPoint({ x, y });
|
||||
return [Math.round(dip.x), Math.round(dip.y)];
|
||||
}
|
||||
} catch {}
|
||||
return [Math.round(x), Math.round(y)];
|
||||
}
|
||||
|
||||
const moveMouse: CapabilityHandler = async (_ctx, node, args) => {
|
||||
// 注意：position 字段是「坐标相对位置」(0=屏幕左上角/1=激活窗口左上角)，坐标本体在 location
|
||||
const [rawX, rawY] = parsePoint(args.location ?? args.point, "鼠标坐标", node.id, node.componentId);
|
||||
const relative = String(args.position ?? "0") === "1";
|
||||
const moveType = String(args.move_type ?? "0");
|
||||
const speed = Math.min(100, Math.max(1, Number(args.speed ?? 35) || 35));
|
||||
|
||||
let dipTarget: [number, number];
|
||||
if (relative) {
|
||||
// 相对激活窗口左上角：窗口矩形是物理像素，换回 DIP 再叠加偏移，最后统一转物理
|
||||
const output = await powershell(`${WINDOW_API}; $fw = [RpaWindowExtended]::GetForegroundWindow(); $r = New-Object RpaWindowExtended+RECT; [void][RpaWindowExtended]::GetWindowRect($fw, [ref]$r); "$($r.Left),$($r.Top)"`);
|
||||
const [wx, wy] = output.split(",").map(Number);
|
||||
if (!Number.isFinite(wx) || !Number.isFinite(wy)) throw new RuntimeError("browser_error", "无法获取激活窗口位置", { nodeId: node.id, componentId: node.componentId });
|
||||
const [dipWx, dipWy] = physicalToDip(wx, wy);
|
||||
dipTarget = [dipWx + rawX, dipWy + rawY];
|
||||
} else {
|
||||
dipTarget = [rawX, rawY];
|
||||
}
|
||||
return moveToSmooth(dipTarget, moveType, speed);
|
||||
};
|
||||
|
||||
async function moveToSmooth(dipTarget: [number, number], moveType: string, speed: number): Promise<{ x: number; y: number }> {
|
||||
const [tx, ty] = dipToPhysical(dipTarget[0], dipTarget[1]);
|
||||
if (moveType === "2") {
|
||||
await powershell(`${WINDOW_API}; [void][RpaWindowExtended]::SetCursorPos(${tx},${ty})`);
|
||||
return { x: Math.round(dipTarget[0]), y: Math.round(dipTarget[1]) };
|
||||
}
|
||||
// 平滑移动：从当前光标位置出发，按速度插值（路径全程物理像素，与目标同一坐标系）
|
||||
const delay = Math.min(80, Math.max(4, Math.round(1500 / speed)));
|
||||
const output = await powershell(`${WINDOW_API}; $cur = [RpaWindowExtended+POINT]::new(); [void][RpaWindowExtended]::GetCursorPos([ref]$cur); "$($cur.X),$($cur.Y)"`);
|
||||
const [cx, cy] = output.split(",").map(Number);
|
||||
const from: [number, number] = Number.isFinite(cx) && Number.isFinite(cy) ? [cx, cy] : [0, 0];
|
||||
const path = mousePath(from, [tx, ty], moveType, speed);
|
||||
const ptsLiteral = path.map(([px, py]) => `@(${px},${py})`).join(",");
|
||||
await powershell(`${WINDOW_API}; $pts = @(${ptsLiteral}); foreach ($pt in $pts) { [void][RpaWindowExtended]::SetCursorPos($pt[0], $pt[1]); Start-Sleep -Milliseconds ${delay} }`);
|
||||
return { x: Math.round(dipTarget[0]), y: Math.round(dipTarget[1]) };
|
||||
}
|
||||
register(["Mouse_MoveToV1", "Mouse_MoveToV2"], moveMouse);
|
||||
|
||||
const dragMouse: CapabilityHandler = async (_ctx, node, args) => {
|
||||
const [sx, sy] = parsePoint(args.Start ?? args.start, "起始坐标", node.id, node.componentId);
|
||||
const [ex, ey] = parsePoint(args.End ?? args.end, "结束坐标", node.id, node.componentId);
|
||||
const [spx, spy] = dipToPhysical(sx, sy);
|
||||
const [epx, epy] = dipToPhysical(ex, ey);
|
||||
const right = ["1", "right"].includes(value(args, ["side"], "0").toLowerCase());
|
||||
const down = right ? 8 : 2;
|
||||
const up = right ? 16 : 4;
|
||||
await powershell(`${WINDOW_API}; [RpaWindowExtended]::SetCursorPos(${spx},${spy}) | Out-Null; [RpaWindowExtended]::mouse_event(${down},0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 80; [RpaWindowExtended]::SetCursorPos(${epx},${epy}) | Out-Null; [RpaWindowExtended]::mouse_event(${up},0,0,0,[UIntPtr]::Zero)`);
|
||||
return { start: [Math.round(sx), Math.round(sy)], end: [Math.round(ex), Math.round(ey)] };
|
||||
};
|
||||
register(["Mouse_DragToV1", "Mouse_DragToV2", "Mouse_DragToNew"], dragMouse);
|
||||
|
||||
register(["Mouse_ClickCombine", "Mouse_ClickCombine_v1", "Mouse_ClickCombine_v2", "Mouse_ClickCombine_v3", "Mouse_ClickCombine_v4"], async (_ctx, node, args) => {
|
||||
if (value(args, ["target"], "0") !== "0") throw new RuntimeError("validation_error", "组合点击当前仅支持坐标目标", { nodeId: node.id, componentId: node.componentId });
|
||||
const [x, y] = parsePoint(args.location, "点击坐标", node.id, node.componentId);
|
||||
const [px, py] = dipToPhysical(x, y);
|
||||
const right = value(args, ["click_key"], "left").toLowerCase() === "right";
|
||||
const doubleClick = value(args, ["click_time"], "click").toLowerCase().includes("double");
|
||||
const down = right ? 8 : 2;
|
||||
const up = right ? 16 : 4;
|
||||
const repeats = doubleClick ? 2 : 1;
|
||||
await powershell(`${WINDOW_API}; [RpaWindowExtended]::SetCursorPos(${px},${py}) | Out-Null; 1..${repeats} | ForEach-Object { [RpaWindowExtended]::mouse_event(${down},0,0,0,[UIntPtr]::Zero); [RpaWindowExtended]::mouse_event(${up},0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 80 }`);
|
||||
return { x: Math.round(x), y: Math.round(y), button: right ? "right" : "left", clicks: repeats };
|
||||
});
|
||||
Reference in new issue
Block a user