import { register } from "../engine/registry"; import { RuntimeError, type CapabilityHandler } from "../engine/types"; import { getHostBridge } from "../engine/host-bridge"; import { defaultEngine } from "../engine/engine"; // 桌面自动化驱动(任务 3.3):窗口矩形/鼠标注入原语统一走 WindowsDriver const getDriver = () => defaultEngine.driver; function value(args: Record, keys: string[], fallback = ""): string { for (const key of keys) { const current = args[key]; if (current !== undefined && current !== null && current !== "") return String(current); } return fallback; } function parseObject(candidate: unknown): Record { if (candidate && typeof candidate === "object") return candidate as Record; if (typeof candidate === "string" && candidate.trim().startsWith("{")) { try { return JSON.parse(candidate); } catch { return {}; } } return {}; } function titleOf(args: Record): string { const objectCandidate = args.window_obj ?? args["窗口对象"] ?? args.uiWindowSelector ?? args.target_window; const object = parseObject(objectCandidate); return String(object.title ?? object.windowTitle ?? args.title ?? args.match_title ?? args.select_title ?? objectCandidate ?? "").trim(); } function requiredTitle(args: Record, nodeId: string, componentId: string): string { const title = titleOf(args); if (!title || title === "{}" || title === "窗口对象") throw new RuntimeError("validation_error", "窗口标题为空", { nodeId, componentId, recommendation: "填写窗口标题或传入窗口对象" }); return title; } function parsePoint(candidate: unknown, label: string, nodeId: string, componentId: string): [number, number] { let parsed: unknown = candidate; if (typeof parsed === "string") { const source = parsed; try { parsed = JSON.parse(source); } catch { parsed = source.split(/[,,xX ]+/).map(Number); } } // 屏幕框选/拾取坐标生成的是 {"x":..,"y":..} 对象,直接转换 if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { const record = parsed as Record; if (record.x !== undefined && record.y !== undefined) parsed = [record.x, record.y]; } if (!Array.isArray(parsed) || parsed.length < 2 || parsed.slice(0, 2).some((item) => !Number.isFinite(Number(item)))) { throw new RuntimeError("validation_error", `${label}无效`, { nodeId, componentId }); } return [Number(parsed[0]), Number(parsed[1])]; } /** 界面(框选/元素拾取/截图)产出的是 DIP 逻辑坐标;DPI 感知进程的 SetCursorPos 用物理像素, * 缩放屏(125%/150%)下直接混用会成比例偏移(用户实测 125% 下差约 25% 距离)。 * 统一在执行前换算;宿主桥未注入时(vitest)回退原值保持旧行为 */ function dipToPhysical(x: number, y: number): [number, number] { try { const bridge = getHostBridge(); if (bridge) { const physical = bridge.dipToScreenPoint({ x, y }); return [Math.round(physical.x), Math.round(physical.y)]; } } catch {} return [Math.round(x), Math.round(y)]; } /** 反向换算:物理像素(如 GetWindowRect 结果) → DIP,用于「相对激活窗口」偏移叠加 */ function physicalToDip(x: number, y: number): [number, number] { try { const bridge = getHostBridge(); if (bridge) { const dip = bridge.screenToDipPoint({ x, y }); return [Math.round(dip.x), Math.round(dip.y)]; } } catch {} return [Math.round(x), Math.round(y)]; } const getWindow: CapabilityHandler = async (_ctx, node, args) => getDriver().window.findByTitle(requiredTitle(args, node.id, node.componentId)); register(["Window_Get", "Window_GetWindow", "Window_GetWindow_v1"], getWindow); register("Window_ActiveWindow_v1", async (_ctx, node, args) => { const title = requiredTitle(args, node.id, node.componentId); const maximize = [true, 1, "1", "true", "True"].includes(args.isMax as never); return getDriver().window.activate(title, { maximize }); }); register(["Window_GetWindowMessage", "Window_GetWindowMessage_v1"], async (_ctx, node, args) => { const object = await getDriver().window.findByTitle(requiredTitle(args, node.id, node.componentId)); const type = value(args, ["switchType"], "all").toLowerCase(); if (type === "title") return object.title; if (type === "pid") return object.pid; if (type === "position" || type === "location") return [object.x, object.y]; if (type === "size") return [object.width, object.height]; return object; }); register(["Window_SetPos", "Window_MoveWindow", "Window_MoveWindow_v1", "Window_SizeWindow"], async (_ctx, node, args) => { const title = requiredTitle(args, node.id, node.componentId); const current = await getDriver().window.findByTitle(title); const moveOnly = node.componentId.includes("MoveWindow"); const sizeOnly = node.componentId === "Window_SizeWindow"; const position = args.location !== undefined ? parsePoint(args.location, "窗口位置", node.id, node.componentId) : [Number(args.x ?? current.x), Number(args.y ?? current.y)] as [number, number]; const sizeCandidate = args.size ?? args.window_size; const size = sizeCandidate !== undefined ? parsePoint(sizeCandidate, "窗口尺寸", node.id, node.componentId) : [Number(args.width ?? current.width), Number(args.height ?? current.height)] as [number, number]; const x = sizeOnly ? Number(current.x) : position[0]; const y = sizeOnly ? Number(current.y) : position[1]; const width = moveOnly ? Number(current.width) : size[0]; const height = moveOnly ? Number(current.height) : size[1]; if (!(width > 0 && height > 0)) throw new RuntimeError("validation_error", "窗口尺寸必须大于 0", { nodeId: node.id, componentId: node.componentId }); return getDriver().window.setPos(title, { x, y, width, height }); }); const moveMouse: CapabilityHandler = async (_ctx, node, args) => { // 注意:position 字段是「坐标相对位置」(0=屏幕左上角/1=激活窗口左上角),坐标本体在 location const [rawX, rawY] = parsePoint(args.location ?? args.point, "鼠标坐标", node.id, node.componentId); const relative = String(args.position ?? "0") === "1"; const moveType = String(args.move_type ?? "0"); const speed = Math.min(100, Math.max(1, Number(args.speed ?? 35) || 35)); let dipTarget: [number, number]; if (relative) { // 相对激活窗口左上角:窗口矩形是物理像素,换回 DIP 再叠加偏移,最后统一转物理 const { x: wx, y: wy } = await getDriver().window.foregroundPosition(); if (!Number.isFinite(wx) || !Number.isFinite(wy)) throw new RuntimeError("browser_error", "无法获取激活窗口位置", { nodeId: node.id, componentId: node.componentId }); const [dipWx, dipWy] = physicalToDip(wx, wy); dipTarget = [dipWx + rawX, dipWy + rawY]; } else { dipTarget = [rawX, rawY]; } const [tx, ty] = dipToPhysical(dipTarget[0], dipTarget[1]); if (moveType === "2") { await getDriver().mouse.moveTo({ x: tx, y: ty }); } else { // 平滑移动:从当前光标位置出发,按速度插值(路径全程物理像素,与目标同一坐标系) await getDriver().mouse.moveSmooth({ x: tx, y: ty }, { ease: moveType === "0", speed }); } return { x: Math.round(dipTarget[0]), y: Math.round(dipTarget[1]) }; }; register(["Mouse_MoveToV1", "Mouse_MoveToV2"], moveMouse); const dragMouse: CapabilityHandler = async (_ctx, node, args) => { const [sx, sy] = parsePoint(args.Start ?? args.start, "起始坐标", node.id, node.componentId); const [ex, ey] = parsePoint(args.End ?? args.end, "结束坐标", node.id, node.componentId); const [spx, spy] = dipToPhysical(sx, sy); const [epx, epy] = dipToPhysical(ex, ey); const right = ["1", "right"].includes(value(args, ["side"], "0").toLowerCase()); // 旧实现按下后停 80ms、拖到终点立即松开(与桌面版 Window_Mouse 不同,松开前不停顿) await getDriver().mouse.drag({ x: spx, y: spy }, { x: epx, y: epy }, { button: right ? "right" : "left", holdMs: 80, releaseDelayMs: 0 }); return { start: [Math.round(sx), Math.round(sy)], end: [Math.round(ex), Math.round(ey)] }; }; register(["Mouse_DragToV1", "Mouse_DragToV2", "Mouse_DragToNew"], dragMouse); register(["Mouse_ClickCombine", "Mouse_ClickCombine_v1", "Mouse_ClickCombine_v2", "Mouse_ClickCombine_v3", "Mouse_ClickCombine_v4"], async (_ctx, node, args) => { if (value(args, ["target"], "0") !== "0") throw new RuntimeError("validation_error", "组合点击当前仅支持坐标目标", { nodeId: node.id, componentId: node.componentId }); const [x, y] = parsePoint(args.location, "点击坐标", node.id, node.componentId); const [px, py] = dipToPhysical(x, y); const right = value(args, ["click_key"], "left").toLowerCase() === "right"; const doubleClick = value(args, ["click_time"], "click").toLowerCase().includes("double"); // 旧实现每次 down-up 之后停 80ms await getDriver().mouse.click({ x: px, y: py }, { button: right ? "right" : "left", double: doubleClick, settleMs: 80 }); return { x: Math.round(x), y: Math.round(y), button: right ? "right" : "left", clicks: doubleClick ? 2 : 1 }; });