import { register } from "../engine/registry"; import { RuntimeError, type CapabilityHandler } from "../engine/types"; import { powershell, psString } from "./desktop"; interface UiaSelector { window?: string; name?: string; id?: string; type?: string; className?: string; index: number; } export function parseUiaSelector(raw: unknown): UiaSelector { let text = ""; if (raw && typeof raw === "object") { const record = raw as Record; if (typeof record.value === "string") text = record.value; else if (typeof record.selector === "string") text = record.selector; else if (record.commonProps && typeof record.commonProps === "object") { const common = record.commonProps as Record; if (typeof common.uiSelector === "string") text = common.uiSelector; } } else if (raw !== undefined && raw !== null) { text = String(raw); } text = text.trim(); if (!text) { throw new RuntimeError("validation_error", "桌面元素选择器为空", { recommendation: "使用 name=元素名称 或 id=AutomationId;window=窗口名称" }); } if (text.startsWith("uia:")) text = text.slice(4); const selector: UiaSelector = { index: 1 }; for (const part of text.split(";")) { const eq = part.indexOf("="); if (eq <= 0) continue; const key = part.slice(0, eq).trim().toLowerCase(); const value = part.slice(eq + 1).trim(); if (!value) continue; if (key === "window" || key === "win") selector.window = value; else if (key === "name" || key === "n") selector.name = value; else if (key === "id" || key === "automationid") selector.id = value; else if (key === "type" || key === "controltype") selector.type = value.charAt(0).toUpperCase() + value.slice(1).toLowerCase(); else if (key === "class" || key === "classname") selector.className = value; else if (key === "index") { const n = Number(value); if (Number.isFinite(n) && n >= 1) selector.index = Math.floor(n); } } if (!selector.name && !selector.id && !selector.className && !selector.type) { throw new RuntimeError("validation_error", `桌面元素选择器缺少定位条件:${text}`, { recommendation: "至少提供 name= / id= / type= 之一" }); } return selector; } const UIA_USING = "Add-Type -AssemblyName UIAutomationClient, UIAutomationTypes;"; const FIND_HELPER = ` function Find-RpaWindow([string]$winName) { $root = [System.Windows.Automation.AutomationElement]::RootElement if (-not $winName) { return $root } $cond = New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::NameProperty, $winName) $win = $root.FindFirst([System.Windows.Automation.TreeScope]::Children, $cond) if (-not $win) { throw "window_not_found:$winName" } return $win } function Find-RpaElement($parent, [string]$name, [string]$id, [string]$type, [string]$className, [int]$index) { $conds = @() if ($name) { $conds += New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::NameProperty, $name) } if ($id) { $conds += New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::AutomationIdProperty, $id) } if ($className) { $conds += New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::ClassNameProperty, $className) } if ($type) { try { $field = [System.Windows.Automation.ControlType].GetField($type); if ($field) { $conds += New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::ControlTypeProperty, $field.GetValue($null)) } } catch {} if ($ct) { $conds += New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::ControlTypeProperty, $ct) } } if ($conds.Count -eq 0) { throw "no_condition" } $final = if ($conds.Count -eq 1) { $conds[0] } else { New-Object System.Windows.Automation.AndCondition($conds) } $all = $parent.FindAll([System.Windows.Automation.TreeScope]::Descendants, $final) if ($all.Count -eq 0) { return $null } $pick = [Math]::Min([Math]::Max($index, 1), $all.Count) return $all[$pick - 1] } function Add-RpaUser32 { Add-Type -MemberDefinition '[DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y); [DllImport("user32.dll")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra); [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd);' -Name RpaUser32Click -Namespace Win32Rpa | Out-Null } `; interface UiaTarget { selector: UiaSelector; args: Record; node: { id: string; componentId: string }; } function target(args: Record, node: { id: string; componentId: string }): UiaTarget { const raw = args.uiSelector ?? args.target ?? args.selector; return { selector: parseUiaSelector(raw), args, node }; } function findScript(t: UiaTarget, timeoutMs: number, gapMs: number): string { const s = t.selector; return `${UIA_USING} ${FIND_HELPER} $deadline = [DateTime]::UtcNow.AddMilliseconds(${Math.max(0, Math.round(timeoutMs))}) $found = $null while ($null -eq $found) { try { $parent = Find-RpaWindow ${psString(s.window ?? "")} $found = Find-RpaElement $parent ${psString(s.name ?? "")} ${psString(s.id ?? "")} ${psString(s.type ?? "")} ${psString(s.className ?? "")} ${s.index} } catch { $err = $_.Exception.Message if ($err -like "window_not_found:*") { $found = $null } } if ($null -eq $found -and [DateTime]::UtcNow -ge $deadline) { break } if ($null -eq $found) { Start-Sleep -Milliseconds ${Math.max(50, Math.round(gapMs))} } } if ($null -eq $found) { Write-Output "JSON_NOT_FOUND" } else { $rect = $found.Current.BoundingRectangle $info = @{ found = $true; name = $found.Current.Name; id = $found.Current.AutomationId; type = $found.Current.ControlType.ProgrammaticName; enabled = $found.Current.IsEnabled; x = [int]$rect.X; y = [int]$rect.Y; w = [int]$rect.Width; h = [int]$rect.Height } Write-Output ($info | ConvertTo-Json -Compress) }`; } async function findElement(t: UiaTarget, timeoutMs = 5000, gapMs = 200): Promise | null> { const out = await powershell(findScript(t, timeoutMs, gapMs), Math.max(30000, timeoutMs + 30000)); const line = out.split(/\r?\n/).filter(Boolean).pop() || ""; if (line.includes("JSON_NOT_FOUND")) return null; try { return JSON.parse(line); } catch { throw new RuntimeError("desktop_command_failed", `桌面元素查找输出异常:${out.slice(0, 200)}`, { nodeId: t.node.id, componentId: t.node.componentId }); } } function clickScript(t: UiaTarget): string { const s = t.selector; const clickway = String(t.args.clickway ?? "0"); const doubleClick = clickway === "1" || /double/i.test(clickway); return `${UIA_USING} ${FIND_HELPER} $parent = Find-RpaWindow ${psString(s.window ?? "")} $el = Find-RpaElement $parent ${psString(s.name ?? "")} ${psString(s.id ?? "")} ${psString(s.type ?? "")} ${psString(s.className ?? "")} ${s.index} if ($null -eq $el) { Write-Output "JSON_NOT_FOUND"; exit 0 } $rect = $el.Current.BoundingRectangle $offscreen = $el.Current.IsOffscreen -or $rect.Width -le 0 -or $rect.Height -le 0 $method = "coordinate_click" $ok = $false if (-not $offscreen) { Add-RpaUser32 $walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker $ancestor = $el $hwnd = [IntPtr]::Zero while ($null -ne $ancestor) { $h = $ancestor.Current.NativeWindowHandle if ($h -ne 0) { $hwnd = [IntPtr]$h; break } $ancestor = $walker.GetParent($ancestor) } if ($hwnd -ne [IntPtr]::Zero) { [WinRpa.RpaUser32Click]::SetForegroundWindow($hwnd) | Out-Null; Start-Sleep -Milliseconds 250 } $x = [int]($rect.X + $rect.Width / 2) $y = [int]($rect.Y + $rect.Height / 2) [WinRpa.RpaUser32Click]::SetCursorPos($x, $y) | Out-Null Start-Sleep -Milliseconds 120 [WinRpa.RpaUser32Click]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero) Start-Sleep -Milliseconds 40 [WinRpa.RpaUser32Click]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero) if (${doubleClick}) { Start-Sleep -Milliseconds 60 [WinRpa.RpaUser32Click]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero) Start-Sleep -Milliseconds 40 [WinRpa.RpaUser32Click]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero) } $ok = $true } else { $method = "invoke_pattern" foreach ($entry in @(@("Invoke", [System.Windows.Automation.InvokePattern]::Pattern), @("Toggle", [System.Windows.Automation.TogglePattern]::Pattern), @("Expand", [System.Windows.Automation.ExpandCollapsePattern]::Pattern), @("Select", [System.Windows.Automation.SelectionItemPattern]::Pattern))) { try { $p = $el.GetCurrentPattern($entry[1]) switch ($entry[0]) { "Invoke" { $p.Invoke(); $ok = $true } "Toggle" { $p.Toggle(); $ok = $true } "Expand" { $p.Expand(); $ok = $true } "Select" { $p.Select(); $ok = $true } } if ($ok) { break } } catch {} } } $rect2 = $el.Current.BoundingRectangle Write-Output (@{ clicked = $ok; method = $method; name = $el.Current.Name; x = [int]$rect2.X; y = [int]$rect2.Y; w = [int]$rect2.Width; h = [int]$rect2.Height } | ConvertTo-Json -Compress)`; } function setValueScript(t: UiaTarget, value: string): string { const s = t.selector; return `${UIA_USING} ${FIND_HELPER} $parent = Find-RpaWindow ${psString(s.window ?? "")} $el = Find-RpaElement $parent ${psString(s.name ?? "")} ${psString(s.id ?? "")} ${psString(s.type ?? "")} ${psString(s.className ?? "")} ${s.index} if ($null -eq $el) { Write-Output "JSON_NOT_FOUND"; exit 0 } $value = ${psString(value)} $method = "value_pattern" $ok = $false try { $vp = $el.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern) $vp.SetValue($value) $ok = $true } catch { $method = "clipboard_paste" Add-Type -AssemblyName System.Windows.Forms $saved = $null try { $saved = [System.Windows.Forms.Clipboard]::GetText() } catch {} try { $el.SetFocus() } catch {} Start-Sleep -Milliseconds 150 try { [System.Windows.Forms.Clipboard]::SetText($value) } catch {} Start-Sleep -Milliseconds 80 [System.Windows.Forms.SendKeys]::SendWait("^a") Start-Sleep -Milliseconds 60 [System.Windows.Forms.SendKeys]::SendWait("^v") Start-Sleep -Milliseconds 300 $ok = $true try { if ($saved) { [System.Windows.Forms.Clipboard]::SetText($saved) } elseif ($saved -ne $null) { [System.Windows.Forms.Clipboard]::Clear() } } catch {} } Write-Output (@{ set = $ok; method = $method; name = $el.Current.Name } | ConvertTo-Json -Compress)`; } function getValueScript(t: UiaTarget): string { const s = t.selector; return `${UIA_USING} ${FIND_HELPER} $parent = Find-RpaWindow ${psString(s.window ?? "")} $el = Find-RpaElement $parent ${psString(s.name ?? "")} ${psString(s.id ?? "")} ${psString(s.type ?? "")} ${psString(s.className ?? "")} ${s.index} if ($null -eq $el) { Write-Output "JSON_NOT_FOUND"; exit 0 } $value = $el.Current.Name try { $vp = $el.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern); $value = $vp.Value } catch {} Write-Output (@{ value = $value; name = $el.Current.Name } | ConvertTo-Json -Compress)`; } async function runScript(script: string, t: UiaTarget, timeoutMs = 60000): Promise | null> { const out = await powershell(script, timeoutMs); const line = out.split(/\r?\n/).filter(Boolean).pop() || ""; if (line.includes("JSON_NOT_FOUND")) return null; try { return JSON.parse(line); } catch { throw new RuntimeError("desktop_command_failed", `桌面元素操作输出异常:${out.slice(0, 200)}`, { nodeId: t.node.id, componentId: t.node.componentId }); } } function requireFound(result: Record | null, t: UiaTarget, action: string): Record { if (!result) { const s = t.selector; throw new RuntimeError("desktop_element_not_found", `桌面元素未找到(${action}):${JSON.stringify({ window: s.window, name: s.name, id: s.id, type: s.type, className: s.className, index: s.index })}`, { nodeId: t.node.id, componentId: t.node.componentId }); } return result; } const clickGround: CapabilityHandler = async (_ctx, node, args) => { const t = target(args, node); const result = await runScript(clickScript(t), t); return requireFound(result, t, "click"); }; const setValueGround: CapabilityHandler = async (_ctx, node, args) => { const t = target(args, node); const value = String(args.value ?? args.text ?? ""); const result = await runScript(setValueScript(t, value), t); return requireFound(result, t, "setValue"); }; const existsGround: CapabilityHandler = async (_ctx, node, args) => { const t = target(args, node); const found = await findElement(t, 1000, 150); return found !== null; }; const getValueGround: CapabilityHandler = async (_ctx, node, args) => { const t = target(args, node); const result = await runScript(getValueScript(t), t); return requireFound(result, t, "getValue").value ?? ""; }; const waitElementGround: CapabilityHandler = async (_ctx, node, args) => { const t = target(args, node); const timeout = Number(args.timeout ?? args.timeoutMs ?? 5000) || 5000; const gap = Number(args.time_gap ?? args.gapMs ?? 200) || 200; const found = await findElement(t, timeout, gap); return found !== null; }; register("Element_ClickGround", clickGround); register("Element_SetValueGround", setValueGround); register("Element_ExistsGround", existsGround); register("Element_GetValueGround", getValueGround); register("Element_WaitElementGround", waitElementGround);