From 8a816fd41d1ef5499a2d990964dd8d3e13d406ff Mon Sep 17 00:00:00 2001 From: yu <191972707@qq.com> Date: Wed, 28 Oct 2026 14:04:56 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=8E=A5=E5=85=A5=E6=A1=8C=E9=9D=A2=20?= =?UTF-8?q?UIA=20=E5=85=83=E7=B4=A0=E8=83=BD=E5=8A=9B=EF=BC=88Ground=20?= =?UTF-8?q?=E7=B3=BB=E5=88=97=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/capabilities/desktop-uia.test.ts | 40 +++ src/main/capabilities/desktop-uia.ts | 293 ++++++++++++++++++++++ src/main/capabilities/index.ts | 2 + 3 files changed, 335 insertions(+) create mode 100644 src/main/capabilities/desktop-uia.test.ts create mode 100644 src/main/capabilities/desktop-uia.ts diff --git a/src/main/capabilities/desktop-uia.test.ts b/src/main/capabilities/desktop-uia.test.ts new file mode 100644 index 0000000..bc0031b --- /dev/null +++ b/src/main/capabilities/desktop-uia.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from "vitest"; +import { getHandler } from "../engine/registry"; +import { parseUiaSelector } from "./desktop-uia"; + +describe("desktop UIA ground capabilities", () => { + it.each([ + "Element_ClickGround", + "Element_SetValueGround", + "Element_ExistsGround", + "Element_GetValueGround", + "Element_WaitElementGround" + ])("registers %s", (id) => { + expect(getHandler(id), id).toBeTruthy(); + }); + + it("parses name selector", () => { + const s = parseUiaSelector("name=添加供应商"); + expect(s.name).toBe("添加供应商"); + expect(s.index).toBe(1); + }); + + it("parses window, id, type, class and index", () => { + const s = parseUiaSelector("window=CC Switch;id=btn-save;type=button;class=btn;index=2"); + expect(s.window).toBe("CC Switch"); + expect(s.id).toBe("btn-save"); + expect(s.type).toBe("Button"); + expect(s.className).toBe("btn"); + expect(s.index).toBe(2); + }); + + it("strips uia: prefix and accepts object form", () => { + const s = parseUiaSelector({ value: "uia:name=确定" }); + expect(s.name).toBe("确定"); + }); + + it("rejects selector without any condition", () => { + expect(() => parseUiaSelector("window=只有窗口")).toThrow(); + expect(() => parseUiaSelector("")).toThrow(); + }); +}); diff --git a/src/main/capabilities/desktop-uia.ts b/src/main/capabilities/desktop-uia.ts new file mode 100644 index 0000000..8e9a154 --- /dev/null +++ b/src/main/capabilities/desktop-uia.ts @@ -0,0 +1,293 @@ +import { register } from "../engine/registry"; +import { RuntimeError, type CapabilityHandler } from "../engine/types"; +import { powershell, psString } from "./desktop"; + +interface UiaSelector { + window?: string; + name?: string; + id?: string; + type?: string; + className?: string; + index: number; +} + +export function parseUiaSelector(raw: unknown): UiaSelector { + let text = ""; + if (raw && typeof raw === "object") { + const record = raw as Record; + if (typeof record.value === "string") text = record.value; + else if (typeof record.selector === "string") text = record.selector; + else if (record.commonProps && typeof record.commonProps === "object") { + const common = record.commonProps as Record; + if (typeof common.uiSelector === "string") text = common.uiSelector; + } + } else if (raw !== undefined && raw !== null) { + text = String(raw); + } + text = text.trim(); + if (!text) { + throw new RuntimeError("validation_error", "桌面元素选择器为空", { recommendation: "使用 name=元素名称 或 id=AutomationId;window=窗口名称" }); + } + if (text.startsWith("uia:")) text = text.slice(4); + const selector: UiaSelector = { index: 1 }; + for (const part of text.split(";")) { + const eq = part.indexOf("="); + if (eq <= 0) continue; + const key = part.slice(0, eq).trim().toLowerCase(); + const value = part.slice(eq + 1).trim(); + if (!value) continue; + if (key === "window" || key === "win") selector.window = value; + else if (key === "name" || key === "n") selector.name = value; + else if (key === "id" || key === "automationid") selector.id = value; + else if (key === "type" || key === "controltype") selector.type = value.charAt(0).toUpperCase() + value.slice(1).toLowerCase(); + else if (key === "class" || key === "classname") selector.className = value; + else if (key === "index") { + const n = Number(value); + if (Number.isFinite(n) && n >= 1) selector.index = Math.floor(n); + } + } + if (!selector.name && !selector.id && !selector.className && !selector.type) { + throw new RuntimeError("validation_error", `桌面元素选择器缺少定位条件：${text}`, { recommendation: "至少提供 name= / id= / type= 之一" }); + } + return selector; +} + +const UIA_USING = "Add-Type -AssemblyName UIAutomationClient, UIAutomationTypes;"; + +const FIND_HELPER = ` +function Find-RpaWindow([string]$winName) { + $root = [System.Windows.Automation.AutomationElement]::RootElement + if (-not $winName) { return $root } + $cond = New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::NameProperty, $winName) + $win = $root.FindFirst([System.Windows.Automation.TreeScope]::Children, $cond) + if (-not $win) { throw "window_not_found:$winName" } + return $win +} +function Find-RpaElement($parent, [string]$name, [string]$id, [string]$type, [string]$className, [int]$index) { + $conds = @() + if ($name) { $conds += New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::NameProperty, $name) } + if ($id) { $conds += New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::AutomationIdProperty, $id) } + if ($className) { $conds += New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::ClassNameProperty, $className) } + if ($type) { + try { $field = [System.Windows.Automation.ControlType].GetField($type); if ($field) { $conds += New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::ControlTypeProperty, $field.GetValue($null)) } } catch {} + if ($ct) { $conds += New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::ControlTypeProperty, $ct) } + } + if ($conds.Count -eq 0) { throw "no_condition" } + $final = if ($conds.Count -eq 1) { $conds[0] } else { New-Object System.Windows.Automation.AndCondition($conds) } + $all = $parent.FindAll([System.Windows.Automation.TreeScope]::Descendants, $final) + if ($all.Count -eq 0) { return $null } + $pick = [Math]::Min([Math]::Max($index, 1), $all.Count) + return $all[$pick - 1] +} +function Add-RpaUser32 { + Add-Type -MemberDefinition '[DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y); [DllImport("user32.dll")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra); [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd);' -Name RpaUser32Click -Namespace Win32Rpa | Out-Null +} +`; + +interface UiaTarget { + selector: UiaSelector; + args: Record; + node: { id: string; componentId: string }; +} + +function target(args: Record, node: { id: string; componentId: string }): UiaTarget { + const raw = args.uiSelector ?? args.target ?? args.selector; + return { selector: parseUiaSelector(raw), args, node }; +} + +function findScript(t: UiaTarget, timeoutMs: number, gapMs: number): string { + const s = t.selector; + return `${UIA_USING} +${FIND_HELPER} +$deadline = [DateTime]::UtcNow.AddMilliseconds(${Math.max(0, Math.round(timeoutMs))}) +$found = $null +while ($null -eq $found) { + try { + $parent = Find-RpaWindow ${psString(s.window ?? "")} + $found = Find-RpaElement $parent ${psString(s.name ?? "")} ${psString(s.id ?? "")} ${psString(s.type ?? "")} ${psString(s.className ?? "")} ${s.index} + } catch { + $err = $_.Exception.Message + if ($err -like "window_not_found:*") { $found = $null } + } + if ($null -eq $found -and [DateTime]::UtcNow -ge $deadline) { break } + if ($null -eq $found) { Start-Sleep -Milliseconds ${Math.max(50, Math.round(gapMs))} } +} +if ($null -eq $found) { Write-Output "JSON_NOT_FOUND" } else { + $rect = $found.Current.BoundingRectangle + $info = @{ found = $true; name = $found.Current.Name; id = $found.Current.AutomationId; type = $found.Current.ControlType.ProgrammaticName; enabled = $found.Current.IsEnabled; x = [int]$rect.X; y = [int]$rect.Y; w = [int]$rect.Width; h = [int]$rect.Height } + Write-Output ($info | ConvertTo-Json -Compress) +}`; +} + +async function findElement(t: UiaTarget, timeoutMs = 5000, gapMs = 200): Promise | null> { + const out = await powershell(findScript(t, timeoutMs, gapMs), Math.max(30000, timeoutMs + 30000)); + const line = out.split(/\r?\n/).filter(Boolean).pop() || ""; + if (line.includes("JSON_NOT_FOUND")) return null; + try { + return JSON.parse(line); + } catch { + throw new RuntimeError("desktop_command_failed", `桌面元素查找输出异常：${out.slice(0, 200)}`, { nodeId: t.node.id, componentId: t.node.componentId }); + } +} + +function clickScript(t: UiaTarget): string { + const s = t.selector; + const clickway = String(t.args.clickway ?? "0"); + const doubleClick = clickway === "1" || /double/i.test(clickway); + return `${UIA_USING} +${FIND_HELPER} +$parent = Find-RpaWindow ${psString(s.window ?? "")} +$el = Find-RpaElement $parent ${psString(s.name ?? "")} ${psString(s.id ?? "")} ${psString(s.type ?? "")} ${psString(s.className ?? "")} ${s.index} +if ($null -eq $el) { Write-Output "JSON_NOT_FOUND"; exit 0 } +$rect = $el.Current.BoundingRectangle +$offscreen = $el.Current.IsOffscreen -or $rect.Width -le 0 -or $rect.Height -le 0 +$method = "coordinate_click" +$ok = $false +if (-not $offscreen) { + Add-RpaUser32 + $walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker + $ancestor = $el + $hwnd = [IntPtr]::Zero + while ($null -ne $ancestor) { + $h = $ancestor.Current.NativeWindowHandle + if ($h -ne 0) { $hwnd = [IntPtr]$h; break } + $ancestor = $walker.GetParent($ancestor) + } + if ($hwnd -ne [IntPtr]::Zero) { [WinRpa.RpaUser32Click]::SetForegroundWindow($hwnd) | Out-Null; Start-Sleep -Milliseconds 250 } + $x = [int]($rect.X + $rect.Width / 2) + $y = [int]($rect.Y + $rect.Height / 2) + [WinRpa.RpaUser32Click]::SetCursorPos($x, $y) | Out-Null + Start-Sleep -Milliseconds 120 + [WinRpa.RpaUser32Click]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero) + Start-Sleep -Milliseconds 40 + [WinRpa.RpaUser32Click]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero) + if (${doubleClick}) { + Start-Sleep -Milliseconds 60 + [WinRpa.RpaUser32Click]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero) + Start-Sleep -Milliseconds 40 + [WinRpa.RpaUser32Click]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero) + } + $ok = $true +} else { + $method = "invoke_pattern" + foreach ($entry in @(@("Invoke", [System.Windows.Automation.InvokePattern]::Pattern), @("Toggle", [System.Windows.Automation.TogglePattern]::Pattern), @("Expand", [System.Windows.Automation.ExpandCollapsePattern]::Pattern), @("Select", [System.Windows.Automation.SelectionItemPattern]::Pattern))) { + try { + $p = $el.GetCurrentPattern($entry[1]) + switch ($entry[0]) { + "Invoke" { $p.Invoke(); $ok = $true } + "Toggle" { $p.Toggle(); $ok = $true } + "Expand" { $p.Expand(); $ok = $true } + "Select" { $p.Select(); $ok = $true } + } + if ($ok) { break } + } catch {} + } +} +$rect2 = $el.Current.BoundingRectangle +Write-Output (@{ clicked = $ok; method = $method; name = $el.Current.Name; x = [int]$rect2.X; y = [int]$rect2.Y; w = [int]$rect2.Width; h = [int]$rect2.Height } | ConvertTo-Json -Compress)`; +} +function setValueScript(t: UiaTarget, value: string): string { + const s = t.selector; + return `${UIA_USING} +${FIND_HELPER} +$parent = Find-RpaWindow ${psString(s.window ?? "")} +$el = Find-RpaElement $parent ${psString(s.name ?? "")} ${psString(s.id ?? "")} ${psString(s.type ?? "")} ${psString(s.className ?? "")} ${s.index} +if ($null -eq $el) { Write-Output "JSON_NOT_FOUND"; exit 0 } +$value = ${psString(value)} +$method = "value_pattern" +$ok = $false +try { + $vp = $el.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern) + $vp.SetValue($value) + $ok = $true +} catch { + $method = "clipboard_paste" + Add-Type -AssemblyName System.Windows.Forms + $saved = $null + try { $saved = [System.Windows.Forms.Clipboard]::GetText() } catch {} + try { $el.SetFocus() } catch {} + Start-Sleep -Milliseconds 150 + try { [System.Windows.Forms.Clipboard]::SetText($value) } catch {} + Start-Sleep -Milliseconds 80 + [System.Windows.Forms.SendKeys]::SendWait("^a") + Start-Sleep -Milliseconds 60 + [System.Windows.Forms.SendKeys]::SendWait("^v") + Start-Sleep -Milliseconds 300 + $ok = $true + try { + if ($saved) { [System.Windows.Forms.Clipboard]::SetText($saved) } + elseif ($saved -ne $null) { [System.Windows.Forms.Clipboard]::Clear() } + } catch {} +} +Write-Output (@{ set = $ok; method = $method; name = $el.Current.Name } | ConvertTo-Json -Compress)`; +} +function getValueScript(t: UiaTarget): string { + const s = t.selector; + return `${UIA_USING} +${FIND_HELPER} +$parent = Find-RpaWindow ${psString(s.window ?? "")} +$el = Find-RpaElement $parent ${psString(s.name ?? "")} ${psString(s.id ?? "")} ${psString(s.type ?? "")} ${psString(s.className ?? "")} ${s.index} +if ($null -eq $el) { Write-Output "JSON_NOT_FOUND"; exit 0 } +$value = $el.Current.Name +try { $vp = $el.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern); $value = $vp.Value } catch {} +Write-Output (@{ value = $value; name = $el.Current.Name } | ConvertTo-Json -Compress)`; +} + +async function runScript(script: string, t: UiaTarget, timeoutMs = 60000): Promise | null> { + const out = await powershell(script, timeoutMs); + const line = out.split(/\r?\n/).filter(Boolean).pop() || ""; + if (line.includes("JSON_NOT_FOUND")) return null; + try { + return JSON.parse(line); + } catch { + throw new RuntimeError("desktop_command_failed", `桌面元素操作输出异常：${out.slice(0, 200)}`, { nodeId: t.node.id, componentId: t.node.componentId }); + } +} + +function requireFound(result: Record | null, t: UiaTarget, action: string): Record { + if (!result) { + const s = t.selector; + throw new RuntimeError("desktop_element_not_found", `桌面元素未找到（${action}）：${JSON.stringify({ window: s.window, name: s.name, id: s.id, type: s.type, className: s.className, index: s.index })}`, { nodeId: t.node.id, componentId: t.node.componentId }); + } + return result; +} + +const clickGround: CapabilityHandler = async (_ctx, node, args) => { + const t = target(args, node); + const result = await runScript(clickScript(t), t); + return requireFound(result, t, "click"); +}; + +const setValueGround: CapabilityHandler = async (_ctx, node, args) => { + const t = target(args, node); + const value = String(args.value ?? args.text ?? ""); + const result = await runScript(setValueScript(t, value), t); + return requireFound(result, t, "setValue"); +}; + +const existsGround: CapabilityHandler = async (_ctx, node, args) => { + const t = target(args, node); + const found = await findElement(t, 1000, 150); + return found !== null; +}; + +const getValueGround: CapabilityHandler = async (_ctx, node, args) => { + const t = target(args, node); + const result = await runScript(getValueScript(t), t); + return requireFound(result, t, "getValue").value ?? ""; +}; + +const waitElementGround: CapabilityHandler = async (_ctx, node, args) => { + const t = target(args, node); + const timeout = Number(args.timeout ?? args.timeoutMs ?? 5000) || 5000; + const gap = Number(args.time_gap ?? args.gapMs ?? 200) || 200; + const found = await findElement(t, timeout, gap); + return found !== null; +}; + +register("Element_ClickGround", clickGround); +register("Element_SetValueGround", setValueGround); +register("Element_ExistsGround", existsGround); +register("Element_GetValueGround", getValueGround); +register("Element_WaitElementGround", waitElementGround); + diff --git a/src/main/capabilities/index.ts b/src/main/capabilities/index.ts index 89e620a..014cca8 100644 --- a/src/main/capabilities/index.ts +++ b/src/main/capabilities/index.ts @@ -9,6 +9,7 @@ import "./browser-common"; import "./browser-control"; import "./office"; import "./desktop"; +import "./desktop-uia"; import "./pdf"; import "./word"; import "./word-extended"; @@ -26,3 +27,4 @@ import "./data-table-common"; import "./common-final"; export { capabilityCount, capabilityIds, getHandler, hasCapability, register } from "../engine/registry"; +