diff --git a/src/main/capabilities/desktop-uia.test.ts b/src/main/capabilities/desktop-uia.test.ts index bc0031b..8812f28 100644 --- a/src/main/capabilities/desktop-uia.test.ts +++ b/src/main/capabilities/desktop-uia.test.ts @@ -8,7 +8,20 @@ describe("desktop UIA ground capabilities", () => { "Element_SetValueGround", "Element_ExistsGround", "Element_GetValueGround", - "Element_WaitElementGround" + "Element_WaitElementGround", + "Element_GetCheckGround", + "Element_SetCheckGround", + "Element_RegionGround", + "Element_GetSelectGround", + "Element_SetSelectGround", + "Element_GetAttributeGround", + "Element_SetAttributeGround", + "Element_GetButtonEnableGround", + "Element_ShotGround", + "Element_MouseToGround", + "Element_ScrollIntoViewGround", + "Element_ChildrenGround", + "Element_DesktopChildren" ])("registers %s", (id) => { expect(getHandler(id), id).toBeTruthy(); }); diff --git a/src/main/capabilities/desktop-uia.ts b/src/main/capabilities/desktop-uia.ts index 7626b41..806b67e 100644 --- a/src/main/capabilities/desktop-uia.ts +++ b/src/main/capabilities/desktop-uia.ts @@ -1,4 +1,6 @@ import { register } from "../engine/registry"; +import fs from "node:fs"; +import path from "node:path"; import { RuntimeError, type CapabilityHandler } from "../engine/types"; import { powershell, psString } from "./desktop"; @@ -393,3 +395,242 @@ register("Element_WaitElementGround", waitElementGround); + +function outputPath(args: Record, extension = "png"): string { + const raw = args.filePath ?? args.folder ?? args.path; + const folder = path.resolve(String(raw && String(raw).trim() ? raw : process.cwd())); + const rawName = String(args.filename ?? args.fileName ?? args.name ?? `element-${Date.now()}.${extension}`) + .replace(/\$\{randomString\}/g, Math.random().toString(36).slice(2, 10)); + const name = /\.[A-Za-z0-9]{2,8}$/.test(rawName) ? rawName : `${rawName}.${extension}`; + return path.join(folder, name); +} + +function propertyValue(scriptLines: string[], t: UiaTarget): string { + const s = t.selector; + return `${UIA_USING} +${FIND_HELPER} +$el = $null +try { + $parent = Find-RpaWindow ${psString(s.window ?? "")} + $el = Find-RpaElement $parent ${psString(s.name ?? "")} ${psString(s.id ?? "")} ${psString(s.type ?? "")} ${psString(s.className ?? "")} ${s.index} +} catch { $el = $null } +if ($null -eq $el) { Write-Output "JSON_NOT_FOUND"; exit 0 } +${scriptLines.join("\n")}`; +} + +const getCheckGround: CapabilityHandler = async (_ctx, node, args) => { + const t = target(args, node); + const result = await runWithImplicitWait(t, "getCheck", propertyValue([ + "$checked = $false", + "try { $checked = $el.GetCurrentPattern([System.Windows.Automation.TogglePattern]::Pattern).Current.ToggleState -eq [System.Windows.Automation.ToggleState]::On } catch {}", + "Write-Output (@{ value = [bool]$checked; name = $el.Current.Name; enabled = [bool]$el.Current.IsEnabled } | ConvertTo-Json -Compress)" + ], t), runScript, args); + return result.value === true; +}; + +const setCheckGround: CapabilityHandler = async (_ctx, node, args) => { + const t = target(args, node); + const desired = ["1", "true", "yes", "on"].includes(String(args.isChecked ?? "1").toLowerCase()); + return runWithImplicitWait(t, "setCheck", propertyValue([ + `$desired = ${desired ? "$true" : "$false"}`, + "$current = $false; $hasToggle = $false", + "try { $pattern = $el.GetCurrentPattern([System.Windows.Automation.TogglePattern]::Pattern); $current = $pattern.Current.ToggleState -eq [System.Windows.Automation.ToggleState]::On; $hasToggle = $true } catch {}", + "$method = if ($hasToggle) { 'toggle_pattern' } else { 'coordinate_click' }", + "if (-not $hasToggle -or $current -ne $desired) {", + " if ($hasToggle) { $pattern.Toggle() | Out-Null }", + " else {", + " $rect = $el.Current.BoundingRectangle; Add-RpaUser32", + " [Win32Rpa.RpaUser32Click]::SetCursorPos([int]($rect.X + $rect.Width / 2), [int]($rect.Y + $rect.Height / 2)) | Out-Null", + " Start-Sleep -Milliseconds 100; [Win32Rpa.RpaUser32Click]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)", + " Start-Sleep -Milliseconds 50; [Win32Rpa.RpaUser32Click]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)", + " }", + "}", + "$after = $false; try { $after = $el.GetCurrentPattern([System.Windows.Automation.TogglePattern]::Pattern).Current.ToggleState -eq [System.Windows.Automation.ToggleState]::On } catch {}", + "Write-Output (@{ set = ($after -eq $desired); checked = [bool]$after; method = $method; name = $el.Current.Name } | ConvertTo-Json -Compress)" + ], t), runScript, args); +}; + +const regionGround: CapabilityHandler = async (_ctx, node, args) => { + const t = target(args, node); + const relative = String(args.position ?? "3") === "2"; + const result = await runWithImplicitWait(t, "getRegion", propertyValue([ + "$rect = $el.Current.BoundingRectangle; $ox = 0; $oy = 0", + relative ? [ + "try {", + " Add-RpaUser32", + ' Add-Type -MemberDefinition \'[DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr hWnd, out RECT rect); public struct RECT { public int Left; public int Top; public int Right; public int Bottom; }\' -Name RpaWindowRect -Namespace Win32RpaRect | Out-Null', + " $walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker; $ancestor = $el", + " while ($null -ne $ancestor) {", + " $hwnd = $ancestor.Current.NativeWindowHandle", + " if ($hwnd -ne 0) { $wr = New-Object Win32RpaRect.RpaWindowRect+RECT; [void][Win32RpaRect.RpaWindowRect]::GetWindowRect([IntPtr]$hwnd, [ref]$wr); $ox = $wr.Left; $oy = $wr.Top; break }", + " $ancestor = $walker.GetParent($ancestor)", + " }", + "} catch {}", + ].join("\n") : "", + "Write-Output (@{ x = [int]($rect.X - $ox); y = [int]($rect.Y - $oy); width = [int]$rect.Width; height = [int]$rect.Height } | ConvertTo-Json -Compress)" + ], t), runScript, args); + return result; +}; + +function getSelectScript(t: UiaTarget): string { + const byText = String(t.args.choose ?? "1") !== "2"; + return propertyValue([ + "$result = @(); $method = 'selection_pattern'", + "try { $sel = $el.GetCurrentPattern([System.Windows.Automation.SelectionPattern]::Pattern); foreach ($item in $sel.Current.GetSelection()) { $value = $item.Current.Name; if (-not " + (byText ? "$true" : "$false") + ") { $value = $item.Current.AutomationId }; $result += @{ text = $item.Current.Name; value = $item.Current.AutomationId; selected = $true } } } catch {}", + "if ($result.Count -eq 0) { try { $value = $el.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern).Value } catch { $value = $el.Current.Name }; $result += @{ text = [string]$value; value = [string]$value; selected = $true }; $method = 'value_pattern' }", + "Write-Output (@{ result = $result; method = $method; name = $el.Current.Name } | ConvertTo-Json -Compress -Depth 3)" + ], t); +} + +const getSelectGround: CapabilityHandler = async (_ctx, node, args) => { + const t = target(args, node); + const result = await runWithImplicitWait(t, "getSelect", getSelectScript(t), runScript, args); + const items = Array.isArray(result.result) ? result.result : []; + return items.length === 1 ? items[0] : items; +}; + +function setSelectScript(t: UiaTarget, values: string[]): string { + const encoded = Buffer.from(JSON.stringify(values), "utf16le").toString("base64"); + const choose = String(t.args.choose ?? "1"); + return propertyValue([ + `$desired = [Text.Encoding]::Unicode.GetString([Convert]::FromBase64String('${encoded}')) | ConvertFrom-Json`, + "$items = @($el.FindAll([System.Windows.Automation.TreeScope]::Descendants, [System.Windows.Automation.Condition]::TrueCondition) | Where-Object { $_.Current.ControlType.ProgrammaticName -match 'ListItem|ComboBoxItem' })", + "if ($items.Count -eq 0) { try { $el.Expand() } catch {}; Start-Sleep -Milliseconds 250; $items = @($el.FindAll([System.Windows.Automation.TreeScope]::Descendants, [System.Windows.Automation.Condition]::TrueCondition) | Where-Object { $_.Current.ControlType.ProgrammaticName -match 'ListItem|ComboBoxItem' }) }", + "$targets = @(); $method = 'selection_item_pattern'", + "foreach ($wanted in @($desired)) {", + ` if ('${choose}' -eq '0') { $ix = [int]$wanted - 1; if ($ix -ge 0 -and $ix -lt $items.Count) { $targets += $items[$ix] } }`, + ` elseif ('${choose}' -eq '2') { $targets += @($items | Where-Object { $_.Current.AutomationId -eq [string]$wanted }) }`, + " else { $targets += @($items | Where-Object { $_.Current.Name -eq [string]$wanted }) }", + "}", + "$ok = $targets.Count -gt 0", + "foreach ($item in $targets) { try { $item.GetCurrentPattern([System.Windows.Automation.SelectionItemPattern]::Pattern).Select() } catch { try { $item.Invoke() } catch { $ok = $false } } }", + "try { $el.Collapse() } catch {}", + "Write-Output (@{ set = [bool]$ok; matched = $targets.Count; available = $items.Count; method = $method; name = $el.Current.Name } | ConvertTo-Json -Compress)" + ], t); +} + +const setSelectGround: CapabilityHandler = async (_ctx, node, args) => { + const t = target(args, node); + const raw = args.itemsToSelect ?? args.value ?? args.text; + let values: string[] = []; + if (Array.isArray(raw)) values = raw.map((value) => String(value)); + else if (raw !== undefined && raw !== null && String(raw).trim() !== "") { + try { const parsed = JSON.parse(String(raw)); values = Array.isArray(parsed) ? parsed.map((value) => String(value)) : [String(parsed)]; } + catch { values = [String(raw)]; } + } + if (!values.length) throw new RuntimeError("validation_error", "下拉框选择项为空", { nodeId: node.id, componentId: node.componentId }); + return runWithImplicitWait(t, "setSelect", setSelectScript(t, values), runScript, args); +}; + +function attrScript(t: UiaTarget, attrName: string, attrValue = ""): string { + const name = attrName.toLowerCase(); + if (name === "value" || name === "text") { + return propertyValue([ + "try { $value = $el.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern).Value } catch { $value = $el.Current.Name }", + "Write-Output (@{ value = [string]$value; name = $el.Current.Name } | ConvertTo-Json -Compress)" + ], t); + } + if (!attrValue) { + return propertyValue([ + "$p = @{ name = $el.Current.Name; id = $el.Current.AutomationId; class = $el.Current.ClassName; type = $el.Current.ControlType.ProgrammaticName; enabled = $el.Current.IsEnabled; help = $el.Current.HelpText; value = $el.Current.Name }", + `$key = ${psString(name || "name")}`, + "$result = if ($p.Contains($key)) { $p[$key] } else { $p.name }", + "Write-Output (@{ value = [string]$result; properties = $p; name = $el.Current.Name } | ConvertTo-Json -Compress)" + ], t); + } + if (name !== "value" && name !== "text") throw new RuntimeError("validation_error", `桌面 UIA 不支持设置属性:${attrName};可设置 value/text`, { nodeId: t.node.id, componentId: t.node.componentId }); + return propertyValue([ + "$ok = $false; $method = 'value_pattern'", + `try { $el.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern).SetValue(${psString(attrValue)}); $ok = $true } catch {}`, + "if (-not $ok) { $method = 'keyboard'; Add-Type -AssemblyName System.Windows.Forms; $el.SetFocus() | Out-Null; Start-Sleep -Milliseconds 100; [System.Windows.Forms.SendKeys]::SendWait('^a'); [System.Windows.Forms.SendKeys]::SendWait('^v'); Start-Sleep -Milliseconds 100; $ok = $true }", + "Write-Output (@{ set = $ok; method = $method; name = $el.Current.Name } | ConvertTo-Json -Compress)" + ], t); +} + +const getAttributeGround: CapabilityHandler = async (_ctx, node, args) => { + const t = target(args, node); + const attrName = String(args.attrName ?? args.attribute ?? "name"); + const result = await runWithImplicitWait(t, "getAttribute", attrScript(t, attrName), runScript, args); + return result.value ?? ""; +}; + +const setAttributeGround: CapabilityHandler = async (_ctx, node, args) => { + const t = target(args, node); + const attrName = String(args.attrName ?? args.attribute ?? "value"); + const attrValue = String(args.attrValue ?? args.value ?? ""); + return runWithImplicitWait(t, "setAttribute", attrScript(t, attrName, attrValue), runScript, args); +}; + +const getButtonEnableGround: CapabilityHandler = async (_ctx, node, args) => { + const t = target(args, node); + const result = await runWithImplicitWait(t, "getButtonEnable", propertyValue(["Write-Output (@{ value = [bool]$el.Current.IsEnabled; name = $el.Current.Name } | ConvertTo-Json -Compress)"], t), runScript, args); + return result.value === true; +}; + +const shotGround: CapabilityHandler = async (_ctx, node, args) => { + const t = target(args, node); + const output = outputPath(args); + fs.mkdirSync(path.dirname(output), { recursive: true }); + const result = await runWithImplicitWait(t, "shot", propertyValue([ + "$rect = $el.Current.BoundingRectangle; if ($rect.Width -le 0 -or $rect.Height -le 0) { throw 'element_not_visible' }", + "Add-Type -AssemblyName System.Drawing | Out-Null", + "$bitmap = New-Object System.Drawing.Bitmap([int]$rect.Width, [int]$rect.Height)", + "$graphics = [System.Drawing.Graphics]::FromImage($bitmap)", + "$graphics.CopyFromScreen([int]$rect.X, [int]$rect.Y, 0, 0, $bitmap.Size); $graphics.Dispose()", + `$bitmap.Save(${psString(output)}, [System.Drawing.Imaging.ImageFormat]::Png); $bitmap.Dispose()`, + `Write-Output (@{ value = ${psString(output)}; x = [int]$rect.X; y = [int]$rect.Y; width = [int]$rect.Width; height = [int]$rect.Height } | ConvertTo-Json -Compress)` + ], t), runScript, args); + return result.value ?? output; +}; + +const mouseToGround: CapabilityHandler = async (_ctx, node, args) => { + const t = target(args, node); + return runWithImplicitWait(t, "mouseTo", propertyValue([ + "$rect = $el.Current.BoundingRectangle; Add-RpaUser32", + "$x = [int]($rect.X + $rect.Width / 2); $y = [int]($rect.Y + $rect.Height / 2)", + "[Win32Rpa.RpaUser32Click]::SetCursorPos($x, $y) | Out-Null", + "Write-Output (@{ x = $x; y = $y; name = $el.Current.Name } | ConvertTo-Json -Compress)" + ], t), runScript, args); +}; + +const scrollIntoViewGround: CapabilityHandler = async (_ctx, node, args) => { + const t = target(args, node); + return runWithImplicitWait(t, "scrollIntoView", propertyValue([ + "$ok = $false; $method = 'scroll_item_pattern'", + "try { $el.GetCurrentPattern([System.Windows.Automation.ScrollItemPattern]::Pattern).ScrollIntoView(); $ok = $true } catch {}", + "if (-not $ok) { $method = 'focus'; try { $el.SetFocus() | Out-Null; $ok = $true } catch {} }", + "Write-Output (@{ scrolled = $ok; method = $method; name = $el.Current.Name } | ConvertTo-Json -Compress)" + ], t), runScript, args); +}; + +function childrenGroundHandler(single: boolean): CapabilityHandler { + return async (_ctx, node, args) => { + const t = target(args, node); + const num = Math.max(0, Math.floor(Number(args.num ?? 0) || 0)); + const result = await runWithImplicitWait(t, "getChildren", propertyValue([ + "$walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker; $child = $walker.GetFirstChild($el); $children = @()", + "while ($null -ne $child) {", + " $r = $child.Current.BoundingRectangle", + " $children += @{ name = $child.Current.Name; id = $child.Current.AutomationId; type = $child.Current.ControlType.ProgrammaticName; className = $child.Current.ClassName; enabled = [bool]$child.Current.IsEnabled; x = [int]$r.X; y = [int]$r.Y; width = [int]$r.Width; height = [int]$r.Height }", + " $child = $walker.GetNextSibling($child)", + "}", + single ? `$n = ${num}; if ($n -ge 1 -and $n -le $children.Count) { $children = @($children[$n - 1]) }` : "", + "Write-Output (@{ result = $children; count = $children.Count } | ConvertTo-Json -Compress -Depth 4)" + ], t), runScript, args); + return result.result ?? []; + }; +} + +register("Element_GetCheckGround", getCheckGround); +register("Element_SetCheckGround", setCheckGround); +register("Element_RegionGround", regionGround); +register("Element_GetSelectGround", getSelectGround); +register("Element_SetSelectGround", setSelectGround); +register("Element_GetAttributeGround", getAttributeGround); +register("Element_SetAttributeGround", setAttributeGround); +register("Element_GetButtonEnableGround", getButtonEnableGround); +register("Element_ShotGround", shotGround); +register("Element_MouseToGround", mouseToGround); +register("Element_ScrollIntoViewGround", scrollIntoViewGround); +register("Element_ChildrenGround", childrenGroundHandler(false)); +register("Element_DesktopChildren", childrenGroundHandler(true));