feat: desktop ground variants and desktop children
This commit is contained in:
yu
parent
adf5f3f7cf
commit
32269a5760
@ -8,7 +8,20 @@ describe("desktop UIA ground capabilities", () => {
|
||||
"Element_SetValueGround",
|
||||
"Element_ExistsGround",
|
||||
"Element_GetValueGround",
|
||||
"Element_WaitElementGround"
|
||||
"Element_WaitElementGround",
|
||||
"Element_GetCheckGround",
|
||||
"Element_SetCheckGround",
|
||||
"Element_RegionGround",
|
||||
"Element_GetSelectGround",
|
||||
"Element_SetSelectGround",
|
||||
"Element_GetAttributeGround",
|
||||
"Element_SetAttributeGround",
|
||||
"Element_GetButtonEnableGround",
|
||||
"Element_ShotGround",
|
||||
"Element_MouseToGround",
|
||||
"Element_ScrollIntoViewGround",
|
||||
"Element_ChildrenGround",
|
||||
"Element_DesktopChildren"
|
||||
])("registers %s", (id) => {
|
||||
expect(getHandler(id), id).toBeTruthy();
|
||||
});
|
||||
|
||||
@ -1,4 +1,6 @@
|
||||
import { register } from "../engine/registry";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { RuntimeError, type CapabilityHandler } from "../engine/types";
|
||||
import { powershell, psString } from "./desktop";
|
||||
|
||||
@ -393,3 +395,242 @@ register("Element_WaitElementGround", waitElementGround);
|
||||
|
||||
|
||||
|
||||
|
||||
function outputPath(args: Record<string, unknown>, extension = "png"): string {
|
||||
const raw = args.filePath ?? args.folder ?? args.path;
|
||||
const folder = path.resolve(String(raw && String(raw).trim() ? raw : process.cwd()));
|
||||
const rawName = String(args.filename ?? args.fileName ?? args.name ?? `element-${Date.now()}.${extension}`)
|
||||
.replace(/\$\{randomString\}/g, Math.random().toString(36).slice(2, 10));
|
||||
const name = /\.[A-Za-z0-9]{2,8}$/.test(rawName) ? rawName : `${rawName}.${extension}`;
|
||||
return path.join(folder, name);
|
||||
}
|
||||
|
||||
function propertyValue(scriptLines: string[], t: UiaTarget): string {
|
||||
const s = t.selector;
|
||||
return `${UIA_USING}
|
||||
${FIND_HELPER}
|
||||
$el = $null
|
||||
try {
|
||||
$parent = Find-RpaWindow ${psString(s.window ?? "")}
|
||||
$el = Find-RpaElement $parent ${psString(s.name ?? "")} ${psString(s.id ?? "")} ${psString(s.type ?? "")} ${psString(s.className ?? "")} ${s.index}
|
||||
} catch { $el = $null }
|
||||
if ($null -eq $el) { Write-Output "JSON_NOT_FOUND"; exit 0 }
|
||||
${scriptLines.join("\n")}`;
|
||||
}
|
||||
|
||||
const getCheckGround: CapabilityHandler = async (_ctx, node, args) => {
|
||||
const t = target(args, node);
|
||||
const result = await runWithImplicitWait(t, "getCheck", propertyValue([
|
||||
"$checked = $false",
|
||||
"try { $checked = $el.GetCurrentPattern([System.Windows.Automation.TogglePattern]::Pattern).Current.ToggleState -eq [System.Windows.Automation.ToggleState]::On } catch {}",
|
||||
"Write-Output (@{ value = [bool]$checked; name = $el.Current.Name; enabled = [bool]$el.Current.IsEnabled } | ConvertTo-Json -Compress)"
|
||||
], t), runScript, args);
|
||||
return result.value === true;
|
||||
};
|
||||
|
||||
const setCheckGround: CapabilityHandler = async (_ctx, node, args) => {
|
||||
const t = target(args, node);
|
||||
const desired = ["1", "true", "yes", "on"].includes(String(args.isChecked ?? "1").toLowerCase());
|
||||
return runWithImplicitWait(t, "setCheck", propertyValue([
|
||||
`$desired = ${desired ? "$true" : "$false"}`,
|
||||
"$current = $false; $hasToggle = $false",
|
||||
"try { $pattern = $el.GetCurrentPattern([System.Windows.Automation.TogglePattern]::Pattern); $current = $pattern.Current.ToggleState -eq [System.Windows.Automation.ToggleState]::On; $hasToggle = $true } catch {}",
|
||||
"$method = if ($hasToggle) { 'toggle_pattern' } else { 'coordinate_click' }",
|
||||
"if (-not $hasToggle -or $current -ne $desired) {",
|
||||
" if ($hasToggle) { $pattern.Toggle() | Out-Null }",
|
||||
" else {",
|
||||
" $rect = $el.Current.BoundingRectangle; Add-RpaUser32",
|
||||
" [Win32Rpa.RpaUser32Click]::SetCursorPos([int]($rect.X + $rect.Width / 2), [int]($rect.Y + $rect.Height / 2)) | Out-Null",
|
||||
" Start-Sleep -Milliseconds 100; [Win32Rpa.RpaUser32Click]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)",
|
||||
" Start-Sleep -Milliseconds 50; [Win32Rpa.RpaUser32Click]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)",
|
||||
" }",
|
||||
"}",
|
||||
"$after = $false; try { $after = $el.GetCurrentPattern([System.Windows.Automation.TogglePattern]::Pattern).Current.ToggleState -eq [System.Windows.Automation.ToggleState]::On } catch {}",
|
||||
"Write-Output (@{ set = ($after -eq $desired); checked = [bool]$after; method = $method; name = $el.Current.Name } | ConvertTo-Json -Compress)"
|
||||
], t), runScript, args);
|
||||
};
|
||||
|
||||
const regionGround: CapabilityHandler = async (_ctx, node, args) => {
|
||||
const t = target(args, node);
|
||||
const relative = String(args.position ?? "3") === "2";
|
||||
const result = await runWithImplicitWait(t, "getRegion", propertyValue([
|
||||
"$rect = $el.Current.BoundingRectangle; $ox = 0; $oy = 0",
|
||||
relative ? [
|
||||
"try {",
|
||||
" Add-RpaUser32",
|
||||
' Add-Type -MemberDefinition \'[DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr hWnd, out RECT rect); public struct RECT { public int Left; public int Top; public int Right; public int Bottom; }\' -Name RpaWindowRect -Namespace Win32RpaRect | Out-Null',
|
||||
" $walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker; $ancestor = $el",
|
||||
" while ($null -ne $ancestor) {",
|
||||
" $hwnd = $ancestor.Current.NativeWindowHandle",
|
||||
" if ($hwnd -ne 0) { $wr = New-Object Win32RpaRect.RpaWindowRect+RECT; [void][Win32RpaRect.RpaWindowRect]::GetWindowRect([IntPtr]$hwnd, [ref]$wr); $ox = $wr.Left; $oy = $wr.Top; break }",
|
||||
" $ancestor = $walker.GetParent($ancestor)",
|
||||
" }",
|
||||
"} catch {}",
|
||||
].join("\n") : "",
|
||||
"Write-Output (@{ x = [int]($rect.X - $ox); y = [int]($rect.Y - $oy); width = [int]$rect.Width; height = [int]$rect.Height } | ConvertTo-Json -Compress)"
|
||||
], t), runScript, args);
|
||||
return result;
|
||||
};
|
||||
|
||||
function getSelectScript(t: UiaTarget): string {
|
||||
const byText = String(t.args.choose ?? "1") !== "2";
|
||||
return propertyValue([
|
||||
"$result = @(); $method = 'selection_pattern'",
|
||||
"try { $sel = $el.GetCurrentPattern([System.Windows.Automation.SelectionPattern]::Pattern); foreach ($item in $sel.Current.GetSelection()) { $value = $item.Current.Name; if (-not " + (byText ? "$true" : "$false") + ") { $value = $item.Current.AutomationId }; $result += @{ text = $item.Current.Name; value = $item.Current.AutomationId; selected = $true } } } catch {}",
|
||||
"if ($result.Count -eq 0) { try { $value = $el.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern).Value } catch { $value = $el.Current.Name }; $result += @{ text = [string]$value; value = [string]$value; selected = $true }; $method = 'value_pattern' }",
|
||||
"Write-Output (@{ result = $result; method = $method; name = $el.Current.Name } | ConvertTo-Json -Compress -Depth 3)"
|
||||
], t);
|
||||
}
|
||||
|
||||
const getSelectGround: CapabilityHandler = async (_ctx, node, args) => {
|
||||
const t = target(args, node);
|
||||
const result = await runWithImplicitWait(t, "getSelect", getSelectScript(t), runScript, args);
|
||||
const items = Array.isArray(result.result) ? result.result : [];
|
||||
return items.length === 1 ? items[0] : items;
|
||||
};
|
||||
|
||||
function setSelectScript(t: UiaTarget, values: string[]): string {
|
||||
const encoded = Buffer.from(JSON.stringify(values), "utf16le").toString("base64");
|
||||
const choose = String(t.args.choose ?? "1");
|
||||
return propertyValue([
|
||||
`$desired = [Text.Encoding]::Unicode.GetString([Convert]::FromBase64String('${encoded}')) | ConvertFrom-Json`,
|
||||
"$items = @($el.FindAll([System.Windows.Automation.TreeScope]::Descendants, [System.Windows.Automation.Condition]::TrueCondition) | Where-Object { $_.Current.ControlType.ProgrammaticName -match 'ListItem|ComboBoxItem' })",
|
||||
"if ($items.Count -eq 0) { try { $el.Expand() } catch {}; Start-Sleep -Milliseconds 250; $items = @($el.FindAll([System.Windows.Automation.TreeScope]::Descendants, [System.Windows.Automation.Condition]::TrueCondition) | Where-Object { $_.Current.ControlType.ProgrammaticName -match 'ListItem|ComboBoxItem' }) }",
|
||||
"$targets = @(); $method = 'selection_item_pattern'",
|
||||
"foreach ($wanted in @($desired)) {",
|
||||
` if ('${choose}' -eq '0') { $ix = [int]$wanted - 1; if ($ix -ge 0 -and $ix -lt $items.Count) { $targets += $items[$ix] } }`,
|
||||
` elseif ('${choose}' -eq '2') { $targets += @($items | Where-Object { $_.Current.AutomationId -eq [string]$wanted }) }`,
|
||||
" else { $targets += @($items | Where-Object { $_.Current.Name -eq [string]$wanted }) }",
|
||||
"}",
|
||||
"$ok = $targets.Count -gt 0",
|
||||
"foreach ($item in $targets) { try { $item.GetCurrentPattern([System.Windows.Automation.SelectionItemPattern]::Pattern).Select() } catch { try { $item.Invoke() } catch { $ok = $false } } }",
|
||||
"try { $el.Collapse() } catch {}",
|
||||
"Write-Output (@{ set = [bool]$ok; matched = $targets.Count; available = $items.Count; method = $method; name = $el.Current.Name } | ConvertTo-Json -Compress)"
|
||||
], t);
|
||||
}
|
||||
|
||||
const setSelectGround: CapabilityHandler = async (_ctx, node, args) => {
|
||||
const t = target(args, node);
|
||||
const raw = args.itemsToSelect ?? args.value ?? args.text;
|
||||
let values: string[] = [];
|
||||
if (Array.isArray(raw)) values = raw.map((value) => String(value));
|
||||
else if (raw !== undefined && raw !== null && String(raw).trim() !== "") {
|
||||
try { const parsed = JSON.parse(String(raw)); values = Array.isArray(parsed) ? parsed.map((value) => String(value)) : [String(parsed)]; }
|
||||
catch { values = [String(raw)]; }
|
||||
}
|
||||
if (!values.length) throw new RuntimeError("validation_error", "下拉框选择项为空", { nodeId: node.id, componentId: node.componentId });
|
||||
return runWithImplicitWait(t, "setSelect", setSelectScript(t, values), runScript, args);
|
||||
};
|
||||
|
||||
function attrScript(t: UiaTarget, attrName: string, attrValue = ""): string {
|
||||
const name = attrName.toLowerCase();
|
||||
if (name === "value" || name === "text") {
|
||||
return propertyValue([
|
||||
"try { $value = $el.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern).Value } catch { $value = $el.Current.Name }",
|
||||
"Write-Output (@{ value = [string]$value; name = $el.Current.Name } | ConvertTo-Json -Compress)"
|
||||
], t);
|
||||
}
|
||||
if (!attrValue) {
|
||||
return propertyValue([
|
||||
"$p = @{ name = $el.Current.Name; id = $el.Current.AutomationId; class = $el.Current.ClassName; type = $el.Current.ControlType.ProgrammaticName; enabled = $el.Current.IsEnabled; help = $el.Current.HelpText; value = $el.Current.Name }",
|
||||
`$key = ${psString(name || "name")}`,
|
||||
"$result = if ($p.Contains($key)) { $p[$key] } else { $p.name }",
|
||||
"Write-Output (@{ value = [string]$result; properties = $p; name = $el.Current.Name } | ConvertTo-Json -Compress)"
|
||||
], t);
|
||||
}
|
||||
if (name !== "value" && name !== "text") throw new RuntimeError("validation_error", `桌面 UIA 不支持设置属性：${attrName}；可设置 value/text`, { nodeId: t.node.id, componentId: t.node.componentId });
|
||||
return propertyValue([
|
||||
"$ok = $false; $method = 'value_pattern'",
|
||||
`try { $el.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern).SetValue(${psString(attrValue)}); $ok = $true } catch {}`,
|
||||
"if (-not $ok) { $method = 'keyboard'; Add-Type -AssemblyName System.Windows.Forms; $el.SetFocus() | Out-Null; Start-Sleep -Milliseconds 100; [System.Windows.Forms.SendKeys]::SendWait('^a'); [System.Windows.Forms.SendKeys]::SendWait('^v'); Start-Sleep -Milliseconds 100; $ok = $true }",
|
||||
"Write-Output (@{ set = $ok; method = $method; name = $el.Current.Name } | ConvertTo-Json -Compress)"
|
||||
], t);
|
||||
}
|
||||
|
||||
const getAttributeGround: CapabilityHandler = async (_ctx, node, args) => {
|
||||
const t = target(args, node);
|
||||
const attrName = String(args.attrName ?? args.attribute ?? "name");
|
||||
const result = await runWithImplicitWait(t, "getAttribute", attrScript(t, attrName), runScript, args);
|
||||
return result.value ?? "";
|
||||
};
|
||||
|
||||
const setAttributeGround: CapabilityHandler = async (_ctx, node, args) => {
|
||||
const t = target(args, node);
|
||||
const attrName = String(args.attrName ?? args.attribute ?? "value");
|
||||
const attrValue = String(args.attrValue ?? args.value ?? "");
|
||||
return runWithImplicitWait(t, "setAttribute", attrScript(t, attrName, attrValue), runScript, args);
|
||||
};
|
||||
|
||||
const getButtonEnableGround: CapabilityHandler = async (_ctx, node, args) => {
|
||||
const t = target(args, node);
|
||||
const result = await runWithImplicitWait(t, "getButtonEnable", propertyValue(["Write-Output (@{ value = [bool]$el.Current.IsEnabled; name = $el.Current.Name } | ConvertTo-Json -Compress)"], t), runScript, args);
|
||||
return result.value === true;
|
||||
};
|
||||
|
||||
const shotGround: CapabilityHandler = async (_ctx, node, args) => {
|
||||
const t = target(args, node);
|
||||
const output = outputPath(args);
|
||||
fs.mkdirSync(path.dirname(output), { recursive: true });
|
||||
const result = await runWithImplicitWait(t, "shot", propertyValue([
|
||||
"$rect = $el.Current.BoundingRectangle; if ($rect.Width -le 0 -or $rect.Height -le 0) { throw 'element_not_visible' }",
|
||||
"Add-Type -AssemblyName System.Drawing | Out-Null",
|
||||
"$bitmap = New-Object System.Drawing.Bitmap([int]$rect.Width, [int]$rect.Height)",
|
||||
"$graphics = [System.Drawing.Graphics]::FromImage($bitmap)",
|
||||
"$graphics.CopyFromScreen([int]$rect.X, [int]$rect.Y, 0, 0, $bitmap.Size); $graphics.Dispose()",
|
||||
`$bitmap.Save(${psString(output)}, [System.Drawing.Imaging.ImageFormat]::Png); $bitmap.Dispose()`,
|
||||
`Write-Output (@{ value = ${psString(output)}; x = [int]$rect.X; y = [int]$rect.Y; width = [int]$rect.Width; height = [int]$rect.Height } | ConvertTo-Json -Compress)`
|
||||
], t), runScript, args);
|
||||
return result.value ?? output;
|
||||
};
|
||||
|
||||
const mouseToGround: CapabilityHandler = async (_ctx, node, args) => {
|
||||
const t = target(args, node);
|
||||
return runWithImplicitWait(t, "mouseTo", propertyValue([
|
||||
"$rect = $el.Current.BoundingRectangle; Add-RpaUser32",
|
||||
"$x = [int]($rect.X + $rect.Width / 2); $y = [int]($rect.Y + $rect.Height / 2)",
|
||||
"[Win32Rpa.RpaUser32Click]::SetCursorPos($x, $y) | Out-Null",
|
||||
"Write-Output (@{ x = $x; y = $y; name = $el.Current.Name } | ConvertTo-Json -Compress)"
|
||||
], t), runScript, args);
|
||||
};
|
||||
|
||||
const scrollIntoViewGround: CapabilityHandler = async (_ctx, node, args) => {
|
||||
const t = target(args, node);
|
||||
return runWithImplicitWait(t, "scrollIntoView", propertyValue([
|
||||
"$ok = $false; $method = 'scroll_item_pattern'",
|
||||
"try { $el.GetCurrentPattern([System.Windows.Automation.ScrollItemPattern]::Pattern).ScrollIntoView(); $ok = $true } catch {}",
|
||||
"if (-not $ok) { $method = 'focus'; try { $el.SetFocus() | Out-Null; $ok = $true } catch {} }",
|
||||
"Write-Output (@{ scrolled = $ok; method = $method; name = $el.Current.Name } | ConvertTo-Json -Compress)"
|
||||
], t), runScript, args);
|
||||
};
|
||||
|
||||
function childrenGroundHandler(single: boolean): CapabilityHandler {
|
||||
return async (_ctx, node, args) => {
|
||||
const t = target(args, node);
|
||||
const num = Math.max(0, Math.floor(Number(args.num ?? 0) || 0));
|
||||
const result = await runWithImplicitWait(t, "getChildren", propertyValue([
|
||||
"$walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker; $child = $walker.GetFirstChild($el); $children = @()",
|
||||
"while ($null -ne $child) {",
|
||||
" $r = $child.Current.BoundingRectangle",
|
||||
" $children += @{ name = $child.Current.Name; id = $child.Current.AutomationId; type = $child.Current.ControlType.ProgrammaticName; className = $child.Current.ClassName; enabled = [bool]$child.Current.IsEnabled; x = [int]$r.X; y = [int]$r.Y; width = [int]$r.Width; height = [int]$r.Height }",
|
||||
" $child = $walker.GetNextSibling($child)",
|
||||
"}",
|
||||
single ? `$n = ${num}; if ($n -ge 1 -and $n -le $children.Count) { $children = @($children[$n - 1]) }` : "",
|
||||
"Write-Output (@{ result = $children; count = $children.Count } | ConvertTo-Json -Compress -Depth 4)"
|
||||
], t), runScript, args);
|
||||
return result.result ?? [];
|
||||
};
|
||||
}
|
||||
|
||||
register("Element_GetCheckGround", getCheckGround);
|
||||
register("Element_SetCheckGround", setCheckGround);
|
||||
register("Element_RegionGround", regionGround);
|
||||
register("Element_GetSelectGround", getSelectGround);
|
||||
register("Element_SetSelectGround", setSelectGround);
|
||||
register("Element_GetAttributeGround", getAttributeGround);
|
||||
register("Element_SetAttributeGround", setAttributeGround);
|
||||
register("Element_GetButtonEnableGround", getButtonEnableGround);
|
||||
register("Element_ShotGround", shotGround);
|
||||
register("Element_MouseToGround", mouseToGround);
|
||||
register("Element_ScrollIntoViewGround", scrollIntoViewGround);
|
||||
register("Element_ChildrenGround", childrenGroundHandler(false));
|
||||
register("Element_DesktopChildren", childrenGroundHandler(true));
|
||||
|
||||
Loading…
Reference in New Issue
Block a user