feat: 接入桌面 UIA 元素能力（Ground 系列）
This commit is contained in:
yu
parent
9de0eeaa88
commit
8a816fd41d
40
src/main/capabilities/desktop-uia.test.ts
Normal file
40
src/main/capabilities/desktop-uia.test.ts
Normal file
@ -0,0 +1,40 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { getHandler } from "../engine/registry";
|
||||
import { parseUiaSelector } from "./desktop-uia";
|
||||
|
||||
describe("desktop UIA ground capabilities", () => {
|
||||
it.each([
|
||||
"Element_ClickGround",
|
||||
"Element_SetValueGround",
|
||||
"Element_ExistsGround",
|
||||
"Element_GetValueGround",
|
||||
"Element_WaitElementGround"
|
||||
])("registers %s", (id) => {
|
||||
expect(getHandler(id), id).toBeTruthy();
|
||||
});
|
||||
|
||||
it("parses name selector", () => {
|
||||
const s = parseUiaSelector("name=添加供应商");
|
||||
expect(s.name).toBe("添加供应商");
|
||||
expect(s.index).toBe(1);
|
||||
});
|
||||
|
||||
it("parses window, id, type, class and index", () => {
|
||||
const s = parseUiaSelector("window=CC Switch;id=btn-save;type=button;class=btn;index=2");
|
||||
expect(s.window).toBe("CC Switch");
|
||||
expect(s.id).toBe("btn-save");
|
||||
expect(s.type).toBe("Button");
|
||||
expect(s.className).toBe("btn");
|
||||
expect(s.index).toBe(2);
|
||||
});
|
||||
|
||||
it("strips uia: prefix and accepts object form", () => {
|
||||
const s = parseUiaSelector({ value: "uia:name=确定" });
|
||||
expect(s.name).toBe("确定");
|
||||
});
|
||||
|
||||
it("rejects selector without any condition", () => {
|
||||
expect(() => parseUiaSelector("window=只有窗口")).toThrow();
|
||||
expect(() => parseUiaSelector("")).toThrow();
|
||||
});
|
||||
});
|
||||
293
src/main/capabilities/desktop-uia.ts
Normal file
293
src/main/capabilities/desktop-uia.ts
Normal file
@ -0,0 +1,293 @@
|
||||
import { register } from "../engine/registry";
|
||||
import { RuntimeError, type CapabilityHandler } from "../engine/types";
|
||||
import { powershell, psString } from "./desktop";
|
||||
|
||||
interface UiaSelector {
|
||||
window?: string;
|
||||
name?: string;
|
||||
id?: string;
|
||||
type?: string;
|
||||
className?: string;
|
||||
index: number;
|
||||
}
|
||||
|
||||
export function parseUiaSelector(raw: unknown): UiaSelector {
|
||||
let text = "";
|
||||
if (raw && typeof raw === "object") {
|
||||
const record = raw as Record<string, unknown>;
|
||||
if (typeof record.value === "string") text = record.value;
|
||||
else if (typeof record.selector === "string") text = record.selector;
|
||||
else if (record.commonProps && typeof record.commonProps === "object") {
|
||||
const common = record.commonProps as Record<string, unknown>;
|
||||
if (typeof common.uiSelector === "string") text = common.uiSelector;
|
||||
}
|
||||
} else if (raw !== undefined && raw !== null) {
|
||||
text = String(raw);
|
||||
}
|
||||
text = text.trim();
|
||||
if (!text) {
|
||||
throw new RuntimeError("validation_error", "桌面元素选择器为空", { recommendation: "使用 name=元素名称 或 id=AutomationId;window=窗口名称" });
|
||||
}
|
||||
if (text.startsWith("uia:")) text = text.slice(4);
|
||||
const selector: UiaSelector = { index: 1 };
|
||||
for (const part of text.split(";")) {
|
||||
const eq = part.indexOf("=");
|
||||
if (eq <= 0) continue;
|
||||
const key = part.slice(0, eq).trim().toLowerCase();
|
||||
const value = part.slice(eq + 1).trim();
|
||||
if (!value) continue;
|
||||
if (key === "window" || key === "win") selector.window = value;
|
||||
else if (key === "name" || key === "n") selector.name = value;
|
||||
else if (key === "id" || key === "automationid") selector.id = value;
|
||||
else if (key === "type" || key === "controltype") selector.type = value.charAt(0).toUpperCase() + value.slice(1).toLowerCase();
|
||||
else if (key === "class" || key === "classname") selector.className = value;
|
||||
else if (key === "index") {
|
||||
const n = Number(value);
|
||||
if (Number.isFinite(n) && n >= 1) selector.index = Math.floor(n);
|
||||
}
|
||||
}
|
||||
if (!selector.name && !selector.id && !selector.className && !selector.type) {
|
||||
throw new RuntimeError("validation_error", `桌面元素选择器缺少定位条件：${text}`, { recommendation: "至少提供 name= / id= / type= 之一" });
|
||||
}
|
||||
return selector;
|
||||
}
|
||||
|
||||
const UIA_USING = "Add-Type -AssemblyName UIAutomationClient, UIAutomationTypes;";
|
||||
|
||||
const FIND_HELPER = `
|
||||
function Find-RpaWindow([string]$winName) {
|
||||
$root = [System.Windows.Automation.AutomationElement]::RootElement
|
||||
if (-not $winName) { return $root }
|
||||
$cond = New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::NameProperty, $winName)
|
||||
$win = $root.FindFirst([System.Windows.Automation.TreeScope]::Children, $cond)
|
||||
if (-not $win) { throw "window_not_found:$winName" }
|
||||
return $win
|
||||
}
|
||||
function Find-RpaElement($parent, [string]$name, [string]$id, [string]$type, [string]$className, [int]$index) {
|
||||
$conds = @()
|
||||
if ($name) { $conds += New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::NameProperty, $name) }
|
||||
if ($id) { $conds += New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::AutomationIdProperty, $id) }
|
||||
if ($className) { $conds += New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::ClassNameProperty, $className) }
|
||||
if ($type) {
|
||||
try { $field = [System.Windows.Automation.ControlType].GetField($type); if ($field) { $conds += New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::ControlTypeProperty, $field.GetValue($null)) } } catch {}
|
||||
if ($ct) { $conds += New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::ControlTypeProperty, $ct) }
|
||||
}
|
||||
if ($conds.Count -eq 0) { throw "no_condition" }
|
||||
$final = if ($conds.Count -eq 1) { $conds[0] } else { New-Object System.Windows.Automation.AndCondition($conds) }
|
||||
$all = $parent.FindAll([System.Windows.Automation.TreeScope]::Descendants, $final)
|
||||
if ($all.Count -eq 0) { return $null }
|
||||
$pick = [Math]::Min([Math]::Max($index, 1), $all.Count)
|
||||
return $all[$pick - 1]
|
||||
}
|
||||
function Add-RpaUser32 {
|
||||
Add-Type -MemberDefinition '[DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y); [DllImport("user32.dll")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra); [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd);' -Name RpaUser32Click -Namespace Win32Rpa | Out-Null
|
||||
}
|
||||
`;
|
||||
|
||||
interface UiaTarget {
|
||||
selector: UiaSelector;
|
||||
args: Record<string, unknown>;
|
||||
node: { id: string; componentId: string };
|
||||
}
|
||||
|
||||
function target(args: Record<string, unknown>, node: { id: string; componentId: string }): UiaTarget {
|
||||
const raw = args.uiSelector ?? args.target ?? args.selector;
|
||||
return { selector: parseUiaSelector(raw), args, node };
|
||||
}
|
||||
|
||||
function findScript(t: UiaTarget, timeoutMs: number, gapMs: number): string {
|
||||
const s = t.selector;
|
||||
return `${UIA_USING}
|
||||
${FIND_HELPER}
|
||||
$deadline = [DateTime]::UtcNow.AddMilliseconds(${Math.max(0, Math.round(timeoutMs))})
|
||||
$found = $null
|
||||
while ($null -eq $found) {
|
||||
try {
|
||||
$parent = Find-RpaWindow ${psString(s.window ?? "")}
|
||||
$found = Find-RpaElement $parent ${psString(s.name ?? "")} ${psString(s.id ?? "")} ${psString(s.type ?? "")} ${psString(s.className ?? "")} ${s.index}
|
||||
} catch {
|
||||
$err = $_.Exception.Message
|
||||
if ($err -like "window_not_found:*") { $found = $null }
|
||||
}
|
||||
if ($null -eq $found -and [DateTime]::UtcNow -ge $deadline) { break }
|
||||
if ($null -eq $found) { Start-Sleep -Milliseconds ${Math.max(50, Math.round(gapMs))} }
|
||||
}
|
||||
if ($null -eq $found) { Write-Output "JSON_NOT_FOUND" } else {
|
||||
$rect = $found.Current.BoundingRectangle
|
||||
$info = @{ found = $true; name = $found.Current.Name; id = $found.Current.AutomationId; type = $found.Current.ControlType.ProgrammaticName; enabled = $found.Current.IsEnabled; x = [int]$rect.X; y = [int]$rect.Y; w = [int]$rect.Width; h = [int]$rect.Height }
|
||||
Write-Output ($info | ConvertTo-Json -Compress)
|
||||
}`;
|
||||
}
|
||||
|
||||
async function findElement(t: UiaTarget, timeoutMs = 5000, gapMs = 200): Promise<Record<string, unknown> | null> {
|
||||
const out = await powershell(findScript(t, timeoutMs, gapMs), Math.max(30000, timeoutMs + 30000));
|
||||
const line = out.split(/\r?\n/).filter(Boolean).pop() || "";
|
||||
if (line.includes("JSON_NOT_FOUND")) return null;
|
||||
try {
|
||||
return JSON.parse(line);
|
||||
} catch {
|
||||
throw new RuntimeError("desktop_command_failed", `桌面元素查找输出异常：${out.slice(0, 200)}`, { nodeId: t.node.id, componentId: t.node.componentId });
|
||||
}
|
||||
}
|
||||
|
||||
function clickScript(t: UiaTarget): string {
|
||||
const s = t.selector;
|
||||
const clickway = String(t.args.clickway ?? "0");
|
||||
const doubleClick = clickway === "1" || /double/i.test(clickway);
|
||||
return `${UIA_USING}
|
||||
${FIND_HELPER}
|
||||
$parent = Find-RpaWindow ${psString(s.window ?? "")}
|
||||
$el = Find-RpaElement $parent ${psString(s.name ?? "")} ${psString(s.id ?? "")} ${psString(s.type ?? "")} ${psString(s.className ?? "")} ${s.index}
|
||||
if ($null -eq $el) { Write-Output "JSON_NOT_FOUND"; exit 0 }
|
||||
$rect = $el.Current.BoundingRectangle
|
||||
$offscreen = $el.Current.IsOffscreen -or $rect.Width -le 0 -or $rect.Height -le 0
|
||||
$method = "coordinate_click"
|
||||
$ok = $false
|
||||
if (-not $offscreen) {
|
||||
Add-RpaUser32
|
||||
$walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker
|
||||
$ancestor = $el
|
||||
$hwnd = [IntPtr]::Zero
|
||||
while ($null -ne $ancestor) {
|
||||
$h = $ancestor.Current.NativeWindowHandle
|
||||
if ($h -ne 0) { $hwnd = [IntPtr]$h; break }
|
||||
$ancestor = $walker.GetParent($ancestor)
|
||||
}
|
||||
if ($hwnd -ne [IntPtr]::Zero) { [WinRpa.RpaUser32Click]::SetForegroundWindow($hwnd) | Out-Null; Start-Sleep -Milliseconds 250 }
|
||||
$x = [int]($rect.X + $rect.Width / 2)
|
||||
$y = [int]($rect.Y + $rect.Height / 2)
|
||||
[WinRpa.RpaUser32Click]::SetCursorPos($x, $y) | Out-Null
|
||||
Start-Sleep -Milliseconds 120
|
||||
[WinRpa.RpaUser32Click]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)
|
||||
Start-Sleep -Milliseconds 40
|
||||
[WinRpa.RpaUser32Click]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)
|
||||
if (${doubleClick}) {
|
||||
Start-Sleep -Milliseconds 60
|
||||
[WinRpa.RpaUser32Click]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)
|
||||
Start-Sleep -Milliseconds 40
|
||||
[WinRpa.RpaUser32Click]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)
|
||||
}
|
||||
$ok = $true
|
||||
} else {
|
||||
$method = "invoke_pattern"
|
||||
foreach ($entry in @(@("Invoke", [System.Windows.Automation.InvokePattern]::Pattern), @("Toggle", [System.Windows.Automation.TogglePattern]::Pattern), @("Expand", [System.Windows.Automation.ExpandCollapsePattern]::Pattern), @("Select", [System.Windows.Automation.SelectionItemPattern]::Pattern))) {
|
||||
try {
|
||||
$p = $el.GetCurrentPattern($entry[1])
|
||||
switch ($entry[0]) {
|
||||
"Invoke" { $p.Invoke(); $ok = $true }
|
||||
"Toggle" { $p.Toggle(); $ok = $true }
|
||||
"Expand" { $p.Expand(); $ok = $true }
|
||||
"Select" { $p.Select(); $ok = $true }
|
||||
}
|
||||
if ($ok) { break }
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
$rect2 = $el.Current.BoundingRectangle
|
||||
Write-Output (@{ clicked = $ok; method = $method; name = $el.Current.Name; x = [int]$rect2.X; y = [int]$rect2.Y; w = [int]$rect2.Width; h = [int]$rect2.Height } | ConvertTo-Json -Compress)`;
|
||||
}
|
||||
function setValueScript(t: UiaTarget, value: string): string {
|
||||
const s = t.selector;
|
||||
return `${UIA_USING}
|
||||
${FIND_HELPER}
|
||||
$parent = Find-RpaWindow ${psString(s.window ?? "")}
|
||||
$el = Find-RpaElement $parent ${psString(s.name ?? "")} ${psString(s.id ?? "")} ${psString(s.type ?? "")} ${psString(s.className ?? "")} ${s.index}
|
||||
if ($null -eq $el) { Write-Output "JSON_NOT_FOUND"; exit 0 }
|
||||
$value = ${psString(value)}
|
||||
$method = "value_pattern"
|
||||
$ok = $false
|
||||
try {
|
||||
$vp = $el.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern)
|
||||
$vp.SetValue($value)
|
||||
$ok = $true
|
||||
} catch {
|
||||
$method = "clipboard_paste"
|
||||
Add-Type -AssemblyName System.Windows.Forms
|
||||
$saved = $null
|
||||
try { $saved = [System.Windows.Forms.Clipboard]::GetText() } catch {}
|
||||
try { $el.SetFocus() } catch {}
|
||||
Start-Sleep -Milliseconds 150
|
||||
try { [System.Windows.Forms.Clipboard]::SetText($value) } catch {}
|
||||
Start-Sleep -Milliseconds 80
|
||||
[System.Windows.Forms.SendKeys]::SendWait("^a")
|
||||
Start-Sleep -Milliseconds 60
|
||||
[System.Windows.Forms.SendKeys]::SendWait("^v")
|
||||
Start-Sleep -Milliseconds 300
|
||||
$ok = $true
|
||||
try {
|
||||
if ($saved) { [System.Windows.Forms.Clipboard]::SetText($saved) }
|
||||
elseif ($saved -ne $null) { [System.Windows.Forms.Clipboard]::Clear() }
|
||||
} catch {}
|
||||
}
|
||||
Write-Output (@{ set = $ok; method = $method; name = $el.Current.Name } | ConvertTo-Json -Compress)`;
|
||||
}
|
||||
function getValueScript(t: UiaTarget): string {
|
||||
const s = t.selector;
|
||||
return `${UIA_USING}
|
||||
${FIND_HELPER}
|
||||
$parent = Find-RpaWindow ${psString(s.window ?? "")}
|
||||
$el = Find-RpaElement $parent ${psString(s.name ?? "")} ${psString(s.id ?? "")} ${psString(s.type ?? "")} ${psString(s.className ?? "")} ${s.index}
|
||||
if ($null -eq $el) { Write-Output "JSON_NOT_FOUND"; exit 0 }
|
||||
$value = $el.Current.Name
|
||||
try { $vp = $el.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern); $value = $vp.Value } catch {}
|
||||
Write-Output (@{ value = $value; name = $el.Current.Name } | ConvertTo-Json -Compress)`;
|
||||
}
|
||||
|
||||
async function runScript(script: string, t: UiaTarget, timeoutMs = 60000): Promise<Record<string, unknown> | null> {
|
||||
const out = await powershell(script, timeoutMs);
|
||||
const line = out.split(/\r?\n/).filter(Boolean).pop() || "";
|
||||
if (line.includes("JSON_NOT_FOUND")) return null;
|
||||
try {
|
||||
return JSON.parse(line);
|
||||
} catch {
|
||||
throw new RuntimeError("desktop_command_failed", `桌面元素操作输出异常：${out.slice(0, 200)}`, { nodeId: t.node.id, componentId: t.node.componentId });
|
||||
}
|
||||
}
|
||||
|
||||
function requireFound(result: Record<string, unknown> | null, t: UiaTarget, action: string): Record<string, unknown> {
|
||||
if (!result) {
|
||||
const s = t.selector;
|
||||
throw new RuntimeError("desktop_element_not_found", `桌面元素未找到（${action}）：${JSON.stringify({ window: s.window, name: s.name, id: s.id, type: s.type, className: s.className, index: s.index })}`, { nodeId: t.node.id, componentId: t.node.componentId });
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
const clickGround: CapabilityHandler = async (_ctx, node, args) => {
|
||||
const t = target(args, node);
|
||||
const result = await runScript(clickScript(t), t);
|
||||
return requireFound(result, t, "click");
|
||||
};
|
||||
|
||||
const setValueGround: CapabilityHandler = async (_ctx, node, args) => {
|
||||
const t = target(args, node);
|
||||
const value = String(args.value ?? args.text ?? "");
|
||||
const result = await runScript(setValueScript(t, value), t);
|
||||
return requireFound(result, t, "setValue");
|
||||
};
|
||||
|
||||
const existsGround: CapabilityHandler = async (_ctx, node, args) => {
|
||||
const t = target(args, node);
|
||||
const found = await findElement(t, 1000, 150);
|
||||
return found !== null;
|
||||
};
|
||||
|
||||
const getValueGround: CapabilityHandler = async (_ctx, node, args) => {
|
||||
const t = target(args, node);
|
||||
const result = await runScript(getValueScript(t), t);
|
||||
return requireFound(result, t, "getValue").value ?? "";
|
||||
};
|
||||
|
||||
const waitElementGround: CapabilityHandler = async (_ctx, node, args) => {
|
||||
const t = target(args, node);
|
||||
const timeout = Number(args.timeout ?? args.timeoutMs ?? 5000) || 5000;
|
||||
const gap = Number(args.time_gap ?? args.gapMs ?? 200) || 200;
|
||||
const found = await findElement(t, timeout, gap);
|
||||
return found !== null;
|
||||
};
|
||||
|
||||
register("Element_ClickGround", clickGround);
|
||||
register("Element_SetValueGround", setValueGround);
|
||||
register("Element_ExistsGround", existsGround);
|
||||
register("Element_GetValueGround", getValueGround);
|
||||
register("Element_WaitElementGround", waitElementGround);
|
||||
|
||||
@ -9,6 +9,7 @@ import "./browser-common";
|
||||
import "./browser-control";
|
||||
import "./office";
|
||||
import "./desktop";
|
||||
import "./desktop-uia";
|
||||
import "./pdf";
|
||||
import "./word";
|
||||
import "./word-extended";
|
||||
@ -26,3 +27,4 @@ import "./data-table-common";
|
||||
import "./common-final";
|
||||
|
||||
export { capabilityCount, capabilityIds, getHandler, hasCapability, register } from "../engine/registry";
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user