1070 lines
53 KiB
TypeScript
1070 lines
53 KiB
TypeScript
import { register } from "../engine/registry";
|
||
import fs from "node:fs";
|
||
import path from "node:path";
|
||
import { RuntimeError, type CapabilityHandler } from "../engine/types";
|
||
import { powershell, psString } from "../driver/windows/powershell";
|
||
import type { UiaSelector } from "../driver/types";
|
||
import { setElementDriverImplementation } from "../driver/windows/windows-driver";
|
||
|
||
// UiaSelector 类型已上移 driver/types.ts（任务 3.2 element 域）
|
||
|
||
/** ControlType 显式映射：type= 值 → .NET ControlType 字段名；复合名称不得用首字母大写推导 */
|
||
const UIA_CONTROL_TYPE_MAP: Record<string, string> = {
|
||
button: "Button",
|
||
calendar: "Calendar",
|
||
checkbox: "CheckBox",
|
||
combo: "ComboBox",
|
||
combobox: "ComboBox",
|
||
custom: "Custom",
|
||
datagrid: "DataGrid",
|
||
dataitem: "DataItem",
|
||
document: "Document",
|
||
edit: "Edit",
|
||
group: "Group",
|
||
header: "Header",
|
||
headeritem: "HeaderItem",
|
||
hyperlink: "Hyperlink",
|
||
image: "Image",
|
||
list: "List",
|
||
listbox: "List",
|
||
listitem: "ListItem",
|
||
menu: "Menu",
|
||
menubar: "MenuBar",
|
||
menuitem: "MenuItem",
|
||
pane: "Pane",
|
||
progressbar: "ProgressBar",
|
||
radio: "RadioButton",
|
||
radiobutton: "RadioButton",
|
||
scrollbar: "ScrollBar",
|
||
separator: "Separator",
|
||
slider: "Slider",
|
||
spinner: "Spinner",
|
||
splitbutton: "SplitButton",
|
||
statusbar: "StatusBar",
|
||
tab: "Tab",
|
||
tabitem: "TabItem",
|
||
table: "Table",
|
||
text: "Text",
|
||
thumb: "Thumb",
|
||
titlebar: "TitleBar",
|
||
toolbar: "ToolBar",
|
||
tooltip: "ToolTip",
|
||
tree: "Tree",
|
||
treeitem: "TreeItem",
|
||
window: "Window"
|
||
};
|
||
|
||
function normalizeUiaControlType(raw: string): string {
|
||
const key = raw.trim().toLowerCase().replace(/[\s_-]+/g, "");
|
||
return UIA_CONTROL_TYPE_MAP[key] ?? raw.charAt(0).toUpperCase() + raw.slice(1).toLowerCase();
|
||
}
|
||
|
||
/** 解析 path= 段：Pane>Edit[name=用户名;index=2]，段间用 > 分隔，段内 [prop=value;...] */
|
||
function parseUiaSelectorPath(value: string): UiaSelector["path"] {
|
||
// 按括号外的 > 分段，避免 path=Edit[name=a>b] 这类属性值被错误拆开
|
||
const rawSegments: string[] = [];
|
||
let buffer = "";
|
||
let bracketDepth = 0;
|
||
for (const char of value) {
|
||
if (char === "[") bracketDepth += 1;
|
||
else if (char === "]") bracketDepth = Math.max(0, bracketDepth - 1);
|
||
if (char === ">" && bracketDepth === 0) {
|
||
rawSegments.push(buffer);
|
||
buffer = "";
|
||
} else {
|
||
buffer += char;
|
||
}
|
||
}
|
||
rawSegments.push(buffer);
|
||
const segments = rawSegments.map(item => item.trim()).filter(Boolean);
|
||
if (!segments.length) return undefined;
|
||
const path: NonNullable<UiaSelector["path"]> = [];
|
||
for (const segment of segments) {
|
||
const match = segment.match(/^([A-Za-z][A-Za-z0-9_]*)?\s*(?:\[(.*)\])?$/);
|
||
if (!match) {
|
||
throw new RuntimeError("validation_error", `UIA 路径段无效：${segment}`, { recommendation: "使用 Type[name=xx;id=yy;class=zz;index=1] > 下一级" });
|
||
}
|
||
const entry: NonNullable<UiaSelector["path"]>[number] = {};
|
||
if (match[1]) entry.type = normalizeUiaControlType(match[1]);
|
||
const props = match[2];
|
||
if (props) {
|
||
for (const pair of props.split(/[;，,]/).map(item => item.trim()).filter(Boolean)) {
|
||
const eq = pair.indexOf("=");
|
||
if (eq <= 0) {
|
||
throw new RuntimeError("validation_error", `UIA 路径属性无效：${pair}`, { recommendation: "使用 name= / id= / class= / index=" });
|
||
}
|
||
const key = pair.slice(0, eq).trim().toLowerCase();
|
||
const val = pair.slice(eq + 1).trim();
|
||
if (key === "name" || key === "n") entry.name = val;
|
||
else if (key === "id" || key === "automationid") entry.id = val;
|
||
else if (key === "class" || key === "classname") entry.className = val;
|
||
else if (key === "type" || key === "controltype") entry.type = normalizeUiaControlType(val);
|
||
else if (key === "index") {
|
||
const n = Number(val);
|
||
if (!Number.isFinite(n) || n < 1) throw new RuntimeError("validation_error", `UIA 路径 index 无效：${val}`);
|
||
entry.index = Math.floor(n);
|
||
}
|
||
else throw new RuntimeError("validation_error", `UIA 路径属性不支持：${key}`);
|
||
}
|
||
}
|
||
if (!entry.type && !entry.name && !entry.id && !entry.className) {
|
||
throw new RuntimeError("validation_error", `UIA 路径段缺少定位条件：${segment}`, { recommendation: "每段至少提供 type/name/id/class 之一" });
|
||
}
|
||
path.push(entry);
|
||
}
|
||
return path;
|
||
}
|
||
|
||
export function parseUiaSelector(raw: unknown): UiaSelector {
|
||
let text = "";
|
||
if (raw && typeof raw === "object") {
|
||
const record = raw as Record<string, unknown>;
|
||
if (typeof record.value === "string") text = record.value;
|
||
else if (typeof record.selector === "string") text = record.selector;
|
||
else if (record.commonProps && typeof record.commonProps === "object") {
|
||
const common = record.commonProps as Record<string, unknown>;
|
||
if (typeof common.uiSelector === "string") text = common.uiSelector;
|
||
}
|
||
} else if (raw !== undefined && raw !== null) {
|
||
text = String(raw);
|
||
}
|
||
text = text.trim();
|
||
if (!text) {
|
||
throw new RuntimeError("validation_error", "桌面元素选择器为空", { recommendation: "使用 name=元素名称 或 id=AutomationId;window=窗口名称" });
|
||
}
|
||
if (text.startsWith("uia:")) text = text.slice(4);
|
||
const selector: UiaSelector = { index: 1 };
|
||
// path 段内的 [name=a;b] 不应被顶层 ; 拆开
|
||
const selectorParts: string[] = [];
|
||
let partBuffer = "";
|
||
let selectorBracketDepth = 0;
|
||
for (const char of text) {
|
||
if (char === "[") selectorBracketDepth += 1;
|
||
else if (char === "]") selectorBracketDepth = Math.max(0, selectorBracketDepth - 1);
|
||
if (char === ";" && selectorBracketDepth === 0) {
|
||
selectorParts.push(partBuffer);
|
||
partBuffer = "";
|
||
} else {
|
||
partBuffer += char;
|
||
}
|
||
}
|
||
selectorParts.push(partBuffer);
|
||
for (const part of selectorParts) {
|
||
const eq = part.indexOf("=");
|
||
if (eq <= 0) continue;
|
||
const key = part.slice(0, eq).trim().toLowerCase();
|
||
const value = part.slice(eq + 1).trim();
|
||
if (!value) continue;
|
||
if (key === "window" || key === "win") selector.window = value;
|
||
else if (key === "name" || key === "n") selector.name = value;
|
||
else if (key === "id" || key === "automationid") selector.id = value;
|
||
else if (key === "type" || key === "controltype") {
|
||
selector.typeRaw = value;
|
||
selector.type = normalizeUiaControlType(value);
|
||
}
|
||
else if (key === "class" || key === "classname") selector.className = value;
|
||
else if (key === "screen") {
|
||
const nums = value.split(/[,，]/).map(item => Number(item.trim()));
|
||
if (nums.length !== 4 || nums.some(n => !Number.isFinite(n))) {
|
||
throw new RuntimeError("validation_error", "screen 坐标格式无效：" + value, { recommendation: "使用 screen=x,y,width,height" });
|
||
}
|
||
selector.screen = { x: nums[0], y: nums[1], w: nums[2], h: nums[3] };
|
||
}
|
||
else if (key === "rel") {
|
||
const nums = value.split(/[,،]/).map(item => Number(item.trim()));
|
||
if (nums.length !== 4 || nums.some(n => !Number.isFinite(n))) {
|
||
throw new RuntimeError("validation_error", "rel coordinate invalid: " + value, { recommendation: "rel=x,y,width,height" });
|
||
}
|
||
selector.rel = { x: nums[0], y: nums[1], w: nums[2], h: nums[3] };
|
||
}
|
||
else if (key === "process") selector.process = value;
|
||
else if (key === "winclass") selector.winClass = value;
|
||
else if (key === "enriched") selector.enriched = value === "1" || value.toLowerCase() === "true";
|
||
else if (key === "aitype") selector.aiType = value;
|
||
else if (key === "aitext") selector.aiText = value;
|
||
else if (key === "ai") selector.ai = value === "1" || value.toLowerCase() === "true";
|
||
else if (key === "index") {
|
||
const n = Number(value);
|
||
if (Number.isFinite(n) && n >= 1) selector.index = Math.floor(n);
|
||
}
|
||
else if (key === "path" || key === "ancestors" || key === "ancestorpath") {
|
||
selector.path = parseUiaSelectorPath(value);
|
||
}
|
||
}
|
||
const hasWindowAnchor = Boolean(selector.window || selector.winClass || selector.process);
|
||
if (!selector.name && !selector.id && !selector.className && !selector.type && !selector.screen && !selector.path && !(selector.rel && hasWindowAnchor)) {
|
||
throw new RuntimeError("validation_error", `桌面元素选择器缺少定位条件：${text}`, { recommendation: "至少提供 name= / id= / type= 之一" });
|
||
}
|
||
return selector;
|
||
}
|
||
|
||
const UIA_USING = [
|
||
"Add-Type -AssemblyName UIAutomationClient, UIAutomationTypes;",
|
||
"try { Add-Type -MemberDefinition '[DllImport(\"user32.dll\")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value);' -Name RpaDpi -Namespace WinRpaDpi | Out-Null; [WinRpaDpi.RpaDpi]::SetProcessDpiAwarenessContext([IntPtr](-4)) | Out-Null } catch {}"
|
||
].join("\n");
|
||
|
||
const FIND_HELPER = `
|
||
function Find-RpaWindow([string]$winName, [string]$winClass, [string]$processName) {
|
||
# 只读查找：绝不移动/还原/缩放窗口；还原、激活、移动是独立显式动作（Restore/Activate/Move-RpaWindow）
|
||
$root = [System.Windows.Automation.AutomationElement]::RootElement
|
||
if (-not $winName -and -not $winClass -and -not $processName) { return $root }
|
||
if ($winName) {
|
||
$cond = New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::NameProperty, $winName)
|
||
$win = $root.FindFirst([System.Windows.Automation.TreeScope]::Children, $cond)
|
||
if ($win) { return $win }
|
||
}
|
||
Add-RpaUser32
|
||
$script:hitHwnd = [IntPtr]::Zero
|
||
$cb = [Win32Rpa.RpaUser32Click+EnumProc]{ param($h, $l)
|
||
if (-not [Win32Rpa.RpaUser32Click]::IsWindowVisible($h)) { return $true }
|
||
$titleSb = New-Object System.Text.StringBuilder 512
|
||
[void][Win32Rpa.RpaUser32Click]::GetWindowTextW($h, $titleSb, 512)
|
||
$title = $titleSb.ToString()
|
||
if ($winName -and $title -notlike "*$winName*") { return $true }
|
||
$classSb = New-Object System.Text.StringBuilder 256
|
||
[void][Win32Rpa.RpaUser32Click]::GetClassNameW($h, $classSb, 256)
|
||
$className = $classSb.ToString()
|
||
if ($winClass -and $className -ne $winClass) { return $true }
|
||
if ($processName) {
|
||
$windowPid = [uint32]0
|
||
[void][Win32Rpa.RpaUser32Click]::GetWindowThreadProcessId($h, [ref]$windowPid)
|
||
try { $procName = (Get-Process -Id $windowPid -ErrorAction Stop).ProcessName } catch { return $true }
|
||
if (-not $procName.Equals($processName, [StringComparison]::OrdinalIgnoreCase)) { return $true }
|
||
}
|
||
$script:hitHwnd = $h
|
||
return $false
|
||
}
|
||
[void][Win32Rpa.RpaUser32Click]::EnumWindows($cb, [IntPtr]::Zero)
|
||
if ($script:hitHwnd -ne [IntPtr]::Zero) {
|
||
try { $win = [System.Windows.Automation.AutomationElement]::FromHandle($script:hitHwnd); if ($win) { return $win } } catch {}
|
||
}
|
||
$missing = (@($winName, $winClass, $processName) | Where-Object { $_ }) -join ","
|
||
throw "window_not_found:$missing"
|
||
}
|
||
function Restore-RpaWindow($win) {
|
||
# 显式动作：还原窗口（最小化/最大化状态恢复），不用于查询
|
||
Add-RpaUser32
|
||
$hwnd = [IntPtr]$win.Current.NativeWindowHandle
|
||
if ($hwnd -eq [IntPtr]::Zero) { throw "window_no_hwnd" }
|
||
[void][WinRpa.RpaUser32Click]::ShowWindow($hwnd, 9)
|
||
Start-Sleep -Milliseconds 300
|
||
return $hwnd
|
||
}
|
||
function Activate-RpaWindow($win) {
|
||
# 显式动作：把窗口带到前台
|
||
Add-RpaUser32
|
||
$hwnd = [IntPtr]$win.Current.NativeWindowHandle
|
||
if ($hwnd -eq [IntPtr]::Zero) { throw "window_no_hwnd" }
|
||
$fgHwnd = [WinRpa.RpaUser32Click]::GetForegroundWindow()
|
||
if ($fgHwnd -ne $hwnd) {
|
||
$fgPid = [uint32]0
|
||
$fgThread = [WinRpa.RpaUser32Click]::GetWindowThreadProcessId($fgHwnd, [ref]$fgPid)
|
||
$thisThread = [WinRpa.RpaUser32Click]::GetCurrentThreadId()
|
||
[void][WinRpa.RpaUser32Click]::AttachThreadInput($thisThread, $fgThread, $true)
|
||
[void][WinRpa.RpaUser32Click]::SetForegroundWindow($hwnd)
|
||
[void][WinRpa.RpaUser32Click]::AttachThreadInput($thisThread, $fgThread, $false)
|
||
}
|
||
Start-Sleep -Milliseconds 200
|
||
return $hwnd
|
||
}
|
||
function Move-RpaWindow($win, [int]$x, [int]$y, [int]$w, [int]$h) {
|
||
# 显式动作：移动/缩放窗口；w/h<=0 时使用安全默认尺寸
|
||
Add-RpaUser32
|
||
$hwnd = [IntPtr]$win.Current.NativeWindowHandle
|
||
if ($hwnd -eq [IntPtr]::Zero) { throw "window_no_hwnd" }
|
||
$sw = [WinRpa.RpaUser32Click]::GetSystemMetrics(0)
|
||
$sh = [WinRpa.RpaUser32Click]::GetSystemMetrics(1)
|
||
if ($w -le 0) { $w = [Math]::Min(1400, $sw - 120) }
|
||
if ($h -le 0) { $h = [Math]::Min(820, $sh - 120) }
|
||
[void][WinRpa.RpaUser32Click]::SetWindowPos($hwnd, [IntPtr]::Zero, $x, $y, $w, $h, 0x0040)
|
||
Start-Sleep -Milliseconds 300
|
||
return $hwnd
|
||
}
|
||
function Find-RpaElement($parent, [string]$name, [string]$id, [string]$type, [string]$className, [int]$index, [switch]$directChild) {
|
||
$conds = @()
|
||
if ($name) { $conds += New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::NameProperty, $name) }
|
||
if ($id) { $conds += New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::AutomationIdProperty, $id) }
|
||
if ($className) { $conds += New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::ClassNameProperty, $className) }
|
||
if ($type) {
|
||
# 不区分大小写匹配 .NET ControlType 字段，兼容 ComboBox/ListItem/DataGrid 等复合名称
|
||
try {
|
||
$ctField = [System.Windows.Automation.ControlType].GetFields([System.Reflection.BindingFlags]"Public,Static") | Where-Object { $_.Name -ieq $type } | Select-Object -First 1
|
||
if ($ctField) { $conds += New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::ControlTypeProperty, $ctField.GetValue($null)) }
|
||
} catch {}
|
||
}
|
||
if ($conds.Count -eq 0) { throw "no_condition" }
|
||
$final = if ($conds.Count -eq 1) { $conds[0] } else { New-Object System.Windows.Automation.AndCondition($conds) }
|
||
$scope = if ($directChild) { [System.Windows.Automation.TreeScope]::Children } else { [System.Windows.Automation.TreeScope]::Descendants }
|
||
$all = $parent.FindAll($scope, $final)
|
||
if ($all.Count -eq 0) { return $null }
|
||
$pick = [Math]::Min([Math]::Max($index, 1), $all.Count)
|
||
return $all[$pick - 1]
|
||
}
|
||
function Find-RpaElementByPath($parent, [string]$pathJsonBase64) {
|
||
# 逐段走直接子级，用稳定属性（id/type/class 优先）解析窗口→目标祖先路径
|
||
$segments = [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String($pathJsonBase64)) | ConvertFrom-Json
|
||
$cur = $parent
|
||
foreach ($seg in @($segments)) {
|
||
$ix = 1
|
||
try { if ($seg.index) { $ix = [int]$seg.index } } catch {}
|
||
$next = Find-RpaElement $cur $seg.name $seg.id $seg.type $seg.className $ix -directChild
|
||
if ($null -eq $next) { return $null }
|
||
$cur = $next
|
||
}
|
||
return $cur
|
||
}
|
||
function Add-RpaUser32 {
|
||
Add-Type -MemberDefinition '[DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y); [DllImport("user32.dll")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra); [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd); [DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); [DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr hWnd, int nCmdShow); [DllImport("user32.dll")] public static extern bool SetWindowPos(IntPtr hWnd, IntPtr after, int X, int Y, int cx, int cy, uint flags); [DllImport("user32.dll")] public static extern bool AttachThreadInput(uint idAttach, uint idAttachTo, bool fAttach); [DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint processId); [DllImport("kernel32.dll")] public static extern uint GetCurrentThreadId(); [DllImport("user32.dll")] public static extern int GetSystemMetrics(int nIndex); [DllImport("user32.dll")] public static extern bool EnumWindows(EnumProc cb, IntPtr lp); public delegate bool EnumProc(IntPtr hWnd, IntPtr lp); [DllImport("user32.dll", CharSet = CharSet.Unicode)] public static extern int GetWindowTextW(IntPtr hWnd, System.Text.StringBuilder sb, int max); [DllImport("user32.dll", CharSet = CharSet.Unicode)] public static extern int GetClassNameW(IntPtr hWnd, System.Text.StringBuilder sb, int max); [DllImport("user32.dll")] public static extern bool IsWindowVisible(IntPtr hWnd);' -Name RpaUser32Click -Namespace Win32Rpa | Out-Null
|
||
}
|
||
`;
|
||
|
||
const RESOLVE_HELPER = `
|
||
function Resolve-RpaScreenRect($s) {
|
||
$hasStable = [bool]($s.name -or $s.id -or $s.type -or $s.className)
|
||
$hasWindowAnchor = [bool]($s.window -or $s.winClass -or $s.process)
|
||
if ($s.enriched -and $hasStable) {
|
||
try {
|
||
$parent = Find-RpaWindow ([string]$s.window) ([string]$s.winClass) ([string]$s.process)
|
||
$el = Find-RpaElement $parent ([string]$s.name) ([string]$s.id) ([string]$s.type) ([string]$s.className) ([int]1)
|
||
if ($el) {
|
||
$r = $el.Current.BoundingRectangle
|
||
if ($r.Width -gt 0 -and $r.Height -gt 0) { return @{ x=[int]$r.X; y=[int]$r.Y; w=[int]$r.Width; h=[int]$r.Height; method='uia' } }
|
||
}
|
||
} catch {}
|
||
}
|
||
if ($s.rel -and $hasWindowAnchor) {
|
||
try {
|
||
$parent = Find-RpaWindow ([string]$s.window) ([string]$s.winClass) ([string]$s.process)
|
||
$r = $parent.Current.BoundingRectangle
|
||
return @{ x=[int]($r.X + $s.rel.x); y=[int]($r.Y + $s.rel.y); w=[int]$s.rel.w; h=[int]$s.rel.h; method='window_rel' }
|
||
} catch {}
|
||
}
|
||
if ($s.screen) { return @{ x=[int]$s.screen.x; y=[int]$s.screen.y; w=[int]$s.screen.w; h=[int]$s.screen.h; method='screen_fallback' } }
|
||
throw "selector_rect_not_found"
|
||
}
|
||
`;
|
||
|
||
interface UiaTarget {
|
||
selector: UiaSelector;
|
||
args: Record<string, unknown>;
|
||
node: { id: string; componentId: string };
|
||
}
|
||
|
||
function target(args: Record<string, unknown>, node: { id: string; componentId: string }): UiaTarget {
|
||
const raw = args.uiSelector ?? args.target ?? args.selector;
|
||
return { selector: parseUiaSelector(raw), args, node };
|
||
}
|
||
|
||
/** 生成“解析窗口 + 解析元素（路径优先，其次平铺属性）”的 PowerShell 语句，结果写入 $el */
|
||
function elementResolveLines(s: UiaSelector): string {
|
||
const parentLine = "$parent = Find-RpaWindow " + psString(s.window ?? "") + " " + psString(s.winClass ?? "") + " " + psString(s.process ?? "");
|
||
if (s.path?.length) {
|
||
const pathB64 = Buffer.from(JSON.stringify(s.path), "utf8").toString("base64");
|
||
return parentLine + "\n$el = Find-RpaElementByPath $parent '" + pathB64 + "'";
|
||
}
|
||
return parentLine
|
||
+ "\n$el = Find-RpaElement $parent " + psString(s.name ?? "")
|
||
+ " " + psString(s.id ?? "")
|
||
+ " " + psString(s.type ?? "")
|
||
+ " " + psString(s.className ?? "")
|
||
+ " " + s.index;
|
||
}
|
||
|
||
const SCREEN_USER32 = "Add-Type -MemberDefinition '[DllImport(\"user32.dll\")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value); [DllImport(\"user32.dll\")] public static extern bool SetCursorPos(int X, int Y); [DllImport(\"user32.dll\")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra);' -Name RpaScreenInput -Namespace RpaScreen | Out-Null; try { [RpaScreen.RpaScreenInput]::SetProcessDpiAwarenessContext([IntPtr](-4)) | Out-Null } catch {}";
|
||
|
||
function resolveScreenPreamble(t: UiaTarget): string {
|
||
const s = t.selector;
|
||
const screen = s.screen ? `@{x=${s.screen.x};y=${s.screen.y};w=${s.screen.w};h=${s.screen.h}}` : "$null";
|
||
const rel = s.rel ? `@{x=${s.rel.x};y=${s.rel.y};w=${s.rel.w};h=${s.rel.h}}` : "$null";
|
||
return `${UIA_USING}
|
||
${FIND_HELPER}
|
||
${RESOLVE_HELPER}
|
||
$s = @{ enriched=${s.enriched ? "$true" : "$false"}; window=${JSON.stringify(s.window ?? "")}; winClass=${JSON.stringify(s.winClass ?? "")}; process=${JSON.stringify(s.process ?? "")}; name=${JSON.stringify(s.name ?? "")}; id=${JSON.stringify(s.id ?? "")}; type=${JSON.stringify(s.type ?? "")}; className=${JSON.stringify(s.className ?? "")}; screen=${screen}; rel=${rel} }
|
||
$rect = Resolve-RpaScreenRect $s
|
||
$x = [int]($rect.x + $rect.w / 2)
|
||
$y = [int]($rect.y + $rect.h / 2)`;
|
||
}
|
||
|
||
function clickScreenScript(t: UiaTarget): string {
|
||
const clickway = String(t.args.clickway ?? "0");
|
||
const doubleClick = clickway === "1" || /double/i.test(clickway);
|
||
return `${resolveScreenPreamble(t)}
|
||
${SCREEN_USER32}
|
||
if (-not [RpaScreen.RpaScreenInput]::SetCursorPos($x, $y)) { throw "cursor_move_failed" }
|
||
Start-Sleep -Milliseconds 100
|
||
[RpaScreen.RpaScreenInput]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)
|
||
Start-Sleep -Milliseconds 40
|
||
[RpaScreen.RpaScreenInput]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)
|
||
if (${doubleClick}) {
|
||
Start-Sleep -Milliseconds 70
|
||
[RpaScreen.RpaScreenInput]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)
|
||
Start-Sleep -Milliseconds 40
|
||
[RpaScreen.RpaScreenInput]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)
|
||
}
|
||
Write-Output (@{ clicked = $true; method = $rect.method; resolvedMethod = $rect.method; x = $x; y = $y } | ConvertTo-Json -Compress)`;
|
||
}
|
||
|
||
function setValueScreenScript(t: UiaTarget, value: string): string {
|
||
return `${resolveScreenPreamble(t)}
|
||
${SCREEN_USER32}
|
||
Add-Type -AssemblyName System.Windows.Forms
|
||
[RpaScreen.RpaScreenInput]::SetCursorPos($x, $y) | Out-Null
|
||
Start-Sleep -Milliseconds 120
|
||
[RpaScreen.RpaScreenInput]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)
|
||
Start-Sleep -Milliseconds 40
|
||
[RpaScreen.RpaScreenInput]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)
|
||
Start-Sleep -Milliseconds 220
|
||
$saved = $null
|
||
try { $saved = [System.Windows.Forms.Clipboard]::GetText() } catch {}
|
||
try { [System.Windows.Forms.Clipboard]::SetText(${psString(value)}) } catch { throw "clipboard_failed" }
|
||
Start-Sleep -Milliseconds 80
|
||
[System.Windows.Forms.SendKeys]::SendWait('^a')
|
||
Start-Sleep -Milliseconds 60
|
||
[System.Windows.Forms.SendKeys]::SendWait('^v')
|
||
Start-Sleep -Milliseconds 280
|
||
try {
|
||
if ($saved) { [System.Windows.Forms.Clipboard]::SetText($saved) } else { [System.Windows.Forms.Clipboard]::Clear() }
|
||
} catch {}
|
||
Write-Output (@{ set = $true; method = $rect.method; resolvedMethod = $rect.method; x = $x; y = $y } | ConvertTo-Json -Compress)`;
|
||
}
|
||
|
||
function getValueScreenScript(t: UiaTarget): string {
|
||
return `${resolveScreenPreamble(t)}
|
||
Add-Type -AssemblyName WindowsBase
|
||
$pt = New-Object System.Windows.Point($x, $y)
|
||
$el = [System.Windows.Automation.AutomationElement]::FromPoint($pt)
|
||
if ($null -eq $el) { Write-Output 'JSON_NOT_FOUND'; exit 0 }
|
||
$value = $el.Current.Name
|
||
try { $vp = $el.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern); $value = $vp.Value } catch {}
|
||
Write-Output (@{ value = $value; name = $el.Current.Name; method = $rect.method } | ConvertTo-Json -Compress)`;
|
||
}
|
||
|
||
function screenExistsScript(t: UiaTarget): string {
|
||
return `${resolveScreenPreamble(t)}
|
||
Add-Type -AssemblyName WindowsBase
|
||
$found = $true
|
||
if ($rect.method -ne 'uia') {
|
||
try {
|
||
$pt = New-Object System.Windows.Point($x, $y)
|
||
$hit = [System.Windows.Automation.AutomationElement]::FromPoint($pt)
|
||
if ($null -eq $hit) { $found = $false }
|
||
else {
|
||
$hr = $hit.Current.BoundingRectangle
|
||
$inside = ($hr.Right -ge $rect.x) -and ($hr.Left -le ($rect.x + $rect.w)) -and ($hr.Bottom -ge $rect.y) -and ($hr.Top -le ($rect.y + $rect.h))
|
||
if (-not $inside) { $found = $false }
|
||
}
|
||
} catch { $found = $false }
|
||
}
|
||
Write-Output (@{ found = $found; method = $rect.method; x = $rect.x; y = $rect.y; w = $rect.w; h = $rect.h } | ConvertTo-Json -Compress)`;
|
||
}
|
||
|
||
async function screenElementExists(t: UiaTarget): Promise<boolean> {
|
||
try {
|
||
const result = await runScript(screenExistsScript(t), t);
|
||
return result?.found === true;
|
||
} catch {
|
||
return false;
|
||
}
|
||
}
|
||
|
||
function findScript(t: UiaTarget, timeoutMs: number, gapMs: number): string {
|
||
const s = t.selector;
|
||
return `${UIA_USING}
|
||
${FIND_HELPER}
|
||
$deadline = [DateTime]::UtcNow.AddMilliseconds(${Math.max(0, Math.round(timeoutMs))})
|
||
$found = $null
|
||
while ($null -eq $found) {
|
||
try {
|
||
${elementResolveLines(s)}
|
||
$found = $el
|
||
} catch {
|
||
$err = $_.Exception.Message
|
||
if ($err -like "window_not_found:*") { $found = $null }
|
||
}
|
||
if ($null -eq $found -and [DateTime]::UtcNow -ge $deadline) { break }
|
||
if ($null -eq $found) { Start-Sleep -Milliseconds ${Math.max(50, Math.round(gapMs))} }
|
||
}
|
||
if ($null -eq $found) { Write-Output "JSON_NOT_FOUND" } else {
|
||
$rect = $found.Current.BoundingRectangle
|
||
$info = @{ found = $true; name = $found.Current.Name; id = $found.Current.AutomationId; type = $found.Current.ControlType.ProgrammaticName; enabled = $found.Current.IsEnabled; x = [int]$rect.X; y = [int]$rect.Y; w = [int]$rect.Width; h = [int]$rect.Height }
|
||
Write-Output ($info | ConvertTo-Json -Compress)
|
||
}`;
|
||
}
|
||
|
||
async function findElement(t: UiaTarget, timeoutMs = 5000, gapMs = 200): Promise<Record<string, unknown> | null> {
|
||
const out = await powershell(findScript(t, timeoutMs, gapMs), Math.max(30000, timeoutMs + 30000));
|
||
const line = out.split(/\r?\n/).filter(Boolean).pop() || "";
|
||
if (line.includes("JSON_NOT_FOUND")) return null;
|
||
try {
|
||
return JSON.parse(line);
|
||
} catch {
|
||
throw new RuntimeError("desktop_command_failed", `桌面元素查找输出异常：${out.slice(0, 200)}`, { nodeId: t.node.id, componentId: t.node.componentId });
|
||
}
|
||
}
|
||
|
||
function clickScript(t: UiaTarget): string {
|
||
const s = t.selector;
|
||
const clickway = String(t.args.clickway ?? "0");
|
||
const doubleClick = clickway === "1" || /double/i.test(clickway);
|
||
return `${UIA_USING}
|
||
${FIND_HELPER}
|
||
$el = $null
|
||
try {
|
||
${elementResolveLines(s)}
|
||
} catch { $el = $null }
|
||
if ($null -eq $el) { Write-Output "JSON_NOT_FOUND"; exit 0 }
|
||
$rect = $el.Current.BoundingRectangle
|
||
Add-RpaUser32
|
||
$vsX = [WinRpa.RpaUser32Click]::GetSystemMetrics(76)
|
||
$vsY = [WinRpa.RpaUser32Click]::GetSystemMetrics(77)
|
||
$vsW = [WinRpa.RpaUser32Click]::GetSystemMetrics(78)
|
||
$vsH = [WinRpa.RpaUser32Click]::GetSystemMetrics(79)
|
||
$inVs = ($rect.X -ge $vsX) -and ($rect.Y -ge $vsY) -and (($rect.X + $rect.Width) -le ($vsX + $vsW)) -and (($rect.Y + $rect.Height) -le ($vsY + $vsH))
|
||
$offscreen = $el.Current.IsOffscreen -or $rect.Width -le 0 -or $rect.Height -le 0 -or (-not $inVs)
|
||
if ($offscreen) {
|
||
# 窗口可能被最小化或移出屏幕：还原并移回可视区后重取坐标
|
||
$walker0 = [System.Windows.Automation.TreeWalker]::ControlViewWalker
|
||
$anc0 = $el
|
||
$topHwnd = [IntPtr]::Zero
|
||
while ($null -ne $anc0) {
|
||
$h0 = $anc0.Current.NativeWindowHandle
|
||
if ($h0 -ne 0) { $topHwnd = [IntPtr]$h0; break }
|
||
$anc0 = $walker0.GetParent($anc0)
|
||
}
|
||
if ($topHwnd -ne [IntPtr]::Zero) {
|
||
[WinRpa.RpaUser32Click]::ShowWindow($topHwnd, 9) | Out-Null
|
||
Start-Sleep -Milliseconds 600
|
||
$sw3 = [WinRpa.RpaUser32Click]::GetSystemMetrics(0)
|
||
$sh3 = [WinRpa.RpaUser32Click]::GetSystemMetrics(1)
|
||
$w3 = [Math]::Min(1400, $sw3 - 120)
|
||
$h3 = [Math]::Min(820, $sh3 - 120)
|
||
[WinRpa.RpaUser32Click]::SetWindowPos($topHwnd, [IntPtr]::Zero, 60, 60, $w3, $h3, 0x0040) | Out-Null
|
||
Start-Sleep -Milliseconds 600
|
||
$rect = $el.Current.BoundingRectangle
|
||
$inVs = ($rect.X -ge $vsX) -and ($rect.Y -ge $vsY) -and (($rect.X + $rect.Width) -le ($vsX + $vsW)) -and (($rect.Y + $rect.Height) -le ($vsY + $vsH))
|
||
$offscreen = $el.Current.IsOffscreen -or $rect.Width -le 0 -or $rect.Height -le 0 -or (-not $inVs)
|
||
}
|
||
}
|
||
$method = "coordinate_click"
|
||
$ok = $false
|
||
if (-not $offscreen) {
|
||
Add-RpaUser32
|
||
$walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker
|
||
$ancestor = $el
|
||
$hwnd = [IntPtr]::Zero
|
||
while ($null -ne $ancestor) {
|
||
$h = $ancestor.Current.NativeWindowHandle
|
||
if ($h -ne 0) { $hwnd = [IntPtr]$h; break }
|
||
$ancestor = $walker.GetParent($ancestor)
|
||
}
|
||
if ($hwnd -ne [IntPtr]::Zero) {
|
||
$fgHwnd = [WinRpa.RpaUser32Click]::GetForegroundWindow()
|
||
if ($fgHwnd -ne $hwnd) {
|
||
$fgPid = [uint32]0
|
||
$fgThread = [WinRpa.RpaUser32Click]::GetWindowThreadProcessId($fgHwnd, [ref]$fgPid)
|
||
$thisThread = [WinRpa.RpaUser32Click]::GetCurrentThreadId()
|
||
[WinRpa.RpaUser32Click]::AttachThreadInput($thisThread, $fgThread, $true) | Out-Null
|
||
[WinRpa.RpaUser32Click]::SetForegroundWindow($hwnd) | Out-Null
|
||
[WinRpa.RpaUser32Click]::AttachThreadInput($thisThread, $fgThread, $false) | Out-Null
|
||
}
|
||
Start-Sleep -Milliseconds 250
|
||
}
|
||
$x = [int]($rect.X + $rect.Width / 2)
|
||
$y = [int]($rect.Y + $rect.Height / 2)
|
||
[WinRpa.RpaUser32Click]::SetCursorPos($x, $y) | Out-Null
|
||
Start-Sleep -Milliseconds 120
|
||
[WinRpa.RpaUser32Click]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)
|
||
Start-Sleep -Milliseconds 40
|
||
[WinRpa.RpaUser32Click]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)
|
||
if (${doubleClick}) {
|
||
Start-Sleep -Milliseconds 60
|
||
[WinRpa.RpaUser32Click]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)
|
||
Start-Sleep -Milliseconds 40
|
||
[WinRpa.RpaUser32Click]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)
|
||
}
|
||
$ok = $true
|
||
} else {
|
||
$method = "invoke_pattern"
|
||
foreach ($entry in @(@("Invoke", [System.Windows.Automation.InvokePattern]::Pattern), @("Toggle", [System.Windows.Automation.TogglePattern]::Pattern), @("Expand", [System.Windows.Automation.ExpandCollapsePattern]::Pattern), @("Select", [System.Windows.Automation.SelectionItemPattern]::Pattern))) {
|
||
try {
|
||
$p = $el.GetCurrentPattern($entry[1])
|
||
switch ($entry[0]) {
|
||
"Invoke" { $p.Invoke(); $ok = $true }
|
||
"Toggle" { $p.Toggle(); $ok = $true }
|
||
"Expand" { $p.Expand(); $ok = $true }
|
||
"Select" { $p.Select(); $ok = $true }
|
||
}
|
||
if ($ok) { break }
|
||
} catch {}
|
||
}
|
||
}
|
||
$rect2 = $el.Current.BoundingRectangle
|
||
Write-Output (@{ clicked = $ok; method = $method; name = $el.Current.Name; x = [int]$rect2.X; y = [int]$rect2.Y; w = [int]$rect2.Width; h = [int]$rect2.Height } | ConvertTo-Json -Compress)`;
|
||
}
|
||
export function setValueScript(t: UiaTarget, value: string): string {
|
||
const s = t.selector;
|
||
return `${UIA_USING}
|
||
${FIND_HELPER}
|
||
${elementResolveLines(s)}
|
||
if ($null -eq $el) { Write-Output "JSON_NOT_FOUND"; exit 0 }
|
||
$value = ${psString(value)}
|
||
$method = "value_pattern"
|
||
$ok = $false
|
||
try {
|
||
$vp = $el.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern)
|
||
$vp.SetValue($value)
|
||
$ok = $true
|
||
} catch {
|
||
$method = "clipboard_paste"
|
||
Add-Type -AssemblyName System.Windows.Forms
|
||
$saved = $null
|
||
try { $saved = [System.Windows.Forms.Clipboard]::GetText() } catch {}
|
||
try { $el.SetFocus() } catch {}
|
||
Start-Sleep -Milliseconds 150
|
||
try { [System.Windows.Forms.Clipboard]::SetText($value) } catch {}
|
||
Start-Sleep -Milliseconds 80
|
||
[System.Windows.Forms.SendKeys]::SendWait("^a")
|
||
Start-Sleep -Milliseconds 60
|
||
[System.Windows.Forms.SendKeys]::SendWait("^v")
|
||
Start-Sleep -Milliseconds 300
|
||
$ok = $true
|
||
try {
|
||
if ($saved) { [System.Windows.Forms.Clipboard]::SetText($saved) }
|
||
elseif ($saved -ne $null) { [System.Windows.Forms.Clipboard]::Clear() }
|
||
} catch {}
|
||
}
|
||
Write-Output (@{ set = $ok; method = $method; name = $el.Current.Name } | ConvertTo-Json -Compress)`;
|
||
}
|
||
function getValueScript(t: UiaTarget): string {
|
||
const s = t.selector;
|
||
return `${UIA_USING}
|
||
${FIND_HELPER}
|
||
${elementResolveLines(s)}
|
||
if ($null -eq $el) { Write-Output "JSON_NOT_FOUND"; exit 0 }
|
||
$value = $el.Current.Name
|
||
try { $vp = $el.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern); $value = $vp.Value } catch {}
|
||
Write-Output (@{ value = $value; name = $el.Current.Name } | ConvertTo-Json -Compress)`;
|
||
}
|
||
|
||
async function runScript(script: string, t: UiaTarget, timeoutMs = 60000): Promise<Record<string, unknown> | null> {
|
||
const out = await powershell(script, timeoutMs);
|
||
const line = out.split(/\r?\n/).filter(Boolean).pop() || "";
|
||
if (line.includes("JSON_NOT_FOUND")) return null;
|
||
try {
|
||
return JSON.parse(line);
|
||
} catch {
|
||
throw new RuntimeError("desktop_command_failed", `桌面元素操作输出异常：${out.slice(0, 200)}`, { nodeId: t.node.id, componentId: t.node.componentId });
|
||
}
|
||
}
|
||
|
||
function sleepMs(ms: number): Promise<void> {
|
||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||
}
|
||
|
||
async function runWithImplicitWait(
|
||
t: UiaTarget,
|
||
action: string,
|
||
script: string,
|
||
exec: (script: string, t: UiaTarget, timeoutMs?: number) => Promise<Record<string, unknown> | null>,
|
||
args: Record<string, unknown>
|
||
): Promise<Record<string, unknown>> {
|
||
const waitMs = Number(args.waitMs ?? args.wait_ms ?? 4000) || 4000;
|
||
const deadline = Date.now() + Math.max(0, waitMs);
|
||
for (;;) {
|
||
try {
|
||
return requireFound(await exec(script, t), t, action);
|
||
} catch (error) {
|
||
const code = (error as { code?: string }).code;
|
||
if (code !== "desktop_element_not_found" || Date.now() >= deadline) throw error;
|
||
await sleepMs(400);
|
||
}
|
||
}
|
||
}
|
||
function requireFound(result: Record<string, unknown> | null, t: UiaTarget, action: string): Record<string, unknown> {
|
||
if (!result) {
|
||
const s = t.selector;
|
||
throw new RuntimeError("desktop_element_not_found", `桌面元素未找到（${action}）：${JSON.stringify({ window: s.window, name: s.name, id: s.id, type: s.type, className: s.className, index: s.index })}。若指定了窗口，请确认应用已启动且主窗口未最小化到托盘`, { nodeId: t.node.id, componentId: t.node.componentId });
|
||
}
|
||
return result;
|
||
}
|
||
|
||
const clickGround: CapabilityHandler = async (_ctx, node, args) => {
|
||
const t = target(args, node);
|
||
const result = t.selector.screen
|
||
? await runScript(clickScreenScript(t), t)
|
||
: await runWithImplicitWait(t, "click", clickScript(t), runScript, args);
|
||
// 元素定位到了但实际没点成（如拾取到全屏透明覆盖层：IsOffscreen=true 且无 Invoke/Toggle/Expand/Select 模式）。
|
||
// 这种"假成功"会掩盖问题，必须按错误契约抛出，让运行记录如实标红。
|
||
if (result && result.clicked === false) {
|
||
const s = t.selector;
|
||
throw new RuntimeError(
|
||
"desktop_error",
|
||
`元素点击未生效：已定位到目标（window=${s.window ?? ""}, name=${s.name ?? ""}, type=${s.type ?? ""}），但该元素不可见点击且不支持任何可调用模式`,
|
||
{
|
||
recommendation: "重新用「桌面拾取」选择真实可见的目标控件；若拾取结果落在全屏透明覆盖层上（如 NVIDIA 游戏内覆盖层、录屏悬浮条），先关闭覆盖层再拾取",
|
||
nodeId: node.id,
|
||
componentId: node.componentId,
|
||
detail: JSON.stringify(result)
|
||
}
|
||
);
|
||
}
|
||
return result;
|
||
};
|
||
|
||
const setValueGround: CapabilityHandler = async (_ctx, node, args) => {
|
||
const t = target(args, node);
|
||
const value = String(args.value ?? args.text ?? "");
|
||
if (t.selector.screen) return runScript(setValueScreenScript(t, value), t);
|
||
return runWithImplicitWait(t, "setValue", setValueScript(t, value), runScript, args);
|
||
};
|
||
|
||
const existsGround: CapabilityHandler = async (_ctx, node, args) => {
|
||
const t = target(args, node);
|
||
if (t.selector.screen) return await screenElementExists(t);
|
||
const found = await findElement(t, 1000, 150);
|
||
return found !== null;
|
||
};
|
||
|
||
const getValueGround: CapabilityHandler = async (_ctx, node, args) => {
|
||
const t = target(args, node);
|
||
if (t.selector.screen) {
|
||
const result = await runScript(getValueScreenScript(t), t);
|
||
return requireFound(result, t, "getValue").value ?? "";
|
||
}
|
||
const result = await runWithImplicitWait(t, "getValue", getValueScript(t), runScript, args);
|
||
return result.value ?? "";
|
||
};
|
||
|
||
const waitElementGround: CapabilityHandler = async (_ctx, node, args) => {
|
||
const t = target(args, node);
|
||
if (t.selector.screen) {
|
||
const timeout = Number(args.timeout ?? args.timeoutMs ?? 5000) || 5000;
|
||
const gap = Number(args.time_gap ?? args.gapMs ?? 200) || 200;
|
||
const deadline = Date.now() + timeout;
|
||
do {
|
||
if (await screenElementExists(t)) return true;
|
||
if (Date.now() >= deadline) break;
|
||
await new Promise(resolve => setTimeout(resolve, gap));
|
||
} while (Date.now() < deadline);
|
||
return false;
|
||
}
|
||
const timeout = Number(args.timeout ?? args.timeoutMs ?? 5000) || 5000;
|
||
const gap = Number(args.time_gap ?? args.gapMs ?? 200) || 200;
|
||
const found = await findElement(t, timeout, gap);
|
||
return found !== null;
|
||
};
|
||
|
||
register("Element_ClickGround", clickGround);
|
||
register("Element_SetValueGround", setValueGround);
|
||
register("Element_ExistsGround", existsGround);
|
||
register("Element_GetValueGround", getValueGround);
|
||
const restoreWindowGround: CapabilityHandler = async (_ctx, node, args) => {
|
||
const t = target(args, node);
|
||
const lines = ["$hwnd = Restore-RpaWindow $el"];
|
||
if ([args.x, args.y, args.width, args.height].some((value) => value !== undefined && value !== null && String(value).trim() !== "")) {
|
||
const x = Math.round(Number(args.x ?? 60)) || 0;
|
||
const y = Math.round(Number(args.y ?? 60)) || 0;
|
||
const width = Math.round(Number(args.width ?? 0)) || 0;
|
||
const height = Math.round(Number(args.height ?? 0)) || 0;
|
||
lines.push("$hwnd = Move-RpaWindow $el " + x + " " + y + " " + width + " " + height);
|
||
}
|
||
return runWithImplicitWait(t, "restoreWindow", propertyValue([...lines, "Write-Output (@{ hwnd = [string]$hwnd; name = $el.Current.Name } | ConvertTo-Json -Compress)"], t), runScript, args);
|
||
};
|
||
|
||
const activateWindowGround: CapabilityHandler = async (_ctx, node, args) => {
|
||
const t = target(args, node);
|
||
return runWithImplicitWait(t, "activateWindow", propertyValue([
|
||
"$hwnd = Activate-RpaWindow $el",
|
||
"Write-Output (@{ hwnd = [string]$hwnd; name = $el.Current.Name } | ConvertTo-Json -Compress)"
|
||
], t), runScript, args);
|
||
};
|
||
|
||
const moveWindowGround: CapabilityHandler = async (_ctx, node, args) => {
|
||
const t = target(args, node);
|
||
const x = Math.round(Number(args.x ?? 60)) || 0;
|
||
const y = Math.round(Number(args.y ?? 60)) || 0;
|
||
const width = Math.round(Number(args.width ?? 0)) || 0;
|
||
const height = Math.round(Number(args.height ?? 0)) || 0;
|
||
return runWithImplicitWait(t, "moveWindow", propertyValue([
|
||
"$hwnd = Move-RpaWindow $el " + x + " " + y + " " + width + " " + height,
|
||
"Write-Output (@{ hwnd = [string]$hwnd; name = $el.Current.Name } | ConvertTo-Json -Compress)"
|
||
], t), runScript, args);
|
||
};
|
||
|
||
register("Element_WaitElementGround", waitElementGround);
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
function outputPath(args: Record<string, unknown>, extension = "png"): string {
|
||
const raw = args.filePath ?? args.folder ?? args.path;
|
||
const folder = path.resolve(String(raw && String(raw).trim() ? raw : process.cwd()));
|
||
const rawName = String(args.filename ?? args.fileName ?? args.name ?? `element-${Date.now()}.${extension}`)
|
||
.replace(/\$\{randomString\}/g, Math.random().toString(36).slice(2, 10));
|
||
const name = /\.[A-Za-z0-9]{2,8}$/.test(rawName) ? rawName : `${rawName}.${extension}`;
|
||
return path.join(folder, name);
|
||
}
|
||
|
||
function propertyValue(scriptLines: string[], t: UiaTarget): string {
|
||
const s = t.selector;
|
||
return `${UIA_USING}
|
||
${FIND_HELPER}
|
||
$el = $null
|
||
try {
|
||
${elementResolveLines(s)}
|
||
} catch { $el = $null }
|
||
if ($null -eq $el) { Write-Output "JSON_NOT_FOUND"; exit 0 }
|
||
${scriptLines.join("\n")}`;
|
||
}
|
||
|
||
const getCheckGround: CapabilityHandler = async (_ctx, node, args) => {
|
||
const t = target(args, node);
|
||
const result = await runWithImplicitWait(t, "getCheck", propertyValue([
|
||
"$checked = $false",
|
||
"try { $checked = $el.GetCurrentPattern([System.Windows.Automation.TogglePattern]::Pattern).Current.ToggleState -eq [System.Windows.Automation.ToggleState]::On } catch {}",
|
||
"Write-Output (@{ value = [bool]$checked; name = $el.Current.Name; enabled = [bool]$el.Current.IsEnabled } | ConvertTo-Json -Compress)"
|
||
], t), runScript, args);
|
||
return result.value === true;
|
||
};
|
||
|
||
const setCheckGround: CapabilityHandler = async (_ctx, node, args) => {
|
||
const t = target(args, node);
|
||
const desired = ["1", "true", "yes", "on"].includes(String(args.isChecked ?? "1").toLowerCase());
|
||
return runWithImplicitWait(t, "setCheck", propertyValue([
|
||
`$desired = ${desired ? "$true" : "$false"}`,
|
||
"$current = $false; $hasToggle = $false",
|
||
"try { $pattern = $el.GetCurrentPattern([System.Windows.Automation.TogglePattern]::Pattern); $current = $pattern.Current.ToggleState -eq [System.Windows.Automation.ToggleState]::On; $hasToggle = $true } catch {}",
|
||
"$method = if ($hasToggle) { 'toggle_pattern' } else { 'coordinate_click' }",
|
||
"if (-not $hasToggle -or $current -ne $desired) {",
|
||
" if ($hasToggle) { $pattern.Toggle() | Out-Null }",
|
||
" else {",
|
||
" $rect = $el.Current.BoundingRectangle; Add-RpaUser32",
|
||
" [Win32Rpa.RpaUser32Click]::SetCursorPos([int]($rect.X + $rect.Width / 2), [int]($rect.Y + $rect.Height / 2)) | Out-Null",
|
||
" Start-Sleep -Milliseconds 100; [Win32Rpa.RpaUser32Click]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)",
|
||
" Start-Sleep -Milliseconds 50; [Win32Rpa.RpaUser32Click]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)",
|
||
" }",
|
||
"}",
|
||
"$after = $false; try { $after = $el.GetCurrentPattern([System.Windows.Automation.TogglePattern]::Pattern).Current.ToggleState -eq [System.Windows.Automation.ToggleState]::On } catch {}",
|
||
"Write-Output (@{ set = ($after -eq $desired); checked = [bool]$after; method = $method; name = $el.Current.Name } | ConvertTo-Json -Compress)"
|
||
], t), runScript, args);
|
||
};
|
||
|
||
const regionGround: CapabilityHandler = async (_ctx, node, args) => {
|
||
const t = target(args, node);
|
||
const relative = String(args.position ?? "3") === "2";
|
||
const result = await runWithImplicitWait(t, "getRegion", propertyValue([
|
||
"$rect = $el.Current.BoundingRectangle; $ox = 0; $oy = 0",
|
||
relative ? [
|
||
"try {",
|
||
" Add-RpaUser32",
|
||
' Add-Type -MemberDefinition \'[DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr hWnd, out RECT rect); public struct RECT { public int Left; public int Top; public int Right; public int Bottom; }\' -Name RpaWindowRect -Namespace Win32RpaRect | Out-Null',
|
||
" $walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker; $ancestor = $el",
|
||
" while ($null -ne $ancestor) {",
|
||
" $hwnd = $ancestor.Current.NativeWindowHandle",
|
||
" if ($hwnd -ne 0) { $wr = New-Object Win32RpaRect.RpaWindowRect+RECT; [void][Win32RpaRect.RpaWindowRect]::GetWindowRect([IntPtr]$hwnd, [ref]$wr); $ox = $wr.Left; $oy = $wr.Top; break }",
|
||
" $ancestor = $walker.GetParent($ancestor)",
|
||
" }",
|
||
"} catch {}",
|
||
].join("\n") : "",
|
||
"Write-Output (@{ x = [int]($rect.X - $ox); y = [int]($rect.Y - $oy); width = [int]$rect.Width; height = [int]$rect.Height } | ConvertTo-Json -Compress)"
|
||
], t), runScript, args);
|
||
return result;
|
||
};
|
||
|
||
function getSelectScript(t: UiaTarget): string {
|
||
const byText = String(t.args.choose ?? "1") !== "2";
|
||
return propertyValue([
|
||
"$result = @(); $method = 'selection_pattern'",
|
||
"try { $sel = $el.GetCurrentPattern([System.Windows.Automation.SelectionPattern]::Pattern); foreach ($item in $sel.Current.GetSelection()) { $value = $item.Current.Name; if (-not " + (byText ? "$true" : "$false") + ") { $value = $item.Current.AutomationId }; $result += @{ text = $item.Current.Name; value = $item.Current.AutomationId; selected = $true } } } catch {}",
|
||
"if ($result.Count -eq 0) { try { $value = $el.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern).Value } catch { $value = $el.Current.Name }; $result += @{ text = [string]$value; value = [string]$value; selected = $true }; $method = 'value_pattern' }",
|
||
"Write-Output (@{ result = $result; method = $method; name = $el.Current.Name } | ConvertTo-Json -Compress -Depth 3)"
|
||
], t);
|
||
}
|
||
|
||
const getSelectGround: CapabilityHandler = async (_ctx, node, args) => {
|
||
const t = target(args, node);
|
||
const result = await runWithImplicitWait(t, "getSelect", getSelectScript(t), runScript, args);
|
||
const items = Array.isArray(result.result) ? result.result : [];
|
||
return items.length === 1 ? items[0] : items;
|
||
};
|
||
|
||
function setSelectScript(t: UiaTarget, values: string[]): string {
|
||
const encoded = Buffer.from(JSON.stringify(values), "utf16le").toString("base64");
|
||
const choose = String(t.args.choose ?? "1");
|
||
return propertyValue([
|
||
`$desired = [Text.Encoding]::Unicode.GetString([Convert]::FromBase64String('${encoded}')) | ConvertFrom-Json`,
|
||
"$items = @($el.FindAll([System.Windows.Automation.TreeScope]::Descendants, [System.Windows.Automation.Condition]::TrueCondition) | Where-Object { $_.Current.ControlType.ProgrammaticName -match 'ListItem|ComboBoxItem' })",
|
||
"if ($items.Count -eq 0) { try { $el.Expand() } catch {}; Start-Sleep -Milliseconds 250; $items = @($el.FindAll([System.Windows.Automation.TreeScope]::Descendants, [System.Windows.Automation.Condition]::TrueCondition) | Where-Object { $_.Current.ControlType.ProgrammaticName -match 'ListItem|ComboBoxItem' }) }",
|
||
"$targets = @(); $method = 'selection_item_pattern'",
|
||
"foreach ($wanted in @($desired)) {",
|
||
` if ('${choose}' -eq '0') { $ix = [int]$wanted - 1; if ($ix -ge 0 -and $ix -lt $items.Count) { $targets += $items[$ix] } }`,
|
||
` elseif ('${choose}' -eq '2') { $targets += @($items | Where-Object { $_.Current.AutomationId -eq [string]$wanted }) }`,
|
||
" else { $targets += @($items | Where-Object { $_.Current.Name -eq [string]$wanted }) }",
|
||
"}",
|
||
"$ok = $targets.Count -gt 0",
|
||
"foreach ($item in $targets) { try { $item.GetCurrentPattern([System.Windows.Automation.SelectionItemPattern]::Pattern).Select() } catch { try { $item.Invoke() } catch { $ok = $false } } }",
|
||
"try { $el.Collapse() } catch {}",
|
||
"Write-Output (@{ set = [bool]$ok; matched = $targets.Count; available = $items.Count; method = $method; name = $el.Current.Name } | ConvertTo-Json -Compress)"
|
||
], t);
|
||
}
|
||
|
||
const setSelectGround: CapabilityHandler = async (_ctx, node, args) => {
|
||
const t = target(args, node);
|
||
const raw = args.itemsToSelect ?? args.value ?? args.text;
|
||
let values: string[] = [];
|
||
if (Array.isArray(raw)) values = raw.map((value) => String(value));
|
||
else if (raw !== undefined && raw !== null && String(raw).trim() !== "") {
|
||
try { const parsed = JSON.parse(String(raw)); values = Array.isArray(parsed) ? parsed.map((value) => String(value)) : [String(parsed)]; }
|
||
catch { values = [String(raw)]; }
|
||
}
|
||
if (!values.length) throw new RuntimeError("validation_error", "下拉框选择项为空", { nodeId: node.id, componentId: node.componentId });
|
||
return runWithImplicitWait(t, "setSelect", setSelectScript(t, values), runScript, args);
|
||
};
|
||
|
||
function attrScript(t: UiaTarget, attrName: string, attrValue = ""): string {
|
||
const name = attrName.toLowerCase();
|
||
if (name === "value" || name === "text") {
|
||
return propertyValue([
|
||
"try { $value = $el.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern).Value } catch { $value = $el.Current.Name }",
|
||
"Write-Output (@{ value = [string]$value; name = $el.Current.Name } | ConvertTo-Json -Compress)"
|
||
], t);
|
||
}
|
||
if (!attrValue) {
|
||
return propertyValue([
|
||
"$p = @{ name = $el.Current.Name; id = $el.Current.AutomationId; class = $el.Current.ClassName; type = $el.Current.ControlType.ProgrammaticName; enabled = $el.Current.IsEnabled; help = $el.Current.HelpText; value = $el.Current.Name }",
|
||
`$key = ${psString(name || "name")}`,
|
||
"$result = if ($p.Contains($key)) { $p[$key] } else { $p.name }",
|
||
"Write-Output (@{ value = [string]$result; properties = $p; name = $el.Current.Name } | ConvertTo-Json -Compress)"
|
||
], t);
|
||
}
|
||
if (name !== "value" && name !== "text") throw new RuntimeError("validation_error", `桌面 UIA 不支持设置属性：${attrName}；可设置 value/text`, { nodeId: t.node.id, componentId: t.node.componentId });
|
||
return propertyValue([
|
||
"$ok = $false; $method = 'value_pattern'",
|
||
`try { $el.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern).SetValue(${psString(attrValue)}); $ok = $true } catch {}`,
|
||
"if (-not $ok) { $method = 'keyboard'; Add-Type -AssemblyName System.Windows.Forms; $el.SetFocus() | Out-Null; Start-Sleep -Milliseconds 100; [System.Windows.Forms.SendKeys]::SendWait('^a'); [System.Windows.Forms.SendKeys]::SendWait('^v'); Start-Sleep -Milliseconds 100; $ok = $true }",
|
||
"Write-Output (@{ set = $ok; method = $method; name = $el.Current.Name } | ConvertTo-Json -Compress)"
|
||
], t);
|
||
}
|
||
|
||
const getAttributeGround: CapabilityHandler = async (_ctx, node, args) => {
|
||
const t = target(args, node);
|
||
const attrName = String(args.attrName ?? args.attribute ?? "name");
|
||
const result = await runWithImplicitWait(t, "getAttribute", attrScript(t, attrName), runScript, args);
|
||
return result.value ?? "";
|
||
};
|
||
|
||
const setAttributeGround: CapabilityHandler = async (_ctx, node, args) => {
|
||
const t = target(args, node);
|
||
const attrName = String(args.attrName ?? args.attribute ?? "value");
|
||
const attrValue = String(args.attrValue ?? args.value ?? "");
|
||
return runWithImplicitWait(t, "setAttribute", attrScript(t, attrName, attrValue), runScript, args);
|
||
};
|
||
|
||
const getButtonEnableGround: CapabilityHandler = async (_ctx, node, args) => {
|
||
const t = target(args, node);
|
||
const result = await runWithImplicitWait(t, "getButtonEnable", propertyValue(["Write-Output (@{ value = [bool]$el.Current.IsEnabled; name = $el.Current.Name } | ConvertTo-Json -Compress)"], t), runScript, args);
|
||
return result.value === true;
|
||
};
|
||
|
||
const shotGround: CapabilityHandler = async (_ctx, node, args) => {
|
||
const t = target(args, node);
|
||
const output = outputPath(args);
|
||
fs.mkdirSync(path.dirname(output), { recursive: true });
|
||
const result = await runWithImplicitWait(t, "shot", propertyValue([
|
||
"$rect = $el.Current.BoundingRectangle; if ($rect.Width -le 0 -or $rect.Height -le 0) { throw 'element_not_visible' }",
|
||
"Add-Type -AssemblyName System.Drawing | Out-Null",
|
||
"$bitmap = New-Object System.Drawing.Bitmap([int]$rect.Width, [int]$rect.Height)",
|
||
"$graphics = [System.Drawing.Graphics]::FromImage($bitmap)",
|
||
"$graphics.CopyFromScreen([int]$rect.X, [int]$rect.Y, 0, 0, $bitmap.Size); $graphics.Dispose()",
|
||
`$bitmap.Save(${psString(output)}, [System.Drawing.Imaging.ImageFormat]::Png); $bitmap.Dispose()`,
|
||
`Write-Output (@{ value = ${psString(output)}; x = [int]$rect.X; y = [int]$rect.Y; width = [int]$rect.Width; height = [int]$rect.Height } | ConvertTo-Json -Compress)`
|
||
], t), runScript, args);
|
||
return result.value ?? output;
|
||
};
|
||
|
||
const mouseToGround: CapabilityHandler = async (_ctx, node, args) => {
|
||
const t = target(args, node);
|
||
return runWithImplicitWait(t, "mouseTo", propertyValue([
|
||
"$rect = $el.Current.BoundingRectangle; Add-RpaUser32",
|
||
"$x = [int]($rect.X + $rect.Width / 2); $y = [int]($rect.Y + $rect.Height / 2)",
|
||
"[Win32Rpa.RpaUser32Click]::SetCursorPos($x, $y) | Out-Null",
|
||
"Write-Output (@{ x = $x; y = $y; name = $el.Current.Name } | ConvertTo-Json -Compress)"
|
||
], t), runScript, args);
|
||
};
|
||
|
||
const scrollIntoViewGround: CapabilityHandler = async (_ctx, node, args) => {
|
||
const t = target(args, node);
|
||
return runWithImplicitWait(t, "scrollIntoView", propertyValue([
|
||
"$ok = $false; $method = 'scroll_item_pattern'",
|
||
"try { $el.GetCurrentPattern([System.Windows.Automation.ScrollItemPattern]::Pattern).ScrollIntoView(); $ok = $true } catch {}",
|
||
"if (-not $ok) { $method = 'focus'; try { $el.SetFocus() | Out-Null; $ok = $true } catch {} }",
|
||
"Write-Output (@{ scrolled = $ok; method = $method; name = $el.Current.Name } | ConvertTo-Json -Compress)"
|
||
], t), runScript, args);
|
||
};
|
||
|
||
function childrenGroundHandler(single: boolean): CapabilityHandler {
|
||
return async (_ctx, node, args) => {
|
||
const t = target(args, node);
|
||
const num = Math.max(0, Math.floor(Number(args.num ?? 0) || 0));
|
||
const result = await runWithImplicitWait(t, "getChildren", propertyValue([
|
||
"$walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker; $child = $walker.GetFirstChild($el); $children = @()",
|
||
"while ($null -ne $child) {",
|
||
" $r = $child.Current.BoundingRectangle",
|
||
" $children += @{ name = $child.Current.Name; id = $child.Current.AutomationId; type = $child.Current.ControlType.ProgrammaticName; className = $child.Current.ClassName; enabled = [bool]$child.Current.IsEnabled; x = [int]$r.X; y = [int]$r.Y; width = [int]$r.Width; height = [int]$r.Height }",
|
||
" $child = $walker.GetNextSibling($child)",
|
||
"}",
|
||
single ? `$n = ${num}; if ($n -ge 1 -and $n -le $children.Count) { $children = @($children[$n - 1]) }` : "",
|
||
"Write-Output (@{ result = $children; count = $children.Count } | ConvertTo-Json -Compress -Depth 4)"
|
||
], t), runScript, args);
|
||
return result.result ?? [];
|
||
};
|
||
}
|
||
|
||
register("Element_GetCheckGround", getCheckGround);
|
||
register("Element_SetCheckGround", setCheckGround);
|
||
register("Element_RegionGround", regionGround);
|
||
register("Element_GetSelectGround", getSelectGround);
|
||
register("Element_SetSelectGround", setSelectGround);
|
||
register("Element_GetAttributeGround", getAttributeGround);
|
||
register("Element_SetAttributeGround", setAttributeGround);
|
||
register("Element_GetButtonEnableGround", getButtonEnableGround);
|
||
register("Element_ShotGround", shotGround);
|
||
register("Element_MouseToGround", mouseToGround);
|
||
register("Element_ScrollIntoViewGround", scrollIntoViewGround);
|
||
register("Element_ChildrenGround", childrenGroundHandler(false));
|
||
register("Element_DesktopChildren", childrenGroundHandler(true));
|
||
register("Element_RestoreWindowGround", restoreWindowGround);
|
||
register("Element_ActivateWindowGround", activateWindowGround);
|
||
register("Element_MoveWindowGround", moveWindowGround);
|
||
|
||
/** ElementDriver 的 PowerShell 实现（任务 3.3，决策点 6）：ground 原语委托本模块内部函数。
|
||
* handler 层面向 driver 接口的收口放到阶段 5/6 按需进行，避免大范围重写引入行为回归。 */
|
||
setElementDriverImplementation({
|
||
parseSelector: (raw) => parseUiaSelector(raw),
|
||
async find(selector, options) {
|
||
return findElement({ selector, args: {}, node: { id: "driver", componentId: "driver" } }, options?.timeoutMs ?? 5000, options?.gapMs ?? 200);
|
||
},
|
||
async exists(selector) {
|
||
return (await findElement({ selector, args: {}, node: { id: "driver", componentId: "driver" } }, 2000, 200)) !== null;
|
||
},
|
||
async click(selector, options) {
|
||
const t = { selector, args: {}, node: { id: "driver", componentId: "driver" } };
|
||
await runScript(clickScript(t), t, options?.timeoutMs ?? 60000);
|
||
},
|
||
async setValue(selector, value) {
|
||
const t = { selector, args: {}, node: { id: "driver", componentId: "driver" } };
|
||
await runScript(setValueScript(t, value), t);
|
||
},
|
||
async getValue(selector) {
|
||
const t = { selector, args: {}, node: { id: "driver", componentId: "driver" } };
|
||
return runScript(getValueScript(t), t);
|
||
}
|
||
});
|