feat: stabilize AI-picked desktop elements with UIA enrichment and window-relative selectors
This commit is contained in:
yu
parent
e5f64c21e2
commit
d7aff54bc6
@ -64,6 +64,18 @@ describe("desktop UIA ground capabilities", () => {
|
||||
expect(s.window).toBe("MyApp");
|
||||
});
|
||||
|
||||
it("parses enriched AI relocation metadata", () => {
|
||||
const selector = parseUiaSelector(
|
||||
"screen=100,200,320,80;window=App;winClass=Chrome_WidgetWin_1;process=chrome;rel=20,30,320,80;name=login;id=loginButton;type=Button;class=Button;aiType=button;aiText=login;enriched=1;ai=1"
|
||||
);
|
||||
expect(selector.screen).toEqual({ x: 100, y: 200, w: 320, h: 80 });
|
||||
expect(selector.rel).toEqual({ x: 20, y: 30, w: 320, h: 80 });
|
||||
expect(selector.window).toBe("App");
|
||||
expect(selector.winClass).toBe("Chrome_WidgetWin_1");
|
||||
expect(selector.process).toBe("chrome");
|
||||
expect(selector).toMatchObject({ name: "login", id: "loginButton", type: "Button", className: "Button", aiType: "button", aiText: "login", enriched: true, ai: true });
|
||||
});
|
||||
|
||||
it("rejects invalid selector", () => {
|
||||
expect(() => parseUiaSelector("")).toThrow();
|
||||
expect(() => parseUiaSelector("screen=1,2,3")).toThrow();
|
||||
@ -94,7 +106,7 @@ describe("desktop UIA selector compatibility", () => {
|
||||
const context = { log: vi.fn() } as any;
|
||||
const node = { id: "n1", componentId: "Element_ClickGround" } as any;
|
||||
|
||||
it("dispatches screen selectors to coordinate execution instead of UIA lookup", async () => {
|
||||
it("resolves screen selectors through UIA/window before coordinate fallback", async () => {
|
||||
const args = {
|
||||
selector: "screen=100,200,320,80;ai=1",
|
||||
clickway: "1"
|
||||
@ -102,8 +114,10 @@ describe("desktop UIA selector compatibility", () => {
|
||||
await expect(getHandler("Element_ClickGround")!(context, node, args, {} as any)).resolves.toEqual({ clicked: true, value: "cached" });
|
||||
const script = String((powershellMock.mock.calls as unknown as Array<[string]>)[0]?.[0] ?? "");
|
||||
expect(script).toContain("RpaScreenInput");
|
||||
expect(script).toContain("SetCursorPos(260, 240)");
|
||||
expect(script).not.toContain("Find-RpaWindow");
|
||||
expect(script).toContain("$x = [int]($rect.x + $rect.w / 2)");
|
||||
expect(script).toContain("SetCursorPos($x, $y)");
|
||||
expect(script).toContain("Resolve-RpaScreenRect");
|
||||
expect(script).toContain("Find-RpaWindow");
|
||||
});
|
||||
|
||||
it("dispatches screen setValue to coordinate paste", async () => {
|
||||
@ -112,18 +126,22 @@ describe("desktop UIA selector compatibility", () => {
|
||||
value: "hello"
|
||||
}, {} as any);
|
||||
const script = String((powershellMock.mock.calls as unknown as Array<[string]>)[0]?.[0] ?? "");
|
||||
expect(script).toContain("screen_coordinate_paste");
|
||||
expect(script).toContain("resolvedMethod = $rect.method");
|
||||
expect(script).toContain("hello");
|
||||
expect(script).not.toContain("Find-RpaWindow");
|
||||
expect(script).toContain("Find-RpaWindow");
|
||||
});
|
||||
|
||||
it("treats a picked screen rectangle as present for exists/wait", async () => {
|
||||
it("validates a picked screen rectangle for exists/wait", async () => {
|
||||
powershellMock.mockImplementation(async () => '{"found":true,"method":"window_rel"}');
|
||||
await expect(getHandler("Element_ExistsGround")!(context, node, {
|
||||
selector: "screen=1,2,3,4"
|
||||
}, {} as any)).resolves.toBe(true);
|
||||
await expect(getHandler("Element_WaitElementGround")!(context, node, {
|
||||
selector: "screen=1,2,3,4"
|
||||
}, {} as any)).resolves.toBe(true);
|
||||
expect(powershellMock).not.toHaveBeenCalled();
|
||||
expect(powershellMock).toHaveBeenCalled();
|
||||
const script = String((powershellMock.mock.calls as unknown as Array<[string]>).at(-1)?.[0] ?? "");
|
||||
expect(script).toContain("Resolve-RpaScreenRect");
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@ -14,6 +14,14 @@ interface UiaSelector {
|
||||
typeRaw?: string;
|
||||
className?: string;
|
||||
screen?: { x: number; y: number; w: number; h: number };
|
||||
/** Relative physical rect inside the top-level window. */
|
||||
rel?: { x: number; y: number; w: number; h: number };
|
||||
process?: string;
|
||||
winClass?: string;
|
||||
enriched?: boolean;
|
||||
aiType?: string;
|
||||
aiText?: string;
|
||||
ai?: boolean;
|
||||
/** 从窗口（或桌面根）到目标元素的祖先路径，逐段用稳定属性定位 */
|
||||
path?: Array<{ type?: string; name?: string; id?: string; className?: string; index?: number }>;
|
||||
index: number;
|
||||
@ -181,6 +189,19 @@ export function parseUiaSelector(raw: unknown): UiaSelector {
|
||||
}
|
||||
selector.screen = { x: nums[0], y: nums[1], w: nums[2], h: nums[3] };
|
||||
}
|
||||
else if (key === "rel") {
|
||||
const nums = value.split(/[,،]/).map(item => Number(item.trim()));
|
||||
if (nums.length !== 4 || nums.some(n => !Number.isFinite(n))) {
|
||||
throw new RuntimeError("validation_error", "rel coordinate invalid: " + value, { recommendation: "rel=x,y,width,height" });
|
||||
}
|
||||
selector.rel = { x: nums[0], y: nums[1], w: nums[2], h: nums[3] };
|
||||
}
|
||||
else if (key === "process") selector.process = value;
|
||||
else if (key === "winclass") selector.winClass = value;
|
||||
else if (key === "enriched") selector.enriched = value === "1" || value.toLowerCase() === "true";
|
||||
else if (key === "aitype") selector.aiType = value;
|
||||
else if (key === "aitext") selector.aiText = value;
|
||||
else if (key === "ai") selector.ai = value === "1" || value.toLowerCase() === "true";
|
||||
else if (key === "index") {
|
||||
const n = Number(value);
|
||||
if (Number.isFinite(n) && n >= 1) selector.index = Math.floor(n);
|
||||
@ -189,7 +210,8 @@ export function parseUiaSelector(raw: unknown): UiaSelector {
|
||||
selector.path = parseUiaSelectorPath(value);
|
||||
}
|
||||
}
|
||||
if (!selector.name && !selector.id && !selector.className && !selector.type && !selector.screen && !selector.path) {
|
||||
const hasWindowAnchor = Boolean(selector.window || selector.winClass || selector.process);
|
||||
if (!selector.name && !selector.id && !selector.className && !selector.type && !selector.screen && !selector.path && !(selector.rel && hasWindowAnchor)) {
|
||||
throw new RuntimeError("validation_error", `桌面元素选择器缺少定位条件：${text}`, { recommendation: "至少提供 name= / id= / type= 之一" });
|
||||
}
|
||||
return selector;
|
||||
@ -201,26 +223,42 @@ const UIA_USING = [
|
||||
].join("\n");
|
||||
|
||||
const FIND_HELPER = `
|
||||
function Find-RpaWindow([string]$winName) {
|
||||
function Find-RpaWindow([string]$winName, [string]$winClass, [string]$processName) {
|
||||
# 只读查找：绝不移动/还原/缩放窗口；还原、激活、移动是独立显式动作（Restore/Activate/Move-RpaWindow）
|
||||
$root = [System.Windows.Automation.AutomationElement]::RootElement
|
||||
if (-not $winName) { return $root }
|
||||
$cond = New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::NameProperty, $winName)
|
||||
$win = $root.FindFirst([System.Windows.Automation.TreeScope]::Children, $cond)
|
||||
if ($win) { return $win }
|
||||
if (-not $winName -and -not $winClass -and -not $processName) { return $root }
|
||||
if ($winName) {
|
||||
$cond = New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::NameProperty, $winName)
|
||||
$win = $root.FindFirst([System.Windows.Automation.TreeScope]::Children, $cond)
|
||||
if ($win) { return $win }
|
||||
}
|
||||
Add-RpaUser32
|
||||
$script:hitHwnd = [IntPtr]::Zero
|
||||
$cb = [WinRpa.RpaUser32Click+EnumProc]{ param($h, $l)
|
||||
$sb = New-Object System.Text.StringBuilder 256
|
||||
[void][WinRpa.RpaUser32Click]::GetWindowTextW($h, $sb, 256)
|
||||
if ($sb.ToString() -eq $winName) { $script:hitHwnd = $h; return $false }
|
||||
return $true
|
||||
$cb = [Win32Rpa.RpaUser32Click+EnumProc]{ param($h, $l)
|
||||
if (-not [Win32Rpa.RpaUser32Click]::IsWindowVisible($h)) { return $true }
|
||||
$titleSb = New-Object System.Text.StringBuilder 512
|
||||
[void][Win32Rpa.RpaUser32Click]::GetWindowTextW($h, $titleSb, 512)
|
||||
$title = $titleSb.ToString()
|
||||
if ($winName -and $title -ne $winName) { return $true }
|
||||
$classSb = New-Object System.Text.StringBuilder 256
|
||||
[void][Win32Rpa.RpaUser32Click]::GetClassNameW($h, $classSb, 256)
|
||||
$className = $classSb.ToString()
|
||||
if ($winClass -and $className -ne $winClass) { return $true }
|
||||
if ($processName) {
|
||||
$windowPid = [uint32]0
|
||||
[void][Win32Rpa.RpaUser32Click]::GetWindowThreadProcessId($h, [ref]$windowPid)
|
||||
try { $procName = (Get-Process -Id $windowPid -ErrorAction Stop).ProcessName } catch { return $true }
|
||||
if (-not $procName.Equals($processName, [StringComparison]::OrdinalIgnoreCase)) { return $true }
|
||||
}
|
||||
$script:hitHwnd = $h
|
||||
return $false
|
||||
}
|
||||
[void][WinRpa.RpaUser32Click]::EnumWindows($cb, [IntPtr]::Zero)
|
||||
[void][Win32Rpa.RpaUser32Click]::EnumWindows($cb, [IntPtr]::Zero)
|
||||
if ($script:hitHwnd -ne [IntPtr]::Zero) {
|
||||
try { $win = [System.Windows.Automation.AutomationElement]::FromHandle($script:hitHwnd); if ($win) { return $win } } catch {}
|
||||
}
|
||||
throw "window_not_found:$winName"
|
||||
$missing = @($winName, $winClass, $processName) | Where-Object { $_ } -join ","
|
||||
throw "window_not_found:$missing"
|
||||
}
|
||||
function Restore-RpaWindow($win) {
|
||||
# 显式动作：还原窗口（最小化/最大化状态恢复），不用于查询
|
||||
@ -295,7 +333,33 @@ function Find-RpaElementByPath($parent, [string]$pathJsonBase64) {
|
||||
return $cur
|
||||
}
|
||||
function Add-RpaUser32 {
|
||||
Add-Type -MemberDefinition '[DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y); [DllImport("user32.dll")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra); [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd); [DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); [DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr hWnd, int nCmdShow); [DllImport("user32.dll")] public static extern bool SetWindowPos(IntPtr hWnd, IntPtr after, int X, int Y, int cx, int cy, uint flags); [DllImport("user32.dll")] public static extern bool AttachThreadInput(uint idAttach, uint idAttachTo, bool fAttach); [DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint processId); [DllImport("kernel32.dll")] public static extern uint GetCurrentThreadId(); [DllImport("user32.dll")] public static extern int GetSystemMetrics(int nIndex); [DllImport("user32.dll")] public static extern bool EnumWindows(EnumProc cb, IntPtr lp); public delegate bool EnumProc(IntPtr hWnd, IntPtr lp); [DllImport("user32.dll", CharSet = CharSet.Unicode)] public static extern int GetWindowTextW(IntPtr hWnd, System.Text.StringBuilder sb, int max);' -Name RpaUser32Click -Namespace Win32Rpa | Out-Null
|
||||
Add-Type -MemberDefinition '[DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y); [DllImport("user32.dll")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra); [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd); [DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); [DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr hWnd, int nCmdShow); [DllImport("user32.dll")] public static extern bool SetWindowPos(IntPtr hWnd, IntPtr after, int X, int Y, int cx, int cy, uint flags); [DllImport("user32.dll")] public static extern bool AttachThreadInput(uint idAttach, uint idAttachTo, bool fAttach); [DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint processId); [DllImport("kernel32.dll")] public static extern uint GetCurrentThreadId(); [DllImport("user32.dll")] public static extern int GetSystemMetrics(int nIndex); [DllImport("user32.dll")] public static extern bool EnumWindows(EnumProc cb, IntPtr lp); public delegate bool EnumProc(IntPtr hWnd, IntPtr lp); [DllImport("user32.dll", CharSet = CharSet.Unicode)] public static extern int GetWindowTextW(IntPtr hWnd, System.Text.StringBuilder sb, int max); [DllImport("user32.dll", CharSet = CharSet.Unicode)] public static extern int GetClassNameW(IntPtr hWnd, System.Text.StringBuilder sb, int max); [DllImport("user32.dll")] public static extern bool IsWindowVisible(IntPtr hWnd);' -Name RpaUser32Click -Namespace Win32Rpa | Out-Null
|
||||
}
|
||||
`;
|
||||
|
||||
const RESOLVE_HELPER = `
|
||||
function Resolve-RpaScreenRect($s) {
|
||||
$hasStable = [bool]($s.name -or $s.id -or $s.type -or $s.className)
|
||||
$hasWindowAnchor = [bool]($s.window -or $s.winClass -or $s.process)
|
||||
if ($s.enriched -and $hasStable) {
|
||||
try {
|
||||
$parent = Find-RpaWindow ([string]$s.window) ([string]$s.winClass) ([string]$s.process)
|
||||
$el = Find-RpaElement $parent ([string]$s.name) ([string]$s.id) ([string]$s.type) ([string]$s.className) ([int]1)
|
||||
if ($el) {
|
||||
$r = $el.Current.BoundingRectangle
|
||||
if ($r.Width -gt 0 -and $r.Height -gt 0) { return @{ x=[int]$r.X; y=[int]$r.Y; w=[int]$r.Width; h=[int]$r.Height; method='uia' } }
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
if ($s.rel -and $hasWindowAnchor) {
|
||||
try {
|
||||
$parent = Find-RpaWindow ([string]$s.window) ([string]$s.winClass) ([string]$s.process)
|
||||
$r = $parent.Current.BoundingRectangle
|
||||
return @{ x=[int]($r.X + $s.rel.x); y=[int]($r.Y + $s.rel.y); w=[int]$s.rel.w; h=[int]$s.rel.h; method='window_rel' }
|
||||
} catch {}
|
||||
}
|
||||
if ($s.screen) { return @{ x=[int]$s.screen.x; y=[int]$s.screen.y; w=[int]$s.screen.w; h=[int]$s.screen.h; method='screen_fallback' } }
|
||||
throw "selector_rect_not_found"
|
||||
}
|
||||
`;
|
||||
|
||||
@ -312,7 +376,7 @@ function target(args: Record<string, unknown>, node: { id: string; componentId:
|
||||
|
||||
/** 生成“解析窗口 + 解析元素（路径优先，其次平铺属性）”的 PowerShell 语句，结果写入 $el */
|
||||
function elementResolveLines(s: UiaSelector): string {
|
||||
const parentLine = "$parent = Find-RpaWindow " + psString(s.window ?? "");
|
||||
const parentLine = "$parent = Find-RpaWindow " + psString(s.window ?? "") + " " + psString(s.winClass ?? "") + " " + psString(s.process ?? "");
|
||||
if (s.path?.length) {
|
||||
const pathB64 = Buffer.from(JSON.stringify(s.path), "utf8").toString("base64");
|
||||
return parentLine + "\n$el = Find-RpaElementByPath $parent '" + pathB64 + "'";
|
||||
@ -327,18 +391,25 @@ function elementResolveLines(s: UiaSelector): string {
|
||||
|
||||
const SCREEN_USER32 = "Add-Type -MemberDefinition '[DllImport(\"user32.dll\")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value); [DllImport(\"user32.dll\")] public static extern bool SetCursorPos(int X, int Y); [DllImport(\"user32.dll\")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra);' -Name RpaScreenInput -Namespace RpaScreen | Out-Null; try { [RpaScreen.RpaScreenInput]::SetProcessDpiAwarenessContext([IntPtr](-4)) | Out-Null } catch {}";
|
||||
|
||||
function screenCenter(selector: UiaSelector): { x: number; y: number; w: number; h: number } {
|
||||
return selector.screen!;
|
||||
function resolveScreenPreamble(t: UiaTarget): string {
|
||||
const s = t.selector;
|
||||
const screen = s.screen ? `@{x=${s.screen.x};y=${s.screen.y};w=${s.screen.w};h=${s.screen.h}}` : "$null";
|
||||
const rel = s.rel ? `@{x=${s.rel.x};y=${s.rel.y};w=${s.rel.w};h=${s.rel.h}}` : "$null";
|
||||
return `${UIA_USING}
|
||||
${FIND_HELPER}
|
||||
${RESOLVE_HELPER}
|
||||
$s = @{ enriched=${s.enriched ? "$true" : "$false"}; window=${JSON.stringify(s.window ?? "")}; winClass=${JSON.stringify(s.winClass ?? "")}; process=${JSON.stringify(s.process ?? "")}; name=${JSON.stringify(s.name ?? "")}; id=${JSON.stringify(s.id ?? "")}; type=${JSON.stringify(s.type ?? "")}; className=${JSON.stringify(s.className ?? "")}; screen=${screen}; rel=${rel} }
|
||||
$rect = Resolve-RpaScreenRect $s
|
||||
$x = [int]($rect.x + $rect.w / 2)
|
||||
$y = [int]($rect.y + $rect.h / 2)`;
|
||||
}
|
||||
|
||||
function clickScreenScript(t: UiaTarget): string {
|
||||
const rect = screenCenter(t.selector);
|
||||
const x = Math.round(rect.x + rect.w / 2);
|
||||
const y = Math.round(rect.y + rect.h / 2);
|
||||
const clickway = String(t.args.clickway ?? "0");
|
||||
const doubleClick = clickway === "1" || /double/i.test(clickway);
|
||||
return `${SCREEN_USER32}
|
||||
if (-not [RpaScreen.RpaScreenInput]::SetCursorPos(${x}, ${y})) { throw "cursor_move_failed" }
|
||||
return `${resolveScreenPreamble(t)}
|
||||
${SCREEN_USER32}
|
||||
if (-not [RpaScreen.RpaScreenInput]::SetCursorPos($x, $y)) { throw "cursor_move_failed" }
|
||||
Start-Sleep -Milliseconds 100
|
||||
[RpaScreen.RpaScreenInput]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)
|
||||
Start-Sleep -Milliseconds 40
|
||||
@ -349,16 +420,14 @@ if (${doubleClick}) {
|
||||
Start-Sleep -Milliseconds 40
|
||||
[RpaScreen.RpaScreenInput]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)
|
||||
}
|
||||
Write-Output (@{ clicked = $true; method = 'screen_coordinate'; x = ${x}; y = ${y} } | ConvertTo-Json -Compress)`;
|
||||
Write-Output (@{ clicked = $true; method = $rect.method; resolvedMethod = $rect.method; x = $x; y = $y } | ConvertTo-Json -Compress)`;
|
||||
}
|
||||
|
||||
function setValueScreenScript(t: UiaTarget, value: string): string {
|
||||
const rect = screenCenter(t.selector);
|
||||
const x = Math.round(rect.x + rect.w / 2);
|
||||
const y = Math.round(rect.y + rect.h / 2);
|
||||
return `${SCREEN_USER32}
|
||||
return `${resolveScreenPreamble(t)}
|
||||
${SCREEN_USER32}
|
||||
Add-Type -AssemblyName System.Windows.Forms
|
||||
[RpaScreen.RpaScreenInput]::SetCursorPos(${x}, ${y}) | Out-Null
|
||||
[RpaScreen.RpaScreenInput]::SetCursorPos($x, $y) | Out-Null
|
||||
Start-Sleep -Milliseconds 120
|
||||
[RpaScreen.RpaScreenInput]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)
|
||||
Start-Sleep -Milliseconds 40
|
||||
@ -375,21 +444,46 @@ Start-Sleep -Milliseconds 280
|
||||
try {
|
||||
if ($saved) { [System.Windows.Forms.Clipboard]::SetText($saved) } else { [System.Windows.Forms.Clipboard]::Clear() }
|
||||
} catch {}
|
||||
Write-Output (@{ set = $true; method = 'screen_coordinate_paste'; x = ${x}; y = ${y} } | ConvertTo-Json -Compress)`;
|
||||
Write-Output (@{ set = $true; method = $rect.method; resolvedMethod = $rect.method; x = $x; y = $y } | ConvertTo-Json -Compress)`;
|
||||
}
|
||||
|
||||
function getValueScreenScript(t: UiaTarget): string {
|
||||
const rect = screenCenter(t.selector);
|
||||
const x = Math.round(rect.x + rect.w / 2);
|
||||
const y = Math.round(rect.y + rect.h / 2);
|
||||
return `${UIA_USING}
|
||||
return `${resolveScreenPreamble(t)}
|
||||
Add-Type -AssemblyName WindowsBase
|
||||
$pt = New-Object System.Windows.Point(${x}, ${y})
|
||||
$pt = New-Object System.Windows.Point($x, $y)
|
||||
$el = [System.Windows.Automation.AutomationElement]::FromPoint($pt)
|
||||
if ($null -eq $el) { Write-Output 'JSON_NOT_FOUND'; exit 0 }
|
||||
$value = $el.Current.Name
|
||||
try { $vp = $el.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern); $value = $vp.Value } catch {}
|
||||
Write-Output (@{ value = $value; name = $el.Current.Name } | ConvertTo-Json -Compress)`;
|
||||
Write-Output (@{ value = $value; name = $el.Current.Name; method = $rect.method } | ConvertTo-Json -Compress)`;
|
||||
}
|
||||
|
||||
function screenExistsScript(t: UiaTarget): string {
|
||||
return `${resolveScreenPreamble(t)}
|
||||
Add-Type -AssemblyName WindowsBase
|
||||
$found = $true
|
||||
if ($rect.method -ne 'uia') {
|
||||
try {
|
||||
$pt = New-Object System.Windows.Point($x, $y)
|
||||
$hit = [System.Windows.Automation.AutomationElement]::FromPoint($pt)
|
||||
if ($null -eq $hit) { $found = $false }
|
||||
else {
|
||||
$hr = $hit.Current.BoundingRectangle
|
||||
$inside = ($hr.Right -ge $rect.x) -and ($hr.Left -le ($rect.x + $rect.w)) -and ($hr.Bottom -ge $rect.y) -and ($hr.Top -le ($rect.y + $rect.h))
|
||||
if (-not $inside) { $found = $false }
|
||||
}
|
||||
} catch { $found = $false }
|
||||
}
|
||||
Write-Output (@{ found = $found; method = $rect.method; x = $rect.x; y = $rect.y; w = $rect.w; h = $rect.h } | ConvertTo-Json -Compress)`;
|
||||
}
|
||||
|
||||
async function screenElementExists(t: UiaTarget): Promise<boolean> {
|
||||
try {
|
||||
const result = await runScript(screenExistsScript(t), t);
|
||||
return result?.found === true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function findScript(t: UiaTarget, timeoutMs: number, gapMs: number): string {
|
||||
@ -628,7 +722,7 @@ const setValueGround: CapabilityHandler = async (_ctx, node, args) => {
|
||||
|
||||
const existsGround: CapabilityHandler = async (_ctx, node, args) => {
|
||||
const t = target(args, node);
|
||||
if (t.selector.screen) return true;
|
||||
if (t.selector.screen) return await screenElementExists(t);
|
||||
const found = await findElement(t, 1000, 150);
|
||||
return found !== null;
|
||||
};
|
||||
@ -645,7 +739,17 @@ const getValueGround: CapabilityHandler = async (_ctx, node, args) => {
|
||||
|
||||
const waitElementGround: CapabilityHandler = async (_ctx, node, args) => {
|
||||
const t = target(args, node);
|
||||
if (t.selector.screen) return true;
|
||||
if (t.selector.screen) {
|
||||
const timeout = Number(args.timeout ?? args.timeoutMs ?? 5000) || 5000;
|
||||
const gap = Number(args.time_gap ?? args.gapMs ?? 200) || 200;
|
||||
const deadline = Date.now() + timeout;
|
||||
do {
|
||||
if (await screenElementExists(t)) return true;
|
||||
if (Date.now() >= deadline) break;
|
||||
await new Promise(resolve => setTimeout(resolve, gap));
|
||||
} while (Date.now() < deadline);
|
||||
return false;
|
||||
}
|
||||
const timeout = Number(args.timeout ?? args.timeoutMs ?? 5000) || 5000;
|
||||
const gap = Number(args.time_gap ?? args.gapMs ?? 200) || 200;
|
||||
const found = await findElement(t, timeout, gap);
|
||||
|
||||
@ -373,6 +373,77 @@ async function capturePaneForAi(rect: { x: number; y: number; w: number; h: numb
|
||||
}
|
||||
}
|
||||
|
||||
interface AiUiaInspection {
|
||||
found: boolean;
|
||||
name?: string;
|
||||
id?: string;
|
||||
type?: string;
|
||||
className?: string;
|
||||
x?: number;
|
||||
y?: number;
|
||||
w?: number;
|
||||
h?: number;
|
||||
topName?: string;
|
||||
topClassName?: string;
|
||||
process?: string;
|
||||
topX?: number;
|
||||
topY?: number;
|
||||
topW?: number;
|
||||
topH?: number;
|
||||
}
|
||||
|
||||
async function inspectAiSelection(
|
||||
rect: { x: number; y: number; w: number; h: number },
|
||||
overlay: BrowserWindow
|
||||
): Promise<AiUiaInspection | null> {
|
||||
// UIA FromPoint 会拿到透明覆盖层，必须先隐藏并给它一帧退出屏幕的时间。
|
||||
if (!overlay.isDestroyed()) overlay.hide();
|
||||
await new Promise(resolve => setTimeout(resolve, 90));
|
||||
const cx = Math.round(rect.x + rect.w / 2);
|
||||
const cy = Math.round(rect.y + rect.h / 2);
|
||||
const script = [
|
||||
"Add-Type -AssemblyName UIAutomationClient, UIAutomationTypes | Out-Null;",
|
||||
"try { Add-Type -MemberDefinition '[DllImport(\"user32.dll\")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value);' -Name AiDpi -Namespace RpaAiPick | Out-Null; [RpaAiPick.AiDpi]::SetProcessDpiAwarenessContext([IntPtr](-4)) | Out-Null } catch {}",
|
||||
"$pt = New-Object System.Windows.Point(" + cx + "," + cy + ")",
|
||||
"Add-Type -AssemblyName WindowsBase | Out-Null",
|
||||
"$el = [System.Windows.Automation.AutomationElement]::FromPoint($pt)",
|
||||
"if ($null -eq $el -or $el.Current.ProcessId -eq 0) { Write-Output '{\"found\":false}'; exit 0 }",
|
||||
"$walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker",
|
||||
"$top = $el",
|
||||
"for ($i = 0; $i -lt 80; $i++) {",
|
||||
" $parent = $walker.GetParent($top)",
|
||||
" if ($null -eq $parent -or $parent.Current.ProcessId -eq 0) { break }",
|
||||
" $top = $parent",
|
||||
"}",
|
||||
"$er = $el.Current.BoundingRectangle",
|
||||
"$tr = $top.Current.BoundingRectangle",
|
||||
"$controlType = ''",
|
||||
"try { $controlType = $el.Current.ControlType.ProgrammaticName -replace '^ControlType\\.', '' } catch {}",
|
||||
"$procName = ''",
|
||||
"try { $procName = (Get-Process -Id $top.Current.ProcessId -ErrorAction Stop).ProcessName } catch {}",
|
||||
"$info = @{",
|
||||
" found = $true",
|
||||
" name = [string]$el.Current.Name",
|
||||
" id = [string]$el.Current.AutomationId",
|
||||
" type = [string]$controlType",
|
||||
" className = [string]$el.Current.ClassName",
|
||||
" x = [int]$er.X; y = [int]$er.Y; w = [int]$er.Width; h = [int]$er.Height",
|
||||
" topName = [string]$top.Current.Name",
|
||||
" topClassName = [string]$top.Current.ClassName",
|
||||
" process = [string]$procName",
|
||||
" topX = [int]$tr.X; topY = [int]$tr.Y; topW = [int]$tr.Width; topH = [int]$tr.Height",
|
||||
"}",
|
||||
"Write-Output ($info | ConvertTo-Json -Compress)"
|
||||
].join("\n");
|
||||
try {
|
||||
const { stdout } = await run("powershell.exe", ["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-Command", script], { timeout: 8000, windowsHide: true });
|
||||
return jsonLine<AiUiaInspection>(stdout, "{");
|
||||
} catch (error) {
|
||||
console.warn("[pane-ai] uia enrichment failed", error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function runPaneAiPick(context: UiaPickOutput): Promise<DesktopPickResult> {
|
||||
// 本地小模型是 Pane AI 的主链路；OCR 使用同一张 Pane 截图，不要求用户配置远程 VLM。
|
||||
const localModelPath = localUiModelPath();
|
||||
@ -543,23 +614,59 @@ async function runPaneAiPick(context: UiaPickOutput): Promise<DesktopPickResult>
|
||||
await detected.ocrTask;
|
||||
|
||||
const box = candidates.find(item => item.candidateId === selectedId)!;
|
||||
const x = detected.capture.cropX + box.x;
|
||||
const y = detected.capture.cropY + box.y;
|
||||
const aiRect = {
|
||||
x: detected.capture.cropX + box.x,
|
||||
y: detected.capture.cropY + box.y,
|
||||
w: box.width,
|
||||
h: box.height
|
||||
};
|
||||
|
||||
// 用户确认后再查 UIA；覆盖层已隐藏，避免 FromPoint 命中拾取浮层。
|
||||
const inspection = await inspectAiSelection(aiRect, win);
|
||||
const ir = inspection?.found ? inspection : null;
|
||||
const intersectsAi = ir !== null &&
|
||||
typeof ir.x === "number" && typeof ir.y === "number" &&
|
||||
typeof ir.w === "number" && typeof ir.h === "number" &&
|
||||
ir.w > 0 && ir.h > 0 &&
|
||||
Math.min(aiRect.x + aiRect.w, ir.x + ir.w) - Math.max(aiRect.x, ir.x) > 2 &&
|
||||
Math.min(aiRect.y + aiRect.h, ir.y + ir.h) - Math.max(aiRect.y, ir.y) > 2;
|
||||
const x = intersectsAi ? Math.round(ir!.x!) : Math.round(aiRect.x);
|
||||
const y = intersectsAi ? Math.round(ir!.y!) : Math.round(aiRect.y);
|
||||
const w = intersectsAi ? Math.round(ir!.w!) : Math.round(aiRect.w);
|
||||
const h = intersectsAi ? Math.round(ir!.h!) : Math.round(aiRect.h);
|
||||
const topName = ir?.topName || context.window;
|
||||
const hasTopRect = ir?.topW !== undefined && ir.topW > 0 && ir.topH !== undefined && ir.topH > 0;
|
||||
const rel = hasTopRect
|
||||
? [Math.round(x - ir!.topX!), Math.round(y - ir!.topY!), w, h].join(",")
|
||||
: undefined;
|
||||
const stableName = ir?.name?.trim();
|
||||
const stableId = ir?.id?.trim();
|
||||
const stableClass = ir?.className?.trim();
|
||||
const stableType = ir?.type?.trim();
|
||||
const enriched = Boolean(ir && (stableId || stableName || stableClass));
|
||||
const selectorParts = [
|
||||
"screen=" + [x, y, box.width, box.height].map(Math.round).join(","),
|
||||
context.window ? "window=" + safeSelectorText(context.window) : "",
|
||||
"name=" + safeSelectorText(box.name || box.ocrText || box.controlType),
|
||||
"type=" + safeSelectorText(box.controlType),
|
||||
"screen=" + [x, y, w, h].map(Math.round).join(","),
|
||||
topName ? "window=" + safeSelectorText(topName) : "",
|
||||
ir?.topClassName ? "winClass=" + safeSelectorText(ir.topClassName) : "",
|
||||
ir?.process ? "process=" + safeSelectorText(ir.process) : "",
|
||||
rel ? "rel=" + rel : "",
|
||||
stableName ? "name=" + safeSelectorText(stableName) : (box.name || box.ocrText ? "name=" + safeSelectorText(box.name || box.ocrText) : ""),
|
||||
stableId ? "id=" + safeSelectorText(stableId) : "",
|
||||
enriched && stableType ? "type=" + safeSelectorText(stableType) : "",
|
||||
stableClass ? "class=" + safeSelectorText(stableClass) : "",
|
||||
enriched ? "enriched=1" : "",
|
||||
"aiType=" + safeSelectorText(box.controlType),
|
||||
box.ocrText ? "aiText=" + safeSelectorText(box.ocrText) : "",
|
||||
"ai=1"
|
||||
].filter(Boolean);
|
||||
return {
|
||||
source: "ai",
|
||||
window: context.window,
|
||||
name: box.name || box.ocrText || box.controlType,
|
||||
controlType: box.controlType,
|
||||
x, y, w: box.width, h: box.height,
|
||||
window: topName,
|
||||
name: stableName || box.name || box.ocrText || box.controlType,
|
||||
automationId: stableId,
|
||||
controlType: stableType || box.controlType,
|
||||
className: stableClass,
|
||||
x, y, w, h,
|
||||
confidence: box.confidence,
|
||||
screenshotPath: detected.capture.imagePath,
|
||||
selector: selectorParts.join(";")
|
||||
@ -654,3 +761,4 @@ export async function pickScreenRegion(): Promise<{ x: number; y: number; width:
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user