diff --git a/src/main/capabilities/desktop-uia.test.ts b/src/main/capabilities/desktop-uia.test.ts index c4c27a7..3bf714b 100644 --- a/src/main/capabilities/desktop-uia.test.ts +++ b/src/main/capabilities/desktop-uia.test.ts @@ -64,6 +64,18 @@ describe("desktop UIA ground capabilities", () => { expect(s.window).toBe("MyApp"); }); + it("parses enriched AI relocation metadata", () => { + const selector = parseUiaSelector( + "screen=100,200,320,80;window=App;winClass=Chrome_WidgetWin_1;process=chrome;rel=20,30,320,80;name=login;id=loginButton;type=Button;class=Button;aiType=button;aiText=login;enriched=1;ai=1" + ); + expect(selector.screen).toEqual({ x: 100, y: 200, w: 320, h: 80 }); + expect(selector.rel).toEqual({ x: 20, y: 30, w: 320, h: 80 }); + expect(selector.window).toBe("App"); + expect(selector.winClass).toBe("Chrome_WidgetWin_1"); + expect(selector.process).toBe("chrome"); + expect(selector).toMatchObject({ name: "login", id: "loginButton", type: "Button", className: "Button", aiType: "button", aiText: "login", enriched: true, ai: true }); + }); + it("rejects invalid selector", () => { expect(() => parseUiaSelector("")).toThrow(); expect(() => parseUiaSelector("screen=1,2,3")).toThrow(); @@ -94,7 +106,7 @@ describe("desktop UIA selector compatibility", () => { const context = { log: vi.fn() } as any; const node = { id: "n1", componentId: "Element_ClickGround" } as any; - it("dispatches screen selectors to coordinate execution instead of UIA lookup", async () => { + it("resolves screen selectors through UIA/window before coordinate fallback", async () => { const args = { selector: "screen=100,200,320,80;ai=1", clickway: "1" @@ -102,8 +114,10 @@ describe("desktop UIA selector compatibility", () => { await expect(getHandler("Element_ClickGround")!(context, node, args, {} as any)).resolves.toEqual({ clicked: true, value: "cached" }); const script = String((powershellMock.mock.calls as unknown as Array<[string]>)[0]?.[0] ?? ""); expect(script).toContain("RpaScreenInput"); - expect(script).toContain("SetCursorPos(260, 240)"); - expect(script).not.toContain("Find-RpaWindow"); + expect(script).toContain("$x = [int]($rect.x + $rect.w / 2)"); + expect(script).toContain("SetCursorPos($x, $y)"); + expect(script).toContain("Resolve-RpaScreenRect"); + expect(script).toContain("Find-RpaWindow"); }); it("dispatches screen setValue to coordinate paste", async () => { @@ -112,18 +126,22 @@ describe("desktop UIA selector compatibility", () => { value: "hello" }, {} as any); const script = String((powershellMock.mock.calls as unknown as Array<[string]>)[0]?.[0] ?? ""); - expect(script).toContain("screen_coordinate_paste"); + expect(script).toContain("resolvedMethod = $rect.method"); expect(script).toContain("hello"); - expect(script).not.toContain("Find-RpaWindow"); + expect(script).toContain("Find-RpaWindow"); }); - it("treats a picked screen rectangle as present for exists/wait", async () => { + it("validates a picked screen rectangle for exists/wait", async () => { + powershellMock.mockImplementation(async () => '{"found":true,"method":"window_rel"}'); await expect(getHandler("Element_ExistsGround")!(context, node, { selector: "screen=1,2,3,4" }, {} as any)).resolves.toBe(true); await expect(getHandler("Element_WaitElementGround")!(context, node, { selector: "screen=1,2,3,4" }, {} as any)).resolves.toBe(true); - expect(powershellMock).not.toHaveBeenCalled(); + expect(powershellMock).toHaveBeenCalled(); + const script = String((powershellMock.mock.calls as unknown as Array<[string]>).at(-1)?.[0] ?? ""); + expect(script).toContain("Resolve-RpaScreenRect"); }); }); + diff --git a/src/main/capabilities/desktop-uia.ts b/src/main/capabilities/desktop-uia.ts index 9ed3f97..0a3c2f0 100644 --- a/src/main/capabilities/desktop-uia.ts +++ b/src/main/capabilities/desktop-uia.ts @@ -14,6 +14,14 @@ interface UiaSelector { typeRaw?: string; className?: string; screen?: { x: number; y: number; w: number; h: number }; + /** Relative physical rect inside the top-level window. */ + rel?: { x: number; y: number; w: number; h: number }; + process?: string; + winClass?: string; + enriched?: boolean; + aiType?: string; + aiText?: string; + ai?: boolean; /** 从窗口（或桌面根）到目标元素的祖先路径，逐段用稳定属性定位 */ path?: Array<{ type?: string; name?: string; id?: string; className?: string; index?: number }>; index: number; @@ -181,6 +189,19 @@ export function parseUiaSelector(raw: unknown): UiaSelector { } selector.screen = { x: nums[0], y: nums[1], w: nums[2], h: nums[3] }; } + else if (key === "rel") { + const nums = value.split(/[,،]/).map(item => Number(item.trim())); + if (nums.length !== 4 || nums.some(n => !Number.isFinite(n))) { + throw new RuntimeError("validation_error", "rel coordinate invalid: " + value, { recommendation: "rel=x,y,width,height" }); + } + selector.rel = { x: nums[0], y: nums[1], w: nums[2], h: nums[3] }; + } + else if (key === "process") selector.process = value; + else if (key === "winclass") selector.winClass = value; + else if (key === "enriched") selector.enriched = value === "1" || value.toLowerCase() === "true"; + else if (key === "aitype") selector.aiType = value; + else if (key === "aitext") selector.aiText = value; + else if (key === "ai") selector.ai = value === "1" || value.toLowerCase() === "true"; else if (key === "index") { const n = Number(value); if (Number.isFinite(n) && n >= 1) selector.index = Math.floor(n); @@ -189,7 +210,8 @@ export function parseUiaSelector(raw: unknown): UiaSelector { selector.path = parseUiaSelectorPath(value); } } - if (!selector.name && !selector.id && !selector.className && !selector.type && !selector.screen && !selector.path) { + const hasWindowAnchor = Boolean(selector.window || selector.winClass || selector.process); + if (!selector.name && !selector.id && !selector.className && !selector.type && !selector.screen && !selector.path && !(selector.rel && hasWindowAnchor)) { throw new RuntimeError("validation_error", `桌面元素选择器缺少定位条件：${text}`, { recommendation: "至少提供 name= / id= / type= 之一" }); } return selector; @@ -201,26 +223,42 @@ const UIA_USING = [ ].join("\n"); const FIND_HELPER = ` -function Find-RpaWindow([string]$winName) { +function Find-RpaWindow([string]$winName, [string]$winClass, [string]$processName) { # 只读查找：绝不移动/还原/缩放窗口；还原、激活、移动是独立显式动作（Restore/Activate/Move-RpaWindow） $root = [System.Windows.Automation.AutomationElement]::RootElement - if (-not $winName) { return $root } - $cond = New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::NameProperty, $winName) - $win = $root.FindFirst([System.Windows.Automation.TreeScope]::Children, $cond) - if ($win) { return $win } + if (-not $winName -and -not $winClass -and -not $processName) { return $root } + if ($winName) { + $cond = New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::NameProperty, $winName) + $win = $root.FindFirst([System.Windows.Automation.TreeScope]::Children, $cond) + if ($win) { return $win } + } Add-RpaUser32 $script:hitHwnd = [IntPtr]::Zero - $cb = [WinRpa.RpaUser32Click+EnumProc]{ param($h, $l) - $sb = New-Object System.Text.StringBuilder 256 - [void][WinRpa.RpaUser32Click]::GetWindowTextW($h, $sb, 256) - if ($sb.ToString() -eq $winName) { $script:hitHwnd = $h; return $false } - return $true + $cb = [Win32Rpa.RpaUser32Click+EnumProc]{ param($h, $l) + if (-not [Win32Rpa.RpaUser32Click]::IsWindowVisible($h)) { return $true } + $titleSb = New-Object System.Text.StringBuilder 512 + [void][Win32Rpa.RpaUser32Click]::GetWindowTextW($h, $titleSb, 512) + $title = $titleSb.ToString() + if ($winName -and $title -ne $winName) { return $true } + $classSb = New-Object System.Text.StringBuilder 256 + [void][Win32Rpa.RpaUser32Click]::GetClassNameW($h, $classSb, 256) + $className = $classSb.ToString() + if ($winClass -and $className -ne $winClass) { return $true } + if ($processName) { + $windowPid = [uint32]0 + [void][Win32Rpa.RpaUser32Click]::GetWindowThreadProcessId($h, [ref]$windowPid) + try { $procName = (Get-Process -Id $windowPid -ErrorAction Stop).ProcessName } catch { return $true } + if (-not $procName.Equals($processName, [StringComparison]::OrdinalIgnoreCase)) { return $true } + } + $script:hitHwnd = $h + return $false } - [void][WinRpa.RpaUser32Click]::EnumWindows($cb, [IntPtr]::Zero) + [void][Win32Rpa.RpaUser32Click]::EnumWindows($cb, [IntPtr]::Zero) if ($script:hitHwnd -ne [IntPtr]::Zero) { try { $win = [System.Windows.Automation.AutomationElement]::FromHandle($script:hitHwnd); if ($win) { return $win } } catch {} } - throw "window_not_found:$winName" + $missing = @($winName, $winClass, $processName) | Where-Object { $_ } -join "," + throw "window_not_found:$missing" } function Restore-RpaWindow($win) { # 显式动作：还原窗口（最小化/最大化状态恢复），不用于查询 @@ -295,7 +333,33 @@ function Find-RpaElementByPath($parent, [string]$pathJsonBase64) { return $cur } function Add-RpaUser32 { - Add-Type -MemberDefinition '[DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y); [DllImport("user32.dll")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra); [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd); [DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); [DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr hWnd, int nCmdShow); [DllImport("user32.dll")] public static extern bool SetWindowPos(IntPtr hWnd, IntPtr after, int X, int Y, int cx, int cy, uint flags); [DllImport("user32.dll")] public static extern bool AttachThreadInput(uint idAttach, uint idAttachTo, bool fAttach); [DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint processId); [DllImport("kernel32.dll")] public static extern uint GetCurrentThreadId(); [DllImport("user32.dll")] public static extern int GetSystemMetrics(int nIndex); [DllImport("user32.dll")] public static extern bool EnumWindows(EnumProc cb, IntPtr lp); public delegate bool EnumProc(IntPtr hWnd, IntPtr lp); [DllImport("user32.dll", CharSet = CharSet.Unicode)] public static extern int GetWindowTextW(IntPtr hWnd, System.Text.StringBuilder sb, int max);' -Name RpaUser32Click -Namespace Win32Rpa | Out-Null + Add-Type -MemberDefinition '[DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y); [DllImport("user32.dll")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra); [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd); [DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); [DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr hWnd, int nCmdShow); [DllImport("user32.dll")] public static extern bool SetWindowPos(IntPtr hWnd, IntPtr after, int X, int Y, int cx, int cy, uint flags); [DllImport("user32.dll")] public static extern bool AttachThreadInput(uint idAttach, uint idAttachTo, bool fAttach); [DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint processId); [DllImport("kernel32.dll")] public static extern uint GetCurrentThreadId(); [DllImport("user32.dll")] public static extern int GetSystemMetrics(int nIndex); [DllImport("user32.dll")] public static extern bool EnumWindows(EnumProc cb, IntPtr lp); public delegate bool EnumProc(IntPtr hWnd, IntPtr lp); [DllImport("user32.dll", CharSet = CharSet.Unicode)] public static extern int GetWindowTextW(IntPtr hWnd, System.Text.StringBuilder sb, int max); [DllImport("user32.dll", CharSet = CharSet.Unicode)] public static extern int GetClassNameW(IntPtr hWnd, System.Text.StringBuilder sb, int max); [DllImport("user32.dll")] public static extern bool IsWindowVisible(IntPtr hWnd);' -Name RpaUser32Click -Namespace Win32Rpa | Out-Null +} +`; + +const RESOLVE_HELPER = ` +function Resolve-RpaScreenRect($s) { + $hasStable = [bool]($s.name -or $s.id -or $s.type -or $s.className) + $hasWindowAnchor = [bool]($s.window -or $s.winClass -or $s.process) + if ($s.enriched -and $hasStable) { + try { + $parent = Find-RpaWindow ([string]$s.window) ([string]$s.winClass) ([string]$s.process) + $el = Find-RpaElement $parent ([string]$s.name) ([string]$s.id) ([string]$s.type) ([string]$s.className) ([int]1) + if ($el) { + $r = $el.Current.BoundingRectangle + if ($r.Width -gt 0 -and $r.Height -gt 0) { return @{ x=[int]$r.X; y=[int]$r.Y; w=[int]$r.Width; h=[int]$r.Height; method='uia' } } + } + } catch {} + } + if ($s.rel -and $hasWindowAnchor) { + try { + $parent = Find-RpaWindow ([string]$s.window) ([string]$s.winClass) ([string]$s.process) + $r = $parent.Current.BoundingRectangle + return @{ x=[int]($r.X + $s.rel.x); y=[int]($r.Y + $s.rel.y); w=[int]$s.rel.w; h=[int]$s.rel.h; method='window_rel' } + } catch {} + } + if ($s.screen) { return @{ x=[int]$s.screen.x; y=[int]$s.screen.y; w=[int]$s.screen.w; h=[int]$s.screen.h; method='screen_fallback' } } + throw "selector_rect_not_found" } `; @@ -312,7 +376,7 @@ function target(args: Record, node: { id: string; componentId: /** 生成“解析窗口 + 解析元素（路径优先，其次平铺属性）”的 PowerShell 语句，结果写入 $el */ function elementResolveLines(s: UiaSelector): string { - const parentLine = "$parent = Find-RpaWindow " + psString(s.window ?? ""); + const parentLine = "$parent = Find-RpaWindow " + psString(s.window ?? "") + " " + psString(s.winClass ?? "") + " " + psString(s.process ?? ""); if (s.path?.length) { const pathB64 = Buffer.from(JSON.stringify(s.path), "utf8").toString("base64"); return parentLine + "\n$el = Find-RpaElementByPath $parent '" + pathB64 + "'"; @@ -327,18 +391,25 @@ function elementResolveLines(s: UiaSelector): string { const SCREEN_USER32 = "Add-Type -MemberDefinition '[DllImport(\"user32.dll\")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value); [DllImport(\"user32.dll\")] public static extern bool SetCursorPos(int X, int Y); [DllImport(\"user32.dll\")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra);' -Name RpaScreenInput -Namespace RpaScreen | Out-Null; try { [RpaScreen.RpaScreenInput]::SetProcessDpiAwarenessContext([IntPtr](-4)) | Out-Null } catch {}"; -function screenCenter(selector: UiaSelector): { x: number; y: number; w: number; h: number } { - return selector.screen!; +function resolveScreenPreamble(t: UiaTarget): string { + const s = t.selector; + const screen = s.screen ? `@{x=${s.screen.x};y=${s.screen.y};w=${s.screen.w};h=${s.screen.h}}` : "$null"; + const rel = s.rel ? `@{x=${s.rel.x};y=${s.rel.y};w=${s.rel.w};h=${s.rel.h}}` : "$null"; + return `${UIA_USING} +${FIND_HELPER} +${RESOLVE_HELPER} +$s = @{ enriched=${s.enriched ? "$true" : "$false"}; window=${JSON.stringify(s.window ?? "")}; winClass=${JSON.stringify(s.winClass ?? "")}; process=${JSON.stringify(s.process ?? "")}; name=${JSON.stringify(s.name ?? "")}; id=${JSON.stringify(s.id ?? "")}; type=${JSON.stringify(s.type ?? "")}; className=${JSON.stringify(s.className ?? "")}; screen=${screen}; rel=${rel} } +$rect = Resolve-RpaScreenRect $s +$x = [int]($rect.x + $rect.w / 2) +$y = [int]($rect.y + $rect.h / 2)`; } function clickScreenScript(t: UiaTarget): string { - const rect = screenCenter(t.selector); - const x = Math.round(rect.x + rect.w / 2); - const y = Math.round(rect.y + rect.h / 2); const clickway = String(t.args.clickway ?? "0"); const doubleClick = clickway === "1" || /double/i.test(clickway); - return `${SCREEN_USER32} -if (-not [RpaScreen.RpaScreenInput]::SetCursorPos(${x}, ${y})) { throw "cursor_move_failed" } + return `${resolveScreenPreamble(t)} +${SCREEN_USER32} +if (-not [RpaScreen.RpaScreenInput]::SetCursorPos($x, $y)) { throw "cursor_move_failed" } Start-Sleep -Milliseconds 100 [RpaScreen.RpaScreenInput]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero) Start-Sleep -Milliseconds 40 @@ -349,16 +420,14 @@ if (${doubleClick}) { Start-Sleep -Milliseconds 40 [RpaScreen.RpaScreenInput]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero) } -Write-Output (@{ clicked = $true; method = 'screen_coordinate'; x = ${x}; y = ${y} } | ConvertTo-Json -Compress)`; +Write-Output (@{ clicked = $true; method = $rect.method; resolvedMethod = $rect.method; x = $x; y = $y } | ConvertTo-Json -Compress)`; } function setValueScreenScript(t: UiaTarget, value: string): string { - const rect = screenCenter(t.selector); - const x = Math.round(rect.x + rect.w / 2); - const y = Math.round(rect.y + rect.h / 2); - return `${SCREEN_USER32} + return `${resolveScreenPreamble(t)} +${SCREEN_USER32} Add-Type -AssemblyName System.Windows.Forms -[RpaScreen.RpaScreenInput]::SetCursorPos(${x}, ${y}) | Out-Null +[RpaScreen.RpaScreenInput]::SetCursorPos($x, $y) | Out-Null Start-Sleep -Milliseconds 120 [RpaScreen.RpaScreenInput]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero) Start-Sleep -Milliseconds 40 @@ -375,21 +444,46 @@ Start-Sleep -Milliseconds 280 try { if ($saved) { [System.Windows.Forms.Clipboard]::SetText($saved) } else { [System.Windows.Forms.Clipboard]::Clear() } } catch {} -Write-Output (@{ set = $true; method = 'screen_coordinate_paste'; x = ${x}; y = ${y} } | ConvertTo-Json -Compress)`; +Write-Output (@{ set = $true; method = $rect.method; resolvedMethod = $rect.method; x = $x; y = $y } | ConvertTo-Json -Compress)`; } function getValueScreenScript(t: UiaTarget): string { - const rect = screenCenter(t.selector); - const x = Math.round(rect.x + rect.w / 2); - const y = Math.round(rect.y + rect.h / 2); - return `${UIA_USING} + return `${resolveScreenPreamble(t)} Add-Type -AssemblyName WindowsBase -$pt = New-Object System.Windows.Point(${x}, ${y}) +$pt = New-Object System.Windows.Point($x, $y) $el = [System.Windows.Automation.AutomationElement]::FromPoint($pt) if ($null -eq $el) { Write-Output 'JSON_NOT_FOUND'; exit 0 } $value = $el.Current.Name try { $vp = $el.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern); $value = $vp.Value } catch {} -Write-Output (@{ value = $value; name = $el.Current.Name } | ConvertTo-Json -Compress)`; +Write-Output (@{ value = $value; name = $el.Current.Name; method = $rect.method } | ConvertTo-Json -Compress)`; +} + +function screenExistsScript(t: UiaTarget): string { + return `${resolveScreenPreamble(t)} +Add-Type -AssemblyName WindowsBase +$found = $true +if ($rect.method -ne 'uia') { + try { + $pt = New-Object System.Windows.Point($x, $y) + $hit = [System.Windows.Automation.AutomationElement]::FromPoint($pt) + if ($null -eq $hit) { $found = $false } + else { + $hr = $hit.Current.BoundingRectangle + $inside = ($hr.Right -ge $rect.x) -and ($hr.Left -le ($rect.x + $rect.w)) -and ($hr.Bottom -ge $rect.y) -and ($hr.Top -le ($rect.y + $rect.h)) + if (-not $inside) { $found = $false } + } + } catch { $found = $false } +} +Write-Output (@{ found = $found; method = $rect.method; x = $rect.x; y = $rect.y; w = $rect.w; h = $rect.h } | ConvertTo-Json -Compress)`; +} + +async function screenElementExists(t: UiaTarget): Promise { + try { + const result = await runScript(screenExistsScript(t), t); + return result?.found === true; + } catch { + return false; + } } function findScript(t: UiaTarget, timeoutMs: number, gapMs: number): string { @@ -628,7 +722,7 @@ const setValueGround: CapabilityHandler = async (_ctx, node, args) => { const existsGround: CapabilityHandler = async (_ctx, node, args) => { const t = target(args, node); - if (t.selector.screen) return true; + if (t.selector.screen) return await screenElementExists(t); const found = await findElement(t, 1000, 150); return found !== null; }; @@ -645,7 +739,17 @@ const getValueGround: CapabilityHandler = async (_ctx, node, args) => { const waitElementGround: CapabilityHandler = async (_ctx, node, args) => { const t = target(args, node); - if (t.selector.screen) return true; + if (t.selector.screen) { + const timeout = Number(args.timeout ?? args.timeoutMs ?? 5000) || 5000; + const gap = Number(args.time_gap ?? args.gapMs ?? 200) || 200; + const deadline = Date.now() + timeout; + do { + if (await screenElementExists(t)) return true; + if (Date.now() >= deadline) break; + await new Promise(resolve => setTimeout(resolve, gap)); + } while (Date.now() < deadline); + return false; + } const timeout = Number(args.timeout ?? args.timeoutMs ?? 5000) || 5000; const gap = Number(args.time_gap ?? args.gapMs ?? 200) || 200; const found = await findElement(t, timeout, gap); diff --git a/src/main/desktop-pick.ts b/src/main/desktop-pick.ts index 08a56d3..ce923bd 100644 --- a/src/main/desktop-pick.ts +++ b/src/main/desktop-pick.ts @@ -373,6 +373,77 @@ async function capturePaneForAi(rect: { x: number; y: number; w: number; h: numb } } +interface AiUiaInspection { + found: boolean; + name?: string; + id?: string; + type?: string; + className?: string; + x?: number; + y?: number; + w?: number; + h?: number; + topName?: string; + topClassName?: string; + process?: string; + topX?: number; + topY?: number; + topW?: number; + topH?: number; +} + +async function inspectAiSelection( + rect: { x: number; y: number; w: number; h: number }, + overlay: BrowserWindow +): Promise { + // UIA FromPoint 会拿到透明覆盖层，必须先隐藏并给它一帧退出屏幕的时间。 + if (!overlay.isDestroyed()) overlay.hide(); + await new Promise(resolve => setTimeout(resolve, 90)); + const cx = Math.round(rect.x + rect.w / 2); + const cy = Math.round(rect.y + rect.h / 2); + const script = [ + "Add-Type -AssemblyName UIAutomationClient, UIAutomationTypes | Out-Null;", + "try { Add-Type -MemberDefinition '[DllImport(\"user32.dll\")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value);' -Name AiDpi -Namespace RpaAiPick | Out-Null; [RpaAiPick.AiDpi]::SetProcessDpiAwarenessContext([IntPtr](-4)) | Out-Null } catch {}", + "$pt = New-Object System.Windows.Point(" + cx + "," + cy + ")", + "Add-Type -AssemblyName WindowsBase | Out-Null", + "$el = [System.Windows.Automation.AutomationElement]::FromPoint($pt)", + "if ($null -eq $el -or $el.Current.ProcessId -eq 0) { Write-Output '{\"found\":false}'; exit 0 }", + "$walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker", + "$top = $el", + "for ($i = 0; $i -lt 80; $i++) {", + " $parent = $walker.GetParent($top)", + " if ($null -eq $parent -or $parent.Current.ProcessId -eq 0) { break }", + " $top = $parent", + "}", + "$er = $el.Current.BoundingRectangle", + "$tr = $top.Current.BoundingRectangle", + "$controlType = ''", + "try { $controlType = $el.Current.ControlType.ProgrammaticName -replace '^ControlType\\.', '' } catch {}", + "$procName = ''", + "try { $procName = (Get-Process -Id $top.Current.ProcessId -ErrorAction Stop).ProcessName } catch {}", + "$info = @{", + " found = $true", + " name = [string]$el.Current.Name", + " id = [string]$el.Current.AutomationId", + " type = [string]$controlType", + " className = [string]$el.Current.ClassName", + " x = [int]$er.X; y = [int]$er.Y; w = [int]$er.Width; h = [int]$er.Height", + " topName = [string]$top.Current.Name", + " topClassName = [string]$top.Current.ClassName", + " process = [string]$procName", + " topX = [int]$tr.X; topY = [int]$tr.Y; topW = [int]$tr.Width; topH = [int]$tr.Height", + "}", + "Write-Output ($info | ConvertTo-Json -Compress)" + ].join("\n"); + try { + const { stdout } = await run("powershell.exe", ["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-Command", script], { timeout: 8000, windowsHide: true }); + return jsonLine(stdout, "{"); + } catch (error) { + console.warn("[pane-ai] uia enrichment failed", error); + return null; + } +} + async function runPaneAiPick(context: UiaPickOutput): Promise { // 本地小模型是 Pane AI 的主链路；OCR 使用同一张 Pane 截图，不要求用户配置远程 VLM。 const localModelPath = localUiModelPath(); @@ -543,23 +614,59 @@ async function runPaneAiPick(context: UiaPickOutput): Promise await detected.ocrTask; const box = candidates.find(item => item.candidateId === selectedId)!; - const x = detected.capture.cropX + box.x; - const y = detected.capture.cropY + box.y; + const aiRect = { + x: detected.capture.cropX + box.x, + y: detected.capture.cropY + box.y, + w: box.width, + h: box.height + }; + + // 用户确认后再查 UIA；覆盖层已隐藏，避免 FromPoint 命中拾取浮层。 + const inspection = await inspectAiSelection(aiRect, win); + const ir = inspection?.found ? inspection : null; + const intersectsAi = ir !== null && + typeof ir.x === "number" && typeof ir.y === "number" && + typeof ir.w === "number" && typeof ir.h === "number" && + ir.w > 0 && ir.h > 0 && + Math.min(aiRect.x + aiRect.w, ir.x + ir.w) - Math.max(aiRect.x, ir.x) > 2 && + Math.min(aiRect.y + aiRect.h, ir.y + ir.h) - Math.max(aiRect.y, ir.y) > 2; + const x = intersectsAi ? Math.round(ir!.x!) : Math.round(aiRect.x); + const y = intersectsAi ? Math.round(ir!.y!) : Math.round(aiRect.y); + const w = intersectsAi ? Math.round(ir!.w!) : Math.round(aiRect.w); + const h = intersectsAi ? Math.round(ir!.h!) : Math.round(aiRect.h); + const topName = ir?.topName || context.window; + const hasTopRect = ir?.topW !== undefined && ir.topW > 0 && ir.topH !== undefined && ir.topH > 0; + const rel = hasTopRect + ? [Math.round(x - ir!.topX!), Math.round(y - ir!.topY!), w, h].join(",") + : undefined; + const stableName = ir?.name?.trim(); + const stableId = ir?.id?.trim(); + const stableClass = ir?.className?.trim(); + const stableType = ir?.type?.trim(); + const enriched = Boolean(ir && (stableId || stableName || stableClass)); const selectorParts = [ - "screen=" + [x, y, box.width, box.height].map(Math.round).join(","), - context.window ? "window=" + safeSelectorText(context.window) : "", - "name=" + safeSelectorText(box.name || box.ocrText || box.controlType), - "type=" + safeSelectorText(box.controlType), + "screen=" + [x, y, w, h].map(Math.round).join(","), + topName ? "window=" + safeSelectorText(topName) : "", + ir?.topClassName ? "winClass=" + safeSelectorText(ir.topClassName) : "", + ir?.process ? "process=" + safeSelectorText(ir.process) : "", + rel ? "rel=" + rel : "", + stableName ? "name=" + safeSelectorText(stableName) : (box.name || box.ocrText ? "name=" + safeSelectorText(box.name || box.ocrText) : ""), + stableId ? "id=" + safeSelectorText(stableId) : "", + enriched && stableType ? "type=" + safeSelectorText(stableType) : "", + stableClass ? "class=" + safeSelectorText(stableClass) : "", + enriched ? "enriched=1" : "", "aiType=" + safeSelectorText(box.controlType), box.ocrText ? "aiText=" + safeSelectorText(box.ocrText) : "", "ai=1" ].filter(Boolean); return { source: "ai", - window: context.window, - name: box.name || box.ocrText || box.controlType, - controlType: box.controlType, - x, y, w: box.width, h: box.height, + window: topName, + name: stableName || box.name || box.ocrText || box.controlType, + automationId: stableId, + controlType: stableType || box.controlType, + className: stableClass, + x, y, w, h, confidence: box.confidence, screenshotPath: detected.capture.imagePath, selector: selectorParts.join(";") @@ -654,3 +761,4 @@ export async function pickScreenRegion(): Promise<{ x: number; y: number; width: +