param( [string]$Mode = "element", [int]$CaptureX = 0, [int]$CaptureY = 0, [int]$CaptureWidth = 1000, [int]$CaptureHeight = 700, [string]$CapturePath = "", [switch]$CaptureExact ) $ErrorActionPreference = "Continue"; [Console]::OutputEncoding = [System.Text.Encoding]::UTF8; $logPath = "D:/RPA/test-output/desktop-picker.log"; function Write-PickLog([string]$msg) { try { $logDirectory = Split-Path -Parent $logPath; if (-not (Test-Path -LiteralPath $logDirectory)) { New-Item -ItemType Directory -Path $logDirectory -Force | Out-Null }; Add-Content -Path $logPath -Value ((Get-Date -Format "HH:mm:ss.fff") + " | " + $msg) -Encoding UTF8 } catch {} } Write-PickLog ("=== start Mode=$Mode ===") Add-Type -AssemblyName UIAutomationClient, UIAutomationTypes, WindowsBase Add-Type -MemberDefinition '[DllImport("user32.dll")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value);' -Name PickDpi -Namespace RpaPick | Out-Null try { [RpaPick.PickDpi]::SetProcessDpiAwarenessContext([IntPtr](-4)) | Out-Null } catch {} Write-PickLog "dpi aware set" Add-Type -AssemblyName System.Windows.Forms Write-PickLog "assemblies ok" Add-Type -MemberDefinition '[DllImport("user32.dll")] public static extern short GetAsyncKeyState(int key); [DllImport("user32.dll", CharSet = CharSet.Unicode)] public static extern int GetWindowTextW(IntPtr h, System.Text.StringBuilder sb, int max); [DllImport("user32.dll")] public static extern int GetWindowLong(IntPtr hWnd, int nIndex); [DllImport("user32.dll")] public static extern int SetWindowLong(IntPtr hWnd, int nIndex, int dwNewLong);' -Name PickW32 -Namespace RpaPick | Out-Null Write-PickLog "w32 ok" if ($Mode -eq "capture") { try { [RpaPick.PickDpi]::SetProcessDpiAwarenessContext([IntPtr](-4)) | Out-Null } catch {} Add-Type -AssemblyName System.Drawing Add-Type -MemberDefinition '[DllImport("user32.dll")] public static extern int GetSystemMetrics(int nIndex);' -Name PickMetrics -Namespace RpaPickCapture | Out-Null $vsX = [RpaPickCapture.PickMetrics]::GetSystemMetrics(76) $vsY = [RpaPickCapture.PickMetrics]::GetSystemMetrics(77) $vsW = [RpaPickCapture.PickMetrics]::GetSystemMetrics(78) $vsH = [RpaPickCapture.PickMetrics]::GetSystemMetrics(79) $cropW = [Math]::Min($CaptureWidth, $vsW) $cropH = [Math]::Min($CaptureHeight, $vsH) if ($CaptureExact) { $cropX = [Math]::Max($vsX, [Math]::Min($CaptureX, ($vsX + $vsW - $cropW))) $cropY = [Math]::Max($vsY, [Math]::Min($CaptureY, ($vsY + $vsH - $cropH))) } else { $cropX = [Math]::Max($vsX, [Math]::Min($CaptureX - [int]($cropW / 2), ($vsX + $vsW - $cropW))) $cropY = [Math]::Max($vsY, [Math]::Min($CaptureY - [int]($cropH / 2), ($vsY + $vsH - $cropH))) } $markerX = $CaptureX - $cropX $markerY = $CaptureY - $cropY if ($CapturePath) { $imagePath = $CapturePath } else { $imagePath = Join-Path $env:TEMP ("ai-pick-" + [Guid]::NewGuid().ToString("N") + ".png") } $imageDir = Split-Path -Parent $imagePath if (-not (Test-Path -LiteralPath $imageDir)) { New-Item -ItemType Directory -Path $imageDir -Force | Out-Null } $bitmap = New-Object System.Drawing.Bitmap $cropW, $cropH $graphics = [System.Drawing.Graphics]::FromImage($bitmap) $graphics.CopyFromScreen($cropX, $cropY, 0, 0, $bitmap.Size) $pen = New-Object System.Drawing.Pen([System.Drawing.Color]::Red, 3) try { $graphics.SmoothingMode = [System.Drawing.Drawing2D.SmoothingMode]::AntiAlias if (-not $CaptureExact) { $graphics.DrawLine($pen, ($markerX - 20), $markerY, ($markerX + 20), $markerY) $graphics.DrawLine($pen, $markerX, ($markerY - 20), $markerX, ($markerY + 20)) $graphics.DrawRectangle($pen, ($markerX - 9), ($markerY - 9), 18, 18) } } finally { $pen.Dispose(); $graphics.Dispose() } $bitmap.Save($imagePath, [System.Drawing.Imaging.ImageFormat]::Png) $bitmap.Dispose() $windowTitle = "" $uiaName = "" $uiaType = "" try { $pt = New-Object System.Windows.Point($CaptureX, $CaptureY) $ae = [System.Windows.Automation.AutomationElement]::FromPoint($pt) if ($ae) { $ac = $ae.Current $uiaName = [string]$ac.Name $uiaType = $ac.ControlType.ProgrammaticName.Replace("ControlType.", "") $walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker $cur = $ae while ($cur -and -not $cur.Equals([System.Windows.Automation.AutomationElement]::RootElement)) { $windowTitle = [string]$cur.Current.Name $cur = $walker.GetParent($cur) } } } catch {} $result = @{ aiCapture = $true cursorX = $CaptureX cursorY = $CaptureY cropX = $cropX cropY = $cropY cropW = $cropW cropH = $cropH markerX = $markerX markerY = $markerY window = $windowTitle uiaName = $uiaName uiaType = $uiaType imagePath = $imagePath } Write-Output (ConvertTo-Json $result -Compress) exit 0 } if ($Mode -eq "windows") { $wins = @() foreach ($p in Get-Process) { if ($p.MainWindowHandle -ne [IntPtr]::Zero -and $p.MainWindowTitle) { $wins += @{ title = $p.MainWindowTitle; process = $p.ProcessName } } } Write-PickLog "windows: $($wins.Count)" Write-Output (ConvertTo-Json @($wins) -Compress) exit 0 } $info = New-Object System.Windows.Forms.Form $info.FormBorderStyle = [System.Windows.Forms.FormBorderStyle]::None $info.TopMost = $true $info.ShowInTaskbar = $false $info.StartPosition = [System.Windows.Forms.FormStartPosition]::Manual $sw2 = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds.Width $info.Location = New-Object System.Drawing.Point(([int]($sw2 / 2) - 280), 10) $info.Size = New-Object System.Drawing.Size(560, 64) $info.BackColor = [System.Drawing.Color]::FromArgb(15, 40, 90) $info.Opacity = 0.92 $infoLabel = New-Object System.Windows.Forms.Label $infoLabel.Dock = [System.Windows.Forms.DockStyle]::Fill $infoLabel.ForeColor = [System.Drawing.Color]::White $infoLabel.Font = New-Object System.Drawing.Font("Microsoft YaHei", 11) $infoLabel.TextAlign = [System.Drawing.ContentAlignment]::MiddleCenter $infoLabel.Text = "移动鼠标到目标元素上" $info.Controls.Add($infoLabel) $hl = New-Object System.Windows.Forms.Form $hl.FormBorderStyle = [System.Windows.Forms.FormBorderStyle]::None $hl.TopMost = $true $hl.ShowInTaskbar = $false $hl.StartPosition = [System.Windows.Forms.FormStartPosition]::Manual $hl.BackColor = [System.Drawing.Color]::Fuchsia $hl.TransparencyKey = [System.Drawing.Color]::Fuchsia $hl.Add_Paint({ param($sender, $e) $pen = New-Object System.Drawing.Pen([System.Drawing.Color]::FromArgb(255, 255, 32, 32), 3) try { $size = $sender.ClientSize $e.Graphics.DrawRectangle($pen, 1, 1, ($size.Width - 3), ($size.Height - 3)) } finally { $pen.Dispose() } }) function Set-ClickThrough($form) { if ($form.IsHandleCreated) { $GWL_EXSTYLE = -20 $WS_EX_LAYERED = 0x80000 $WS_EX_TRANSPARENT = 0x20 $WS_EX_NOACTIVATE = 0x08000000 $WS_EX_TOOLWINDOW = 0x80 $style = [RpaPick.PickW32]::GetWindowLong($form.Handle, $GWL_EXSTYLE) [void][RpaPick.PickW32]::SetWindowLong($form.Handle, $GWL_EXSTYLE, $style -bor $WS_EX_LAYERED -bor $WS_EX_TRANSPARENT -bor $WS_EX_NOACTIVATE -bor $WS_EX_TOOLWINDOW) } } function Set-HighlightBox([int]$x, [int]$y, [int]$w, [int]$h) { $bounds = New-Object System.Drawing.Rectangle(($x - 2), ($y - 2), ($w + 4), ($h + 4)) if (-not $hl.Visible) { $hl.Bounds = $bounds $hl.Show() Set-ClickThrough $hl } elseif ($hl.Bounds.X -ne $bounds.X -or $hl.Bounds.Y -ne $bounds.Y -or $hl.Bounds.Width -ne $bounds.Width -or $hl.Bounds.Height -ne $bounds.Height) { # 不重设隐藏;同一元素内移动时不重复改窗口,避免 WinForms 分层窗体重绘闪烁 $hl.Bounds = $bounds } } function Hide-Highlight { $hl.Hide() } function Get-DeepestAtPoint([double]$px, [double]$py) { $point = New-Object System.Windows.Point($px, $py) $cur = [System.Windows.Automation.AutomationElement]::FromPoint($point) if (-not $cur) { return $null } for ($depth = 0; $depth -lt 25; $depth++) { $children = $cur.FindAll([System.Windows.Automation.TreeScope]::Children, [System.Windows.Automation.Condition]::TrueCondition) $next = $null $nextArea = [double]::MaxValue foreach ($c in $children) { try { $r = $c.Current.BoundingRectangle if ($r.Width -gt 1 -and $r.Height -gt 1 -and $px -ge $r.X -and $px -le ($r.X + $r.Width) -and $py -ge $r.Y -and $py -le ($r.Y + $r.Height)) { $area = $r.Width * $r.Height if ($area -lt $nextArea) { $next = $c; $nextArea = $area } } } catch {} } if (-not $next) { break } $cur = $next } return $cur } $info.Show() Set-ClickThrough $info $stage = "hover" $chain = New-Object System.Collections.ArrayList $selIdx = -1 $prevSpace = $false $prevPgUp = $false $prevPgDn = $false $prevF2 = $false $prevF4 = $false $cacheX = [int]::MinValue $cacheY = [int]::MinValue $cacheEl = $null $cacheCtrlName = "" $cacheName = "" $cachePaneX = 0 $cachePaneY = 0 $cachePaneW = 0 $cachePaneH = 0 $cacheAt = [DateTime]::MinValue $lastQueryAt = [DateTime]::MinValue $queryGapMs = 45 $cacheTtlMs = 1800 Write-PickLog "loop start" while ($true) { try { $space = (([RpaPick.PickW32]::GetAsyncKeyState(32) -band 0x8001) -ne 0) $esc = (([RpaPick.PickW32]::GetAsyncKeyState(27) -band 0x8001) -ne 0) $pgUp = (([RpaPick.PickW32]::GetAsyncKeyState(33) -band 0x8001) -ne 0) $pgDn = (([RpaPick.PickW32]::GetAsyncKeyState(34) -band 0x8001) -ne 0) $f2 = (([RpaPick.PickW32]::GetAsyncKeyState(113) -band 0x8001) -ne 0) $f4 = (([RpaPick.PickW32]::GetAsyncKeyState(115) -band 0x8001) -ne 0) if ($esc) { Write-PickLog "esc"; break } if ($stage -eq "hover") { $ptX = [System.Windows.Forms.Cursor]::Position.X $ptY = [System.Windows.Forms.Cursor]::Position.Y $now = [DateTime]::UtcNow $samePoint = ($ptX -eq $cacheX -and $ptY -eq $cacheY) $insideCache = $cacheEl -ne $null -and $ptX -ge $cachePaneX -and $ptX -lt ($cachePaneX + $cachePaneW) -and $ptY -ge $cachePaneY -and $ptY -lt ($cachePaneY + $cachePaneH) $cacheAlive = ($null -ne $cacheEl -and ($now - $cacheAt).TotalMilliseconds -le $cacheTtlMs) # 鼠标还在当前元素矩形内部时不丢弃缓存;UIA 查询完成后才切换,避免移动过程先隐藏再显示 if (-not $samePoint -and -not $insideCache) { $cacheEl = $null } if (-not $cacheAlive -and ($now - $lastQueryAt).TotalMilliseconds -ge $queryGapMs) { $queryX = $ptX $queryY = $ptY $lastQueryAt = [DateTime]::UtcNow try { $queried = Get-DeepestAtPoint $queryX $queryY } catch { $queried = $null; Write-PickLog ("QUERY ERR: " + $_.Exception.Message) } $afterX = [System.Windows.Forms.Cursor]::Position.X $afterY = [System.Windows.Forms.Cursor]::Position.Y if ($afterX -ne $queryX -or $afterY -ne $queryY) { $queried = $null Write-PickLog "query stale: cursor moved" } if ($null -ne $queried) { $qc = $queried.Current $qr = $qc.BoundingRectangle $cacheEl = $queried $cacheX = $queryX $cacheY = $queryY $cacheAt = [DateTime]::UtcNow $cacheCtrlName = $qc.ControlType.ProgrammaticName.Replace("ControlType.", "") $cacheName = [string]$qc.Name $cachePaneX = [int]$qr.X $cachePaneY = [int]$qr.Y $cachePaneW = [int]$qr.Width $cachePaneH = [int]$qr.Height } else { $cacheEl = $null $cacheCtrlName = "" $cacheName = "" } } $el = $cacheEl if ($el) { Set-HighlightBox $cachePaneX $cachePaneY $cachePaneW $cachePaneH $aiHint = if ($cacheCtrlName -eq "Pane") { " [F2/F4]AI识别" } else { "" } $infoLabel.Text = "名称: " + $cacheName + " 类型: " + $cacheCtrlName + " [空格]锁定" + $aiHint + " [Esc]取消" } else { Hide-Highlight } $aiTrigger = ($f2 -and -not $prevF2) -or ($f4 -and -not $prevF4) if ($aiTrigger -and $el -and $cacheCtrlName -eq "Pane") { Write-PickLog "f2/f4 ai" $walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker $cur = $el $topTitle = "" while ($cur -and -not $cur.Equals([System.Windows.Automation.AutomationElement]::RootElement)) { $topTitle = [string]$cur.Current.Name $cur = $walker.GetParent($cur) } $result = @{ aiRequested = $true; cursorX = $ptX; cursorY = $ptY; window = $topTitle; name = $cacheName; controlType = $cacheCtrlName; paneX = $cachePaneX; paneY = $cachePaneY; paneW = $cachePaneW; paneH = $cachePaneH } break } if ($space -and -not $prevSpace -and $el) { Write-PickLog "space lock" $stage = "locked" $chain = New-Object System.Collections.ArrayList $walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker $cur = $cacheEl while ($cur -and -not $cur.Equals([System.Windows.Automation.AutomationElement]::RootElement)) { $c = $cur.Current $r = $c.BoundingRectangle [void]$chain.Insert(0, @{ el = $cur; name = [string]$c.Name; automationId = [string]$c.AutomationId; controlType = $c.ControlType.ProgrammaticName.Replace("ControlType.", ""); x = [int]$r.X; y = [int]$r.Y; w = [int]$r.Width; h = [int]$r.Height; topLevel = $walker.GetParent($cur).Equals([System.Windows.Automation.AutomationElement]::RootElement) }) $cur = $walker.GetParent($cur) } $selIdx = $chain.Count - 1 Write-PickLog ("locked chain=" + $chain.Count) $infoLabel.Text = "已锁定 [" + ($selIdx + 1) + "/" + $chain.Count + "] [PageUp/PageDown]调层级 [空格]确认回填 [Esc]取消" } } if ($stage -eq "locked") { $item = $chain[$selIdx] if ($item) { Set-HighlightBox $item.x $item.y $item.w $item.h $infoLabel.Text = "层级 " + ($selIdx + 1) + "/" + $chain.Count + " 名称: " + $item.name + " 类型: " + $item.controlType + " [PageUp/PageDown]调层级 [空格]确认 [Esc]取消" } if ($pgUp -and -not $prevPgUp -and $selIdx -gt 0) { $selIdx -= 1 } if ($pgDn -and -not $prevPgDn -and $selIdx -lt ($chain.Count - 1)) { $selIdx += 1 } if ($space -and -not $prevSpace) { $item = $chain[$selIdx] $rootItem = $chain | Where-Object { $_.topLevel -and $_.name } | Select-Object -First 1 if ($rootItem) { $rootTitle = $rootItem.name } else { foreach ($it in $chain) { if ($it.name) { $rootTitle = $it.name; break } } } $result = @{ selected = $selIdx; window = $rootTitle; name = $item.name; automationId = $item.automationId; controlType = $item.controlType; x = $item.x; y = $item.y; w = $item.w; h = $item.h } break } } } catch { Write-PickLog ("LOOP ERR: " + $_.Exception.Message) } [System.Windows.Forms.Application]::DoEvents() $prevSpace = $space $prevPgUp = $pgUp $prevPgDn = $pgDn $prevF2 = $f2 $prevF4 = $f4 Start-Sleep -Milliseconds 60 } $info.Close() $hl.Close() if ($result) { Write-Output (ConvertTo-Json $result -Compress) } else { Write-Output (ConvertTo-Json @{ cancelled = $true } -Compress) }