diff --git a/resources/desktop-picker.ps1 b/resources/desktop-picker.ps1 index 953096a..7046c2e 100644 --- a/resources/desktop-picker.ps1 +++ b/resources/desktop-picker.ps1 @@ -1,20 +1,21 @@ param([string]$Mode = "element") $ErrorActionPreference = "Continue"; +[Console]::OutputEncoding = [System.Text.Encoding]::UTF8; $logPath = "D:/RPA/test-output/desktop-picker.log"; function Write-PickLog([string]$msg) { - try { Add-Content -Path $logPath -Value ((Get-Date -Format "HH:mm:ss.fff") + " | " + $msg) -Encoding UTF8 } catch {} + try { $logDirectory = Split-Path -Parent $logPath; if (-not (Test-Path -LiteralPath $logDirectory)) { New-Item -ItemType Directory -Path $logDirectory -Force | Out-Null }; Add-Content -Path $logPath -Value ((Get-Date -Format "HH:mm:ss.fff") + " | " + $msg) -Encoding UTF8 } catch {} } Write-PickLog ("=== start Mode=$Mode ===") Add-Type -AssemblyName UIAutomationClient, UIAutomationTypes, WindowsBase Add-Type -AssemblyName System.Windows.Forms Write-PickLog "assemblies ok" -Add-Type -MemberDefinition '[DllImport("user32.dll")] public static extern short GetAsyncKeyState(int key); [DllImport("user32.dll", CharSet = CharSet.Unicode)] public static extern int GetWindowTextW(IntPtr h, System.Text.StringBuilder sb, int max);' -Name PickW32 -Namespace RpaPick | Out-Null +Add-Type -MemberDefinition '[DllImport("user32.dll")] public static extern short GetAsyncKeyState(int key); [DllImport("user32.dll", CharSet = CharSet.Unicode)] public static extern int GetWindowTextW(IntPtr h, System.Text.StringBuilder sb, int max); [DllImport("user32.dll")] public static extern int GetWindowLong(IntPtr hWnd, int nIndex); [DllImport("user32.dll")] public static extern int SetWindowLong(IntPtr hWnd, int nIndex, int dwNewLong);' -Name PickW32 -Namespace RpaPick | Out-Null Write-PickLog "w32 ok" if ($Mode -eq "windows") { $wins = @() foreach ($p in Get-Process) { if ($p.MainWindowHandle -ne [IntPtr]::Zero -and $p.MainWindowTitle) { $wins += @{ title = $p.MainWindowTitle; process = $p.ProcessName } } } - Write-PickLog "windows: $($_wins.Count)" - Write-Output (ConvertTo-Json $wins -Compress -AsArray) + Write-PickLog "windows: $($wins.Count)" + Write-Output (ConvertTo-Json @($wins) -Compress) exit 0 } $info = New-Object System.Windows.Forms.Form @@ -41,10 +42,23 @@ $hl.ShowInTaskbar = $false $hl.StartPosition = [System.Windows.Forms.FormStartPosition]::Manual $hl.BackColor = [System.Drawing.Color]::Red $hl.Opacity = 0.35 +function Set-ClickThrough($form) { + if ($form.IsHandleCreated) { + $GWL_EXSTYLE = -20 + $WS_EX_LAYERED = 0x80000 + $WS_EX_TRANSPARENT = 0x20 + $WS_EX_NOACTIVATE = 0x08000000 + $WS_EX_TOOLWINDOW = 0x80 + $style = [RpaPick.PickW32]::GetWindowLong($form.Handle, $GWL_EXSTYLE) + [void][RpaPick.PickW32]::SetWindowLong($form.Handle, $GWL_EXSTYLE, $style -bor $WS_EX_LAYERED -bor $WS_EX_TRANSPARENT -bor $WS_EX_NOACTIVATE -bor $WS_EX_TOOLWINDOW) + } +} + function Set-HighlightBox([int]$x, [int]$y, [int]$w, [int]$h) { $hl.Location = New-Object System.Drawing.Point(($x - 2), ($y - 2)) $hl.Size = New-Object System.Drawing.Size(($w + 4), ($h + 4)) $hl.Show() + Set-ClickThrough $hl } function Hide-Highlight { $hl.Hide() } function Get-DeepestAtPoint([double]$px, [double]$py) { @@ -70,6 +84,7 @@ function Get-DeepestAtPoint([double]$px, [double]$py) { return $cur } $info.Show() +Set-ClickThrough $info $stage = "hover" $chain = New-Object System.Collections.ArrayList $selIdx = -1 @@ -91,7 +106,7 @@ while ($true) { if ($el) { $c = $el.Current $ctrlName = $c.ControlType.ProgrammaticName.Replace("ControlType.", "") - Set-HighlightBox ([int]$c.Current.BoundingRectangle.X) ([int]$c.Current.BoundingRectangle.Y) ([int]$c.Current.BoundingRectangle.Width) ([int]$c.Current.BoundingRectangle.Height) + $r = $c.BoundingRectangle; Set-HighlightBox ([int]$r.X) ([int]$r.Y) ([int]$r.Width) ([int]$r.Height) $infoLabel.Text = "名称: " + $c.Name + " 类型: " + $ctrlName + " [空格]锁定 [Esc]取消" } else { Hide-Highlight } if ($space -and -not $prevSpace -and $el) { @@ -103,7 +118,7 @@ while ($true) { while ($cur -and -not $cur.Equals([System.Windows.Automation.AutomationElement]::RootElement)) { $c = $cur.Current $r = $c.BoundingRectangle - [void]$chain.Insert(0, @{ el = $cur; name = [string]$c.Name; automationId = [string]$c.AutomationId; controlType = $c.ControlType.ProgrammaticName.Replace("ControlType.", ""); x = [int]$r.X; y = [int]$r.Y; w = [int]$r.Width; h = [int]$r.Height }) + [void]$chain.Insert(0, @{ el = $cur; name = [string]$c.Name; automationId = [string]$c.AutomationId; controlType = $c.ControlType.ProgrammaticName.Replace("ControlType.", ""); x = [int]$r.X; y = [int]$r.Y; w = [int]$r.Width; h = [int]$r.Height; topLevel = $walker.GetParent($cur).Equals([System.Windows.Automation.AutomationElement]::RootElement) }) $cur = $walker.GetParent($cur) } $selIdx = $chain.Count - 1 @@ -121,8 +136,8 @@ while ($true) { if ($pgDn -and -not $prevPgDn -and $selIdx -lt ($chain.Count - 1)) { $selIdx += 1 } if ($space -and -not $prevSpace) { $item = $chain[$selIdx] - $rootTitle = "" - foreach ($it in $chain) { if ($it.name) { $rootTitle = $it.name; break } } + $rootItem = $chain | Where-Object { $_.topLevel -and $_.name } | Select-Object -First 1 + if ($rootItem) { $rootTitle = $rootItem.name } else { foreach ($it in $chain) { if ($it.name) { $rootTitle = $it.name; break } } } $result = @{ selected = $selIdx; window = $rootTitle; name = $item.name; automationId = $item.automationId; controlType = $item.controlType; x = $item.x; y = $item.y; w = $item.w; h = $item.h } break } @@ -131,8 +146,11 @@ while ($true) { Write-PickLog ("LOOP ERR: " + $_.Exception.Message) } [System.Windows.Forms.Application]::DoEvents() + $prevSpace = $space + $prevPgUp = $pgUp + $prevPgDn = $pgDn Start-Sleep -Milliseconds 60 } $info.Close() $hl.Close() -if ($result) { Write-Output (ConvertTo-Json $result -Compress) } else { Write-Output (ConvertTo-Json @{ cancelled = $true } -Compress) } \ No newline at end of file +if ($result) { Write-Output (ConvertTo-Json $result -Compress) } else { Write-Output (ConvertTo-Json @{ cancelled = $true } -Compress) } diff --git a/src/main/desktop-pick.ts b/src/main/desktop-pick.ts index 89f9bcc..beb6f1e 100644 --- a/src/main/desktop-pick.ts +++ b/src/main/desktop-pick.ts @@ -30,19 +30,24 @@ export interface DesktopPickResult { export async function pickDesktopElement(): Promise { const appWindows = BrowserWindow.getAllWindows().filter(w => !w.isDestroyed() && w.isVisible()); for (const w of appWindows) w.minimize(); - const { stdout } = await run("powershell.exe", ["-NoProfile", "-STA", "-ExecutionPolicy", "Bypass", "-File", pickerScriptPath()], { timeout: 180000, windowsHide: false, maxBuffer: 4 * 1024 * 1024 }); - const jsonLine = stdout.trim().split(/\r?\n/).reverse().find(l => l.trim().startsWith("{")); - if (!jsonLine) throw new RuntimeError("picker_failed", "拾取未返回结果"); - const parsed = JSON.parse(jsonLine) as Omit; - if (parsed.cancelled) return { ...parsed, selector: "" }; - const parts: string[] = []; - if (parsed.window) parts.push("window=" + parsed.window); - if (parsed.name) parts.push("name=" + parsed.name); - if (parsed.controlType) parts.push("type=" + parsed.controlType); - if (parsed.automationId) parts.push("id=" + parsed.automationId); - return { ...parsed, selector: parts.join(";") }; + try { + const { stdout } = await run("powershell.exe", ["-NoProfile", "-STA", "-ExecutionPolicy", "Bypass", "-File", pickerScriptPath()], { timeout: 180000, windowsHide: true, maxBuffer: 4 * 1024 * 1024 }); + const jsonLine = stdout.trim().split(/\r?\n/).reverse().find(l => l.trim().startsWith("{")); + if (!jsonLine) throw new RuntimeError("picker_failed", "拾取未返回结果"); + const parsed = JSON.parse(jsonLine) as Omit; + if (parsed.cancelled) return { ...parsed, selector: "" }; + const parts: string[] = []; + if (parsed.window) parts.push("window=" + parsed.window); + if (parsed.name) parts.push("name=" + parsed.name); + if (parsed.controlType) parts.push("type=" + parsed.controlType); + if (parsed.automationId) parts.push("id=" + parsed.automationId); + return { ...parsed, selector: parts.join(";") }; + } finally { + for (const w of appWindows) { + if (!w.isDestroyed()) w.restore(); + } + } } - export async function listVisibleWindows(): Promise> { const { stdout } = await run("powershell.exe", ["-NoProfile", "-ExecutionPolicy", "Bypass", "-File", pickerScriptPath(), "-Mode", "windows"], { timeout: 30000, windowsHide: true, maxBuffer: 4 * 1024 * 1024 }); const jsonLine = stdout.trim().split(/\r?\n/).reverse().find(l => l.trim().startsWith("["));