diff --git a/resources/desktop-picker.ps1 b/resources/desktop-picker.ps1 new file mode 100644 index 0000000..b4ab039 --- /dev/null +++ b/resources/desktop-picker.ps1 @@ -0,0 +1,72 @@ +param([string]$Mode = "element") +$ErrorActionPreference = "Continue" +Add-Type -AssemblyName UIAutomationClient, UIAutomationTypes +Add-Type -AssemblyName System.Windows.Forms +Add-Type -MemberDefinition "[DllImport(`"user32.dll`")] public static extern short GetAsyncKeyState(int key); [DllImport(`"user32.dll`", CharSet = CharSet.Unicode)] public static extern int GetWindowTextW(IntPtr h, System.Text.StringBuilder sb, int max);" -Name PickW32 -Namespace RpaPick | Out-Null + +if ($Mode -eq "windows") { + $wins = @() + foreach ($p in Get-Process) { + if ($p.MainWindowHandle -ne [IntPtr]::Zero -and $p.MainWindowTitle) { + $wins += @{ title = $p.MainWindowTitle; process = $p.ProcessName } + } + } + Write-Output (ConvertTo-Json $wins -Compress -AsArray) + exit 0 +} + +$form = New-Object System.Windows.Forms.Form +$form.FormBorderStyle = [System.Windows.Forms.FormBorderStyle]::None +$form.TopMost = $true +$form.StartPosition = [System.Windows.Forms.FormStartPosition]::Manual +$form.Location = New-Object System.Drawing.Point(20, 20) +$form.Size = New-Object System.Drawing.Size(460, 130) +$form.BackColor = [System.Drawing.Color]::FromArgb(20, 60, 120) +$label = New-Object System.Windows.Forms.Label +$label.Dock = [System.Windows.Forms.DockStyle]::Fill +$label.ForeColor = [System.Drawing.Color]::White +$label.Font = New-Object System.Drawing.Font("Microsoft YaHei", 11) +$label.Text = "移动鼠标选择元素..." +$form.Controls.Add($label) +$form.Show() + +$prevDown = $false +$result = $null +while ($true) { + $ptX = [System.Windows.Forms.Cursor]::Position.X + $ptY = [System.Windows.Forms.Cursor]::Position.Y + $point = New-Object System.Windows.Automation.Point($ptX, $ptY) + $el = $null + try { $el = [System.Windows.Automation.AutomationElement]::FromPoint($point) } catch {} + $down = ([RpaPick.PickW32]::GetAsyncKeyState(32) -band 0x8000) -ne 0 + $esc = ([RpaPick.PickW32]::GetAsyncKeyState(27) -band 0x8000) -ne 0 + if ($esc) { break } + if ($el) { + $c = $el.Current + $ctrlName = $c.ControlType.ProgrammaticName.Replace("ControlType.", "") + $label.Text = "名称: " + $c.Name + [Environment]::NewLine + "类型: " + $ctrlName + [Environment]::NewLine + "空格=拾取 Esc=取消" + } + if ($down -and $el) { + $c = $el.Current + $rootName = "" + $walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker + $anc = $walker.GetParent($el) + while ($anc -and -not $anc.Equals([System.Windows.Automation.AutomationElement]::RootElement)) { + $h = $anc.Current.NativeWindowHandle + if ($h -ne 0) { + $sb = New-Object System.Text.StringBuilder 512 + [RpaPick.PickW32]::GetWindowTextW([IntPtr]$h, $sb, 512) | Out-Null + if ($sb.ToString()) { $rootName = $sb.ToString(); break } + } + $anc = $walker.GetParent($anc) + } + $ctrlName = $c.ControlType.ProgrammaticName.Replace("ControlType.", "") + $result = @{ window = $rootName; name = $c.Name; automationId = $c.AutomationId; controlType = $ctrlName; className = $c.ClassName } + break + } + $prevDown = $down + Start-Sleep -Milliseconds 80 +} +$form.Close() + +if ($result) { Write-Output (ConvertTo-Json $result -Compress) } else { Write-Output (ConvertTo-Json @{ cancelled = $true } -Compress) } diff --git a/src/main/desktop-pick.ts b/src/main/desktop-pick.ts new file mode 100644 index 0000000..0513c89 --- /dev/null +++ b/src/main/desktop-pick.ts @@ -0,0 +1,106 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { app, BrowserWindow, screen } from "electron"; +import { RuntimeError } from "./engine/types"; + +const run = promisify(execFile); + +function pickerScriptPath(): string { + const candidates = [ + path.join(process.resourcesPath || "", "desktop-picker.ps1"), + path.join(app.getAppPath(), "resources", "desktop-picker.ps1") + ].filter(Boolean); + const found = candidates.find(p => fs.existsSync(p)); + if (!found) throw new RuntimeError("picker_unavailable", "桌面拾取脚本缺失", { recommendation: "重新安装应用" }); + return found; +} +export interface DesktopPickResult { + cancelled?: boolean; + window?: string; + name?: string; + automationId?: string; + controlType?: string; + className?: string; + selector: string; +} + +export async function pickDesktopElement(): Promise { + const { stdout } = await run("powershell.exe", ["-NoProfile", "-STA", "-ExecutionPolicy", "Bypass", "-File", pickerScriptPath()], { timeout: 180000, windowsHide: false, maxBuffer: 4 * 1024 * 1024 }); + const jsonLine = stdout.trim().split(/\\r?\\n/).reverse().find(l => l.trim().startsWith("{")); + if (!jsonLine) throw new RuntimeError("picker_failed", "拾取未返回结果"); + const parsed = JSON.parse(jsonLine) as Omit; + if (parsed.cancelled) return { ...parsed, selector: "" }; + const parts: string[] = []; + if (parsed.window) parts.push("window=" + parsed.window); + if (parsed.name) parts.push("name=" + parsed.name); + if (parsed.controlType) parts.push("type=" + parsed.controlType); + if (parsed.automationId) parts.push("id=" + parsed.automationId); + return { ...parsed, selector: parts.join(";") }; +} + +export async function listVisibleWindows(): Promise> { + const { stdout } = await run("powershell.exe", ["-NoProfile", "-ExecutionPolicy", "Bypass", "-File", pickerScriptPath(), "-Mode", "windows"], { timeout: 30000, windowsHide: true, maxBuffer: 4 * 1024 * 1024 }); + const jsonLine = stdout.trim().split(/\\r?\\n/).reverse().find(l => l.trim().startsWith("[")); + if (!jsonLine) return []; + return JSON.parse(jsonLine) as Array<{ title: string; process: string }>; +} +export async function pickScreenRegion(): Promise<{ x: number; y: number; width: number; height: number; imagePath: string }> { + const display = screen.getPrimaryDisplay(); + const sx = display.bounds.x, sy = display.bounds.y, sw = display.bounds.width, sh = display.bounds.height; + const win = new BrowserWindow({ + x: sx, y: sy, width: sw, height: sh, + frame: false, transparent: true, alwaysOnTop: true, skipTaskbar: true, resizable: false, movable: false, + webPreferences: { contextIsolation: true, nodeIntegration: false, sandbox: true } + }); + const htmlParts = [ + "", + "
拖拽框选区域，松开鼠标完成；Esc 取消
", + "" + ]; + void win.loadURL("data:text/html;charset=utf-8," + encodeURIComponent(htmlParts.join(""))); + const deadline = Date.now() + 60000; + let region: { x: number; y: number; width: number; height: number } | null = null; + while (Date.now() < deadline) { + await new Promise(r => setTimeout(r, 250)); + try { + const state = await win.webContents.executeJavaScript("JSON.stringify({ done: window.__done === true, cancelled: window.__cancelled === true, region: window.__region || null })", true); + const parsed = JSON.parse(state) as { done: boolean; cancelled: boolean; region: { x: number; y: number; width: number; height: number } | null }; + if (parsed.cancelled) { win.destroy(); throw new RuntimeError("pick_cancelled", "已取消框选"); } + if (parsed.done && parsed.region && parsed.region.width > 2 && parsed.region.height > 2) { region = parsed.region; break; } + if (parsed.done) break; + } catch (error) { + if ((error as Error).message.includes("已取消框选")) throw error; + } + } + win.destroy(); + if (!region) throw new RuntimeError("pick_cancelled", "未完成框选"); + await new Promise(r => setTimeout(r, 300)); + const outDir = "D:/RPA/test-output"; + fs.mkdirSync(outDir, { recursive: true }); + const imagePath = outDir + "/pick-region-" + Date.now() + ".png"; + const psScript = [ + "Add-Type -AssemblyName System.Drawing | Out-Null;", + " = New-Object System.Drawing.Bitmap " + region.width + ", " + region.height + ";", + " = [System.Drawing.Graphics]::FromImage();", + ".CopyFromScreen(" + region.x + ", " + region.y + ", 0, 0, .Size);", + ".Dispose();", + ".Save(" + JSON.stringify(imagePath) + ");", + ".Dispose();", + "Write-Output ok" + ].join(""); + await run("powershell.exe", ["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-Command", psScript], { timeout: 30000, windowsHide: true }); + return { ...region, imagePath }; +} \ No newline at end of file diff --git a/src/preload/index.ts b/src/preload/index.ts index 07f71cf..9ee136c 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -8,6 +8,11 @@ const api = { catalog: { get: (): Promise => ipcRenderer.invoke("catalog:get") }, + desktop: { + pickElement: () => ipcRenderer.invoke('desktop:pick-element'), + pickWindowList: () => ipcRenderer.invoke('desktop:pick-window-list'), + pickRegion: () => ipcRenderer.invoke('desktop:pick-region') + }, flows: { list: (): Promise => ipcRenderer.invoke("flows:list"), create: (name?: string): Promise => ipcRenderer.invoke("flows:create", name), diff --git a/src/renderer/src/panels/Properties.tsx b/src/renderer/src/panels/Properties.tsx index 0e5bae4..724aa6f 100644 --- a/src/renderer/src/panels/Properties.tsx +++ b/src/renderer/src/panels/Properties.tsx @@ -468,6 +468,7 @@ function FieldControl({ } }; + const pickElement = async () => { setPicking(true); try {