feat: 桌面拾取器——UIA 元素拾取（空格确认/Esc 取消）、窗口列表、屏幕区域框选截图，属性面板按能力类型切换网页/桌面拾取
This commit is contained in:
yu
parent
1e649d7121
commit
8b0004bf72
72
resources/desktop-picker.ps1
Normal file
72
resources/desktop-picker.ps1
Normal file
@ -0,0 +1,72 @@
|
||||
param([string]$Mode = "element")
|
||||
$ErrorActionPreference = "Continue"
|
||||
Add-Type -AssemblyName UIAutomationClient, UIAutomationTypes
|
||||
Add-Type -AssemblyName System.Windows.Forms
|
||||
Add-Type -MemberDefinition "[DllImport(`"user32.dll`")] public static extern short GetAsyncKeyState(int key); [DllImport(`"user32.dll`", CharSet = CharSet.Unicode)] public static extern int GetWindowTextW(IntPtr h, System.Text.StringBuilder sb, int max);" -Name PickW32 -Namespace RpaPick | Out-Null
|
||||
|
||||
if ($Mode -eq "windows") {
|
||||
$wins = @()
|
||||
foreach ($p in Get-Process) {
|
||||
if ($p.MainWindowHandle -ne [IntPtr]::Zero -and $p.MainWindowTitle) {
|
||||
$wins += @{ title = $p.MainWindowTitle; process = $p.ProcessName }
|
||||
}
|
||||
}
|
||||
Write-Output (ConvertTo-Json $wins -Compress -AsArray)
|
||||
exit 0
|
||||
}
|
||||
|
||||
$form = New-Object System.Windows.Forms.Form
|
||||
$form.FormBorderStyle = [System.Windows.Forms.FormBorderStyle]::None
|
||||
$form.TopMost = $true
|
||||
$form.StartPosition = [System.Windows.Forms.FormStartPosition]::Manual
|
||||
$form.Location = New-Object System.Drawing.Point(20, 20)
|
||||
$form.Size = New-Object System.Drawing.Size(460, 130)
|
||||
$form.BackColor = [System.Drawing.Color]::FromArgb(20, 60, 120)
|
||||
$label = New-Object System.Windows.Forms.Label
|
||||
$label.Dock = [System.Windows.Forms.DockStyle]::Fill
|
||||
$label.ForeColor = [System.Drawing.Color]::White
|
||||
$label.Font = New-Object System.Drawing.Font("Microsoft YaHei", 11)
|
||||
$label.Text = "移动鼠标选择元素..."
|
||||
$form.Controls.Add($label)
|
||||
$form.Show()
|
||||
|
||||
$prevDown = $false
|
||||
$result = $null
|
||||
while ($true) {
|
||||
$ptX = [System.Windows.Forms.Cursor]::Position.X
|
||||
$ptY = [System.Windows.Forms.Cursor]::Position.Y
|
||||
$point = New-Object System.Windows.Automation.Point($ptX, $ptY)
|
||||
$el = $null
|
||||
try { $el = [System.Windows.Automation.AutomationElement]::FromPoint($point) } catch {}
|
||||
$down = ([RpaPick.PickW32]::GetAsyncKeyState(32) -band 0x8000) -ne 0
|
||||
$esc = ([RpaPick.PickW32]::GetAsyncKeyState(27) -band 0x8000) -ne 0
|
||||
if ($esc) { break }
|
||||
if ($el) {
|
||||
$c = $el.Current
|
||||
$ctrlName = $c.ControlType.ProgrammaticName.Replace("ControlType.", "")
|
||||
$label.Text = "名称: " + $c.Name + [Environment]::NewLine + "类型: " + $ctrlName + [Environment]::NewLine + "空格=拾取 Esc=取消"
|
||||
}
|
||||
if ($down -and $el) {
|
||||
$c = $el.Current
|
||||
$rootName = ""
|
||||
$walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker
|
||||
$anc = $walker.GetParent($el)
|
||||
while ($anc -and -not $anc.Equals([System.Windows.Automation.AutomationElement]::RootElement)) {
|
||||
$h = $anc.Current.NativeWindowHandle
|
||||
if ($h -ne 0) {
|
||||
$sb = New-Object System.Text.StringBuilder 512
|
||||
[RpaPick.PickW32]::GetWindowTextW([IntPtr]$h, $sb, 512) | Out-Null
|
||||
if ($sb.ToString()) { $rootName = $sb.ToString(); break }
|
||||
}
|
||||
$anc = $walker.GetParent($anc)
|
||||
}
|
||||
$ctrlName = $c.ControlType.ProgrammaticName.Replace("ControlType.", "")
|
||||
$result = @{ window = $rootName; name = $c.Name; automationId = $c.AutomationId; controlType = $ctrlName; className = $c.ClassName }
|
||||
break
|
||||
}
|
||||
$prevDown = $down
|
||||
Start-Sleep -Milliseconds 80
|
||||
}
|
||||
$form.Close()
|
||||
|
||||
if ($result) { Write-Output (ConvertTo-Json $result -Compress) } else { Write-Output (ConvertTo-Json @{ cancelled = $true } -Compress) }
|
||||
106
src/main/desktop-pick.ts
Normal file
106
src/main/desktop-pick.ts
Normal file
@ -0,0 +1,106 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { app, BrowserWindow, screen } from "electron";
|
||||
import { RuntimeError } from "./engine/types";
|
||||
|
||||
const run = promisify(execFile);
|
||||
|
||||
function pickerScriptPath(): string {
|
||||
const candidates = [
|
||||
path.join(process.resourcesPath || "", "desktop-picker.ps1"),
|
||||
path.join(app.getAppPath(), "resources", "desktop-picker.ps1")
|
||||
].filter(Boolean);
|
||||
const found = candidates.find(p => fs.existsSync(p));
|
||||
if (!found) throw new RuntimeError("picker_unavailable", "桌面拾取脚本缺失", { recommendation: "重新安装应用" });
|
||||
return found;
|
||||
}
|
||||
export interface DesktopPickResult {
|
||||
cancelled?: boolean;
|
||||
window?: string;
|
||||
name?: string;
|
||||
automationId?: string;
|
||||
controlType?: string;
|
||||
className?: string;
|
||||
selector: string;
|
||||
}
|
||||
|
||||
export async function pickDesktopElement(): Promise<DesktopPickResult> {
|
||||
const { stdout } = await run("powershell.exe", ["-NoProfile", "-STA", "-ExecutionPolicy", "Bypass", "-File", pickerScriptPath()], { timeout: 180000, windowsHide: false, maxBuffer: 4 * 1024 * 1024 });
|
||||
const jsonLine = stdout.trim().split(/\\r?\\n/).reverse().find(l => l.trim().startsWith("{"));
|
||||
if (!jsonLine) throw new RuntimeError("picker_failed", "拾取未返回结果");
|
||||
const parsed = JSON.parse(jsonLine) as Omit<DesktopPickResult, "selector">;
|
||||
if (parsed.cancelled) return { ...parsed, selector: "" };
|
||||
const parts: string[] = [];
|
||||
if (parsed.window) parts.push("window=" + parsed.window);
|
||||
if (parsed.name) parts.push("name=" + parsed.name);
|
||||
if (parsed.controlType) parts.push("type=" + parsed.controlType);
|
||||
if (parsed.automationId) parts.push("id=" + parsed.automationId);
|
||||
return { ...parsed, selector: parts.join(";") };
|
||||
}
|
||||
|
||||
export async function listVisibleWindows(): Promise<Array<{ title: string; process: string }>> {
|
||||
const { stdout } = await run("powershell.exe", ["-NoProfile", "-ExecutionPolicy", "Bypass", "-File", pickerScriptPath(), "-Mode", "windows"], { timeout: 30000, windowsHide: true, maxBuffer: 4 * 1024 * 1024 });
|
||||
const jsonLine = stdout.trim().split(/\\r?\\n/).reverse().find(l => l.trim().startsWith("["));
|
||||
if (!jsonLine) return [];
|
||||
return JSON.parse(jsonLine) as Array<{ title: string; process: string }>;
|
||||
}
|
||||
export async function pickScreenRegion(): Promise<{ x: number; y: number; width: number; height: number; imagePath: string }> {
|
||||
const display = screen.getPrimaryDisplay();
|
||||
const sx = display.bounds.x, sy = display.bounds.y, sw = display.bounds.width, sh = display.bounds.height;
|
||||
const win = new BrowserWindow({
|
||||
x: sx, y: sy, width: sw, height: sh,
|
||||
frame: false, transparent: true, alwaysOnTop: true, skipTaskbar: true, resizable: false, movable: false,
|
||||
webPreferences: { contextIsolation: true, nodeIntegration: false, sandbox: true }
|
||||
});
|
||||
const htmlParts = [
|
||||
"<!DOCTYPE html><html><head><meta charset='utf-8'><style>",
|
||||
"html,body{margin:0;height:100%;background:rgba(0,30,80,0.25);cursor:crosshair;overflow:hidden}",
|
||||
"#sel{position:absolute;border:2px solid #2563eb;background:rgba(37,99,235,0.15);display:none}",
|
||||
"#tip{position:fixed;top:12px;left:50%;transform:translateX(-50%);color:#fff;background:rgba(0,0,0,0.65);padding:8px 18px;border-radius:6px;font-family:Microsoft YaHei;font-size:14px}",
|
||||
"</style></head><body>",
|
||||
"<div id=sel></div><div id=tip>拖拽框选区域，松开鼠标完成；Esc 取消</div>",
|
||||
"<script>",
|
||||
"var sel = document.getElementById('sel'); var start = null;",
|
||||
"document.addEventListener('mousedown', function(e) { start = [e.clientX, e.clientY]; sel.style.display = 'block'; sel.style.left = e.clientX + 'px'; sel.style.top = e.clientY + 'px'; sel.style.width = '0px'; sel.style.height = '0px'; });",
|
||||
"document.addEventListener('mousemove', function(e) { if (!start) return; sel.style.left = Math.min(start[0], e.clientX) + 'px'; sel.style.top = Math.min(start[1], e.clientY) + 'px'; sel.style.width = Math.abs(e.clientX - start[0]) + 'px'; sel.style.height = Math.abs(e.clientY - start[1]) + 'px'; });",
|
||||
"document.addEventListener('mouseup', function(e) { if (!start) return; window.__region = { x: Math.min(start[0], e.clientX), y: Math.min(start[1], e.clientY), width: Math.abs(e.clientX - start[0]), height: Math.abs(e.clientY - start[1]) }; window.__done = true; });",
|
||||
"document.addEventListener('keydown', function(e) { if (e.key === 'Escape') { window.__cancelled = true; window.__done = true; } });",
|
||||
"</script></body></html>"
|
||||
];
|
||||
void win.loadURL("data:text/html;charset=utf-8," + encodeURIComponent(htmlParts.join("")));
|
||||
const deadline = Date.now() + 60000;
|
||||
let region: { x: number; y: number; width: number; height: number } | null = null;
|
||||
while (Date.now() < deadline) {
|
||||
await new Promise(r => setTimeout(r, 250));
|
||||
try {
|
||||
const state = await win.webContents.executeJavaScript("JSON.stringify({ done: window.__done === true, cancelled: window.__cancelled === true, region: window.__region || null })", true);
|
||||
const parsed = JSON.parse(state) as { done: boolean; cancelled: boolean; region: { x: number; y: number; width: number; height: number } | null };
|
||||
if (parsed.cancelled) { win.destroy(); throw new RuntimeError("pick_cancelled", "已取消框选"); }
|
||||
if (parsed.done && parsed.region && parsed.region.width > 2 && parsed.region.height > 2) { region = parsed.region; break; }
|
||||
if (parsed.done) break;
|
||||
} catch (error) {
|
||||
if ((error as Error).message.includes("已取消框选")) throw error;
|
||||
}
|
||||
}
|
||||
win.destroy();
|
||||
if (!region) throw new RuntimeError("pick_cancelled", "未完成框选");
|
||||
await new Promise(r => setTimeout(r, 300));
|
||||
const outDir = "D:/RPA/test-output";
|
||||
fs.mkdirSync(outDir, { recursive: true });
|
||||
const imagePath = outDir + "/pick-region-" + Date.now() + ".png";
|
||||
const psScript = [
|
||||
"Add-Type -AssemblyName System.Drawing | Out-Null;",
|
||||
" = New-Object System.Drawing.Bitmap " + region.width + ", " + region.height + ";",
|
||||
" = [System.Drawing.Graphics]::FromImage();",
|
||||
".CopyFromScreen(" + region.x + ", " + region.y + ", 0, 0, .Size);",
|
||||
".Dispose();",
|
||||
".Save(" + JSON.stringify(imagePath) + ");",
|
||||
".Dispose();",
|
||||
"Write-Output ok"
|
||||
].join("");
|
||||
await run("powershell.exe", ["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-Command", psScript], { timeout: 30000, windowsHide: true });
|
||||
return { ...region, imagePath };
|
||||
}
|
||||
@ -8,6 +8,11 @@ const api = {
|
||||
catalog: {
|
||||
get: (): Promise<Catalog> => ipcRenderer.invoke("catalog:get")
|
||||
},
|
||||
desktop: {
|
||||
pickElement: () => ipcRenderer.invoke('desktop:pick-element'),
|
||||
pickWindowList: () => ipcRenderer.invoke('desktop:pick-window-list'),
|
||||
pickRegion: () => ipcRenderer.invoke('desktop:pick-region')
|
||||
},
|
||||
flows: {
|
||||
list: (): Promise<FlowSummary[]> => ipcRenderer.invoke("flows:list"),
|
||||
create: (name?: string): Promise<FlowDefinition> => ipcRenderer.invoke("flows:create", name),
|
||||
|
||||
@ -468,6 +468,7 @@ function FieldControl({
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
const pickElement = async () => {
|
||||
setPicking(true);
|
||||
try {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user