feat: 新增桌面AI拾取功能，支持视觉识别控件
This commit is contained in:
yu
parent
f74d1865da
commit
85dcb94056
@ -1,4 +1,12 @@
|
||||
param([string]$Mode = "element")
|
||||
param(
|
||||
[string]$Mode = "element",
|
||||
[int]$CaptureX = 0,
|
||||
[int]$CaptureY = 0,
|
||||
[int]$CaptureWidth = 1000,
|
||||
[int]$CaptureHeight = 700,
|
||||
[string]$CapturePath = "",
|
||||
[switch]$CaptureExact
|
||||
)
|
||||
$ErrorActionPreference = "Continue";
|
||||
[Console]::OutputEncoding = [System.Text.Encoding]::UTF8;
|
||||
$logPath = "D:/RPA/test-output/desktop-picker.log";
|
||||
@ -14,6 +22,80 @@ Add-Type -AssemblyName System.Windows.Forms
|
||||
Write-PickLog "assemblies ok"
|
||||
Add-Type -MemberDefinition '[DllImport("user32.dll")] public static extern short GetAsyncKeyState(int key); [DllImport("user32.dll", CharSet = CharSet.Unicode)] public static extern int GetWindowTextW(IntPtr h, System.Text.StringBuilder sb, int max); [DllImport("user32.dll")] public static extern int GetWindowLong(IntPtr hWnd, int nIndex); [DllImport("user32.dll")] public static extern int SetWindowLong(IntPtr hWnd, int nIndex, int dwNewLong);' -Name PickW32 -Namespace RpaPick | Out-Null
|
||||
Write-PickLog "w32 ok"
|
||||
if ($Mode -eq "capture") {
|
||||
try { [RpaPick.PickDpi]::SetProcessDpiAwarenessContext([IntPtr](-4)) | Out-Null } catch {}
|
||||
Add-Type -AssemblyName System.Drawing
|
||||
Add-Type -MemberDefinition '[DllImport("user32.dll")] public static extern int GetSystemMetrics(int nIndex);' -Name PickMetrics -Namespace RpaPickCapture | Out-Null
|
||||
$vsX = [RpaPickCapture.PickMetrics]::GetSystemMetrics(76)
|
||||
$vsY = [RpaPickCapture.PickMetrics]::GetSystemMetrics(77)
|
||||
$vsW = [RpaPickCapture.PickMetrics]::GetSystemMetrics(78)
|
||||
$vsH = [RpaPickCapture.PickMetrics]::GetSystemMetrics(79)
|
||||
$cropW = [Math]::Min($CaptureWidth, $vsW)
|
||||
$cropH = [Math]::Min($CaptureHeight, $vsH)
|
||||
if ($CaptureExact) {
|
||||
$cropX = [Math]::Max($vsX, [Math]::Min($CaptureX, ($vsX + $vsW - $cropW)))
|
||||
$cropY = [Math]::Max($vsY, [Math]::Min($CaptureY, ($vsY + $vsH - $cropH)))
|
||||
} else {
|
||||
$cropX = [Math]::Max($vsX, [Math]::Min($CaptureX - [int]($cropW / 2), ($vsX + $vsW - $cropW)))
|
||||
$cropY = [Math]::Max($vsY, [Math]::Min($CaptureY - [int]($cropH / 2), ($vsY + $vsH - $cropH)))
|
||||
}
|
||||
$markerX = $CaptureX - $cropX
|
||||
$markerY = $CaptureY - $cropY
|
||||
if ($CapturePath) { $imagePath = $CapturePath } else { $imagePath = Join-Path $env:TEMP ("ai-pick-" + [Guid]::NewGuid().ToString("N") + ".png") }
|
||||
$imageDir = Split-Path -Parent $imagePath
|
||||
if (-not (Test-Path -LiteralPath $imageDir)) { New-Item -ItemType Directory -Path $imageDir -Force | Out-Null }
|
||||
$bitmap = New-Object System.Drawing.Bitmap $cropW, $cropH
|
||||
$graphics = [System.Drawing.Graphics]::FromImage($bitmap)
|
||||
$graphics.CopyFromScreen($cropX, $cropY, 0, 0, $bitmap.Size)
|
||||
$pen = New-Object System.Drawing.Pen([System.Drawing.Color]::Red, 3)
|
||||
try {
|
||||
$graphics.SmoothingMode = [System.Drawing.Drawing2D.SmoothingMode]::AntiAlias
|
||||
if (-not $CaptureExact) {
|
||||
$graphics.DrawLine($pen, ($markerX - 20), $markerY, ($markerX + 20), $markerY)
|
||||
$graphics.DrawLine($pen, $markerX, ($markerY - 20), $markerX, ($markerY + 20))
|
||||
$graphics.DrawRectangle($pen, ($markerX - 9), ($markerY - 9), 18, 18)
|
||||
}
|
||||
} finally {
|
||||
$pen.Dispose(); $graphics.Dispose()
|
||||
}
|
||||
$bitmap.Save($imagePath, [System.Drawing.Imaging.ImageFormat]::Png)
|
||||
$bitmap.Dispose()
|
||||
$windowTitle = ""
|
||||
$uiaName = ""
|
||||
$uiaType = ""
|
||||
try {
|
||||
$pt = New-Object System.Windows.Point($CaptureX, $CaptureY)
|
||||
$ae = [System.Windows.Automation.AutomationElement]::FromPoint($pt)
|
||||
if ($ae) {
|
||||
$ac = $ae.Current
|
||||
$uiaName = [string]$ac.Name
|
||||
$uiaType = $ac.ControlType.ProgrammaticName.Replace("ControlType.", "")
|
||||
$walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker
|
||||
$cur = $ae
|
||||
while ($cur -and -not $cur.Equals([System.Windows.Automation.AutomationElement]::RootElement)) {
|
||||
$windowTitle = [string]$cur.Current.Name
|
||||
$cur = $walker.GetParent($cur)
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
$result = @{
|
||||
aiCapture = $true
|
||||
cursorX = $CaptureX
|
||||
cursorY = $CaptureY
|
||||
cropX = $cropX
|
||||
cropY = $cropY
|
||||
cropW = $cropW
|
||||
cropH = $cropH
|
||||
markerX = $markerX
|
||||
markerY = $markerY
|
||||
window = $windowTitle
|
||||
uiaName = $uiaName
|
||||
uiaType = $uiaType
|
||||
imagePath = $imagePath
|
||||
}
|
||||
Write-Output (ConvertTo-Json $result -Compress)
|
||||
exit 0
|
||||
}
|
||||
if ($Mode -eq "windows") {
|
||||
$wins = @()
|
||||
foreach ($p in Get-Process) { if ($p.MainWindowHandle -ne [IntPtr]::Zero -and $p.MainWindowTitle) { $wins += @{ title = $p.MainWindowTitle; process = $p.ProcessName } } }
|
||||
@ -104,6 +186,7 @@ $selIdx = -1
|
||||
$prevSpace = $false
|
||||
$prevPgUp = $false
|
||||
$prevPgDn = $false
|
||||
$prevF2 = $false
|
||||
Write-PickLog "loop start"
|
||||
while ($true) {
|
||||
try {
|
||||
@ -111,6 +194,7 @@ while ($true) {
|
||||
$esc = (([RpaPick.PickW32]::GetAsyncKeyState(27) -band 0x8001) -ne 0)
|
||||
$pgUp = (([RpaPick.PickW32]::GetAsyncKeyState(33) -band 0x8001) -ne 0)
|
||||
$pgDn = (([RpaPick.PickW32]::GetAsyncKeyState(34) -band 0x8001) -ne 0)
|
||||
$f2 = (([RpaPick.PickW32]::GetAsyncKeyState(113) -band 0x8001) -ne 0)
|
||||
if ($esc) { Write-PickLog "esc"; break }
|
||||
if ($stage -eq "hover") {
|
||||
$ptX = [System.Windows.Forms.Cursor]::Position.X
|
||||
@ -119,9 +203,24 @@ while ($true) {
|
||||
if ($el) {
|
||||
$c = $el.Current
|
||||
$ctrlName = $c.ControlType.ProgrammaticName.Replace("ControlType.", "")
|
||||
$r = $c.BoundingRectangle; Set-HighlightBox ([int]$r.X) ([int]$r.Y) ([int]$r.Width) ([int]$r.Height)
|
||||
$infoLabel.Text = "名称: " + $c.Name + " 类型: " + $ctrlName + " [空格]锁定 [Esc]取消"
|
||||
$r = $c.BoundingRectangle
|
||||
$paneX = [int]$r.X; $paneY = [int]$r.Y; $paneW = [int]$r.Width; $paneH = [int]$r.Height
|
||||
Set-HighlightBox $paneX $paneY $paneW $paneH
|
||||
$aiHint = if ($ctrlName -eq "Pane") { " [F2]AI识别" } else { "" }
|
||||
$infoLabel.Text = "名称: " + $c.Name + " 类型: " + $ctrlName + " [空格]锁定" + $aiHint + " [Esc]取消"
|
||||
} else { Hide-Highlight }
|
||||
if ($f2 -and -not $prevF2 -and $el -and $ctrlName -eq "Pane") {
|
||||
Write-PickLog "f2 ai"
|
||||
$walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker
|
||||
$cur = $el
|
||||
$topTitle = ""
|
||||
while ($cur -and -not $cur.Equals([System.Windows.Automation.AutomationElement]::RootElement)) {
|
||||
$topTitle = [string]$cur.Current.Name
|
||||
$cur = $walker.GetParent($cur)
|
||||
}
|
||||
$result = @{ aiRequested = $true; cursorX = $ptX; cursorY = $ptY; window = $topTitle; name = [string]$c.Name; controlType = $ctrlName; paneX = $paneX; paneY = $paneY; paneW = $paneW; paneH = $paneH }
|
||||
break
|
||||
}
|
||||
if ($space -and -not $prevSpace -and $el) {
|
||||
Write-PickLog "space lock"
|
||||
$stage = "locked"
|
||||
@ -162,6 +261,7 @@ while ($true) {
|
||||
$prevSpace = $space
|
||||
$prevPgUp = $pgUp
|
||||
$prevPgDn = $pgDn
|
||||
$prevF2 = $f2
|
||||
Start-Sleep -Milliseconds 60
|
||||
}
|
||||
$info.Close()
|
||||
|
||||
@ -1,33 +1,87 @@
|
||||
import { loadSettings } from "../settings";
|
||||
import { RuntimeError } from "../engine/types";
|
||||
|
||||
export interface ChatMessage { role: "system" | "user" | "assistant"; content: string }
|
||||
export type ChatContentPart =
|
||||
| { type: "text"; text: string }
|
||||
| { type: "image_url"; image_url: { url: string } };
|
||||
|
||||
export function getAiConfig(): { baseUrl: string; apiKey: string; model: string } {
|
||||
return loadSettings().aiConfig;
|
||||
export interface ChatMessage {
|
||||
role: "system" | "user" | "assistant";
|
||||
content: string | ChatContentPart[];
|
||||
}
|
||||
|
||||
export function requireAiConfig() {
|
||||
export interface ResolvedAiConfig {
|
||||
baseUrl: string;
|
||||
apiKey: string;
|
||||
model: string;
|
||||
visionModel: string;
|
||||
}
|
||||
|
||||
export function getAiConfig(): ResolvedAiConfig {
|
||||
const config = loadSettings().aiConfig;
|
||||
return { ...config, visionModel: config.visionModel || config.model };
|
||||
}
|
||||
|
||||
export function requireAiConfig(options: { vision?: boolean } = {}): ResolvedAiConfig {
|
||||
const config = getAiConfig();
|
||||
if (!config.baseUrl || !config.model) {
|
||||
throw new RuntimeError("ai_not_configured", "AI 服务未配置", { recommendation: "到 设置 → AI 服务 填写 Base URL 与模型名（兼容 OpenAI 协议）" });
|
||||
if (!config.baseUrl || !config.model || (options.vision && !config.visionModel)) {
|
||||
throw new RuntimeError("ai_not_configured", "AI 服务未配置", {
|
||||
recommendation: options.vision
|
||||
? "到 设置 → AI 服务 填写 Base URL 与多模态视觉模型（兼容 OpenAI 协议）"
|
||||
: "到 设置 → AI 服务 填写 Base URL 与模型名（兼容 OpenAI 协议）"
|
||||
});
|
||||
}
|
||||
return config;
|
||||
}
|
||||
|
||||
export async function chatCompletion(messages: ChatMessage[], options: { temperature?: number } = {}): Promise<string> {
|
||||
const config = requireAiConfig();
|
||||
export async function chatCompletion(messages: ChatMessage[], options: { temperature?: number; vision?: boolean } = {}): Promise<string> {
|
||||
const config = requireAiConfig({ vision: options.vision });
|
||||
const base = config.baseUrl.replace(/\/+$/, "");
|
||||
const response = await fetch(base + "/chat/completions", {
|
||||
signal: AbortSignal.timeout(170000),
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", Authorization: "Bearer " + config.apiKey },
|
||||
body: JSON.stringify({ model: config.model, messages, temperature: options.temperature ?? 0.3 })
|
||||
});
|
||||
if (!response.ok) {
|
||||
const maxAttempts = 3;
|
||||
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch(base + "/chat/completions", {
|
||||
signal: AbortSignal.timeout(170000),
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", Authorization: "Bearer " + config.apiKey },
|
||||
body: JSON.stringify({
|
||||
model: options.vision ? config.visionModel : config.model,
|
||||
messages,
|
||||
temperature: options.temperature ?? 0.3
|
||||
})
|
||||
});
|
||||
} catch (error) {
|
||||
if (attempt === maxAttempts) throw error;
|
||||
await new Promise(resolve => setTimeout(resolve, 1200 * attempt));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (response.ok) {
|
||||
const data = (await response.json()) as { choices?: Array<{ message?: { content?: string } }> };
|
||||
return data.choices?.[0]?.message?.content ?? "";
|
||||
}
|
||||
|
||||
const detail = (await response.text()).slice(0, 300);
|
||||
const retryable = response.status === 408 || response.status === 429 || response.status >= 500;
|
||||
if (retryable && attempt < maxAttempts) {
|
||||
const retryAfter = Number(response.headers.get("retry-after"));
|
||||
const fallbackDelay = 1400 * attempt;
|
||||
const delay = Number.isFinite(retryAfter) && retryAfter >= 0 && retryAfter <= 15
|
||||
? retryAfter * 1000
|
||||
: fallbackDelay;
|
||||
await new Promise(resolve => setTimeout(resolve, delay));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (response.status === 429) {
|
||||
const suffix = detail ? " 详情：" + detail.slice(0, 180) : "";
|
||||
throw new RuntimeError("ai_rate_limited", "AI 服务限流或配额不足（HTTP 429），请稍后重试或更换可用视觉模型。" + suffix);
|
||||
}
|
||||
|
||||
throw new RuntimeError("ai_request_failed", "AI 请求失败（HTTP " + response.status + "）：" + detail);
|
||||
}
|
||||
const data = (await response.json()) as { choices?: Array<{ message?: { content?: string } }> };
|
||||
return data.choices?.[0]?.message?.content ?? "";
|
||||
|
||||
throw new RuntimeError("ai_request_failed", "AI 请求失败：已重试仍未成功");
|
||||
}
|
||||
|
||||
@ -33,8 +33,14 @@ describe("desktop UIA ground capabilities", () => {
|
||||
expect(s.name).toBe("确定");
|
||||
});
|
||||
|
||||
it("rejects selector without any condition", () => {
|
||||
expect(() => parseUiaSelector("window=只有窗口")).toThrow();
|
||||
it("parses AI screen coordinate selector", () => {
|
||||
const s = parseUiaSelector("screen=100,200,320,80;window= MyApp;ai=1");
|
||||
expect(s.screen).toEqual({ x: 100, y: 200, w: 320, h: 80 });
|
||||
expect(s.window).toBe("MyApp");
|
||||
});
|
||||
|
||||
it("rejects invalid selector", () => {
|
||||
expect(() => parseUiaSelector("")).toThrow();
|
||||
expect(() => parseUiaSelector("screen=1,2,3")).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
@ -8,6 +8,7 @@ interface UiaSelector {
|
||||
id?: string;
|
||||
type?: string;
|
||||
className?: string;
|
||||
screen?: { x: number; y: number; w: number; h: number };
|
||||
index: number;
|
||||
}
|
||||
|
||||
@ -41,12 +42,19 @@ export function parseUiaSelector(raw: unknown): UiaSelector {
|
||||
else if (key === "id" || key === "automationid") selector.id = value;
|
||||
else if (key === "type" || key === "controltype") selector.type = value.charAt(0).toUpperCase() + value.slice(1).toLowerCase();
|
||||
else if (key === "class" || key === "classname") selector.className = value;
|
||||
else if (key === "screen") {
|
||||
const nums = value.split(/[,，]/).map(item => Number(item.trim()));
|
||||
if (nums.length !== 4 || nums.some(n => !Number.isFinite(n))) {
|
||||
throw new RuntimeError("validation_error", "screen 坐标格式无效：" + value, { recommendation: "使用 screen=x,y,width,height" });
|
||||
}
|
||||
selector.screen = { x: nums[0], y: nums[1], w: nums[2], h: nums[3] };
|
||||
}
|
||||
else if (key === "index") {
|
||||
const n = Number(value);
|
||||
if (Number.isFinite(n) && n >= 1) selector.index = Math.floor(n);
|
||||
}
|
||||
}
|
||||
if (!selector.name && !selector.id && !selector.className && !selector.type) {
|
||||
if (!selector.name && !selector.id && !selector.className && !selector.type && !selector.screen) {
|
||||
throw new RuntimeError("validation_error", `桌面元素选择器缺少定位条件：${text}`, { recommendation: "至少提供 name= / id= / type= 之一" });
|
||||
}
|
||||
return selector;
|
||||
@ -119,6 +127,73 @@ function target(args: Record<string, unknown>, node: { id: string; componentId:
|
||||
return { selector: parseUiaSelector(raw), args, node };
|
||||
}
|
||||
|
||||
const SCREEN_USER32 = "Add-Type -MemberDefinition '[DllImport(\"user32.dll\")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value); [DllImport(\"user32.dll\")] public static extern bool SetCursorPos(int X, int Y); [DllImport(\"user32.dll\")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra);' -Name RpaScreenInput -Namespace RpaScreen | Out-Null; try { [RpaScreen.RpaScreenInput]::SetProcessDpiAwarenessContext([IntPtr](-4)) | Out-Null } catch {}";
|
||||
|
||||
function screenCenter(selector: UiaSelector): { x: number; y: number; w: number; h: number } {
|
||||
return selector.screen!;
|
||||
}
|
||||
|
||||
function clickScreenScript(t: UiaTarget): string {
|
||||
const rect = screenCenter(t.selector);
|
||||
const x = Math.round(rect.x + rect.w / 2);
|
||||
const y = Math.round(rect.y + rect.h / 2);
|
||||
const clickway = String(t.args.clickway ?? "0");
|
||||
const doubleClick = clickway === "1" || /double/i.test(clickway);
|
||||
return `${SCREEN_USER32}
|
||||
if (-not [RpaScreen.RpaScreenInput]::SetCursorPos(${x}, ${y})) { throw "cursor_move_failed" }
|
||||
Start-Sleep -Milliseconds 100
|
||||
[RpaScreen.RpaScreenInput]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)
|
||||
Start-Sleep -Milliseconds 40
|
||||
[RpaScreen.RpaScreenInput]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)
|
||||
if (${doubleClick}) {
|
||||
Start-Sleep -Milliseconds 70
|
||||
[RpaScreen.RpaScreenInput]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)
|
||||
Start-Sleep -Milliseconds 40
|
||||
[RpaScreen.RpaScreenInput]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)
|
||||
}
|
||||
Write-Output (@{ clicked = $true; method = 'screen_coordinate'; x = ${x}; y = ${y} } | ConvertTo-Json -Compress)`;
|
||||
}
|
||||
|
||||
function setValueScreenScript(t: UiaTarget, value: string): string {
|
||||
const rect = screenCenter(t.selector);
|
||||
const x = Math.round(rect.x + rect.w / 2);
|
||||
const y = Math.round(rect.y + rect.h / 2);
|
||||
return `${SCREEN_USER32}
|
||||
Add-Type -AssemblyName System.Windows.Forms
|
||||
[RpaScreen.RpaScreenInput]::SetCursorPos(${x}, ${y}) | Out-Null
|
||||
Start-Sleep -Milliseconds 120
|
||||
[RpaScreen.RpaScreenInput]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)
|
||||
Start-Sleep -Milliseconds 40
|
||||
[RpaScreen.RpaScreenInput]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)
|
||||
Start-Sleep -Milliseconds 220
|
||||
$saved = $null
|
||||
try { $saved = [System.Windows.Forms.Clipboard]::GetText() } catch {}
|
||||
try { [System.Windows.Forms.Clipboard]::SetText(${psString(value)}) } catch { throw "clipboard_failed" }
|
||||
Start-Sleep -Milliseconds 80
|
||||
[System.Windows.Forms.SendKeys]::SendWait('^a')
|
||||
Start-Sleep -Milliseconds 60
|
||||
[System.Windows.Forms.SendKeys]::SendWait('^v')
|
||||
Start-Sleep -Milliseconds 280
|
||||
try {
|
||||
if ($saved) { [System.Windows.Forms.Clipboard]::SetText($saved) } else { [System.Windows.Forms.Clipboard]::Clear() }
|
||||
} catch {}
|
||||
Write-Output (@{ set = $true; method = 'screen_coordinate_paste'; x = ${x}; y = ${y} } | ConvertTo-Json -Compress)`;
|
||||
}
|
||||
|
||||
function getValueScreenScript(t: UiaTarget): string {
|
||||
const rect = screenCenter(t.selector);
|
||||
const x = Math.round(rect.x + rect.w / 2);
|
||||
const y = Math.round(rect.y + rect.h / 2);
|
||||
return `${UIA_USING}
|
||||
Add-Type -AssemblyName WindowsBase
|
||||
$pt = New-Object System.Windows.Point(${x}, ${y})
|
||||
$el = [System.Windows.Automation.AutomationElement]::FromPoint($pt)
|
||||
if ($null -eq $el) { Write-Output 'JSON_NOT_FOUND'; exit 0 }
|
||||
$value = $el.Current.Name
|
||||
try { $vp = $el.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern); $value = $vp.Value } catch {}
|
||||
Write-Output (@{ value = $value; name = $el.Current.Name } | ConvertTo-Json -Compress)`;
|
||||
}
|
||||
|
||||
function findScript(t: UiaTarget, timeoutMs: number, gapMs: number): string {
|
||||
const s = t.selector;
|
||||
return `${UIA_USING}
|
||||
|
||||
@ -5,6 +5,7 @@ import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { app, BrowserWindow, screen } from "electron";
|
||||
import { RuntimeError } from "./engine/types";
|
||||
import { chatCompletion, requireAiConfig } from "./capabilities/ai-service";
|
||||
|
||||
const run = promisify(execFile);
|
||||
|
||||
@ -17,6 +18,7 @@ function pickerScriptPath(): string {
|
||||
if (!found) throw new RuntimeError("picker_unavailable", "桌面拾取脚本缺失", { recommendation: "重新安装应用" });
|
||||
return found;
|
||||
}
|
||||
|
||||
export interface DesktopPickResult {
|
||||
cancelled?: boolean;
|
||||
window?: string;
|
||||
@ -24,36 +26,641 @@ export interface DesktopPickResult {
|
||||
automationId?: string;
|
||||
controlType?: string;
|
||||
className?: string;
|
||||
source?: "uia" | "ai";
|
||||
x?: number;
|
||||
y?: number;
|
||||
w?: number;
|
||||
h?: number;
|
||||
confidence?: number;
|
||||
screenshotPath?: string;
|
||||
aiRequested?: boolean;
|
||||
selector: string;
|
||||
}
|
||||
|
||||
export async function pickDesktopElement(): Promise<DesktopPickResult> {
|
||||
interface UiaPickOutput {
|
||||
cancelled?: boolean;
|
||||
aiRequested?: boolean;
|
||||
cursorX?: number;
|
||||
cursorY?: number;
|
||||
paneX?: number;
|
||||
paneY?: number;
|
||||
paneW?: number;
|
||||
paneH?: number;
|
||||
window?: string;
|
||||
name?: string;
|
||||
automationId?: string;
|
||||
controlType?: string;
|
||||
className?: string;
|
||||
x?: number;
|
||||
y?: number;
|
||||
w?: number;
|
||||
h?: number;
|
||||
}
|
||||
|
||||
interface AiCandidate {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
name: string;
|
||||
controlType: string;
|
||||
confidence: number;
|
||||
}
|
||||
|
||||
interface AiCaptureOutput {
|
||||
cursorX: number;
|
||||
cursorY: number;
|
||||
cropX: number;
|
||||
cropY: number;
|
||||
cropW: number;
|
||||
cropH: number;
|
||||
markerX: number;
|
||||
markerY: number;
|
||||
window?: string;
|
||||
uiaName?: string;
|
||||
uiaType?: string;
|
||||
imagePath: string;
|
||||
}
|
||||
|
||||
function jsonLine<T>(stdout: string, startsWith: "{" | "["): T {
|
||||
const line = stdout.trim().split(/\r?\n/).reverse().find(item => item.trim().startsWith(startsWith));
|
||||
if (!line) throw new RuntimeError("picker_failed", "桌面拾取未返回结果");
|
||||
return JSON.parse(line) as T;
|
||||
}
|
||||
|
||||
function safeSelectorText(value: string | undefined): string {
|
||||
return String(value || "").replace(/[;\r\n]+/g, " ").trim();
|
||||
}
|
||||
|
||||
export async function pickDesktopElement(options: { ai?: boolean } = {}): Promise<DesktopPickResult> {
|
||||
if (options.ai) requireAiConfig({ vision: true });
|
||||
const appWindows = BrowserWindow.getAllWindows().filter(w => !w.isDestroyed());
|
||||
for (const w of appWindows) w.minimize();
|
||||
try {
|
||||
const { stdout } = await run("powershell.exe", ["-NoProfile", "-STA", "-ExecutionPolicy", "Bypass", "-File", pickerScriptPath()], { timeout: 180000, windowsHide: true, maxBuffer: 4 * 1024 * 1024 });
|
||||
const jsonLine = stdout.trim().split(/\r?\n/).reverse().find(l => l.trim().startsWith("{"));
|
||||
if (!jsonLine) throw new RuntimeError("picker_failed", "拾取未返回结果");
|
||||
const parsed = JSON.parse(jsonLine) as Omit<DesktopPickResult, "selector">;
|
||||
if (options.ai) return await runAiPick();
|
||||
const { stdout } = await run(
|
||||
"powershell.exe",
|
||||
["-NoProfile", "-STA", "-ExecutionPolicy", "Bypass", "-File", pickerScriptPath()],
|
||||
{ timeout: 180000, windowsHide: true, maxBuffer: 4 * 1024 * 1024 }
|
||||
);
|
||||
const parsed = jsonLine<UiaPickOutput>(stdout, "{");
|
||||
if (parsed.cancelled) return { ...parsed, selector: "" };
|
||||
const paneAiReady = [parsed.paneX, parsed.paneY, parsed.paneW, parsed.paneH]
|
||||
.map(value => Number(value))
|
||||
.every(value => Number.isFinite(value)) && Number(parsed.paneW) > 4 && Number(parsed.paneH) > 4;
|
||||
if (parsed.aiRequested && paneAiReady) return await runPaneAiPick(parsed);
|
||||
if (parsed.aiRequested) return await runAiPick(parsed);
|
||||
const parts: string[] = [];
|
||||
if (parsed.window) parts.push("window=" + parsed.window);
|
||||
if (parsed.name) parts.push("name=" + parsed.name);
|
||||
if (parsed.controlType) parts.push("type=" + parsed.controlType);
|
||||
if (parsed.automationId) parts.push("id=" + parsed.automationId);
|
||||
return { ...parsed, selector: parts.join(";") };
|
||||
return { ...parsed, source: "uia", selector: parts.join(";") };
|
||||
} finally {
|
||||
for (const w of appWindows) {
|
||||
if (!w.isDestroyed()) w.restore();
|
||||
}
|
||||
}
|
||||
}
|
||||
export async function listVisibleWindows(): Promise<Array<{ title: string; process: string }>> {
|
||||
const { stdout } = await run("powershell.exe", ["-NoProfile", "-ExecutionPolicy", "Bypass", "-File", pickerScriptPath(), "-Mode", "windows"], { timeout: 30000, windowsHide: true, maxBuffer: 4 * 1024 * 1024 });
|
||||
const jsonLine = stdout.trim().split(/\r?\n/).reverse().find(l => l.trim().startsWith("["));
|
||||
if (!jsonLine) return [];
|
||||
return JSON.parse(jsonLine) as Array<{ title: string; process: string }>;
|
||||
|
||||
function paneAiOverlayHtml(): string {
|
||||
return `<!DOCTYPE html><html><head><meta charset='utf-8'><style>
|
||||
html,body{margin:0;width:100%;height:100%;overflow:hidden;background:transparent;cursor:default;font-family:'Microsoft YaHei',Arial,sans-serif}
|
||||
#frame{position:absolute;inset:0;display:none;border:1px solid rgba(56,189,248,.30);animation:framePulse 1.9s ease-in-out infinite;z-index:4}#frame i{position:absolute;width:30px;height:30px;border:3px solid #38bdf8;filter:drop-shadow(0 0 7px rgba(56,189,248,.85))}#frame i:nth-child(1){left:2px;top:2px;border-right:0;border-bottom:0;border-radius:10px 0 0 0}#frame i:nth-child(2){right:2px;top:2px;border-left:0;border-bottom:0;border-radius:0 10px 0 0}#frame i:nth-child(3){left:2px;bottom:2px;border-right:0;border-top:0;border-radius:0 0 0 10px}#frame i:nth-child(4){right:2px;bottom:2px;border-left:0;border-top:0;border-radius:0 0 10px 0}
|
||||
#scan{position:absolute;left:0;right:0;top:-20%;height:19%;display:none;background:linear-gradient(to bottom,rgba(56,189,248,0),rgba(56,189,248,.06) 48%,rgba(125,211,252,.20) 82%,rgba(224,242,254,.16));z-index:5;animation:scan 1.65s linear infinite;will-change:top}#scan b{position:absolute;left:3%;right:3%;bottom:-2px;height:3px;border-radius:999px;background:linear-gradient(90deg,rgba(56,189,248,0),#7dd3fc 14%,#fff 50%,#7dd3fc 86%,rgba(56,189,248,0));box-shadow:0 0 10px #38bdf8,0 0 28px rgba(56,189,248,.78),0 0 54px rgba(56,189,248,.38)}#scan b:after{content:'';position:absolute;left:0;right:0;top:2px;height:9px;background:linear-gradient(to bottom,rgba(56,189,248,.18),rgba(56,189,248,0))}
|
||||
#tip{position:absolute;left:50%;top:18px;transform:translateX(-50%);display:none;padding:8px 15px;border-radius:999px;color:#f8fafc;background:rgba(2,20,40,.78);border:1px solid rgba(125,211,252,.42);box-shadow:0 8px 28px rgba(0,0,0,.28),inset 0 1px 0 rgba(255,255,255,.09);font-size:13px;line-height:1;white-space:nowrap;z-index:2147483647}#tip:before{content:'';display:inline-block;width:7px;height:7px;margin-right:8px;border-radius:50%;background:#38bdf8;box-shadow:0 0 8px #38bdf8;vertical-align:1px}body.scanning #tip:before{animation:tipPulse 1.1s ease-in-out infinite}
|
||||
.candidate{position:absolute;border:1px solid rgba(255,255,255,.28);background:transparent;cursor:pointer;box-sizing:border-box}
|
||||
.candidate:hover,.candidate.active{border:3px solid #ff2d2d;box-shadow:0 0 0 1px rgba(255,255,255,.45);z-index:2147483646}
|
||||
.badge{position:absolute;display:none;padding:3px 7px;border-radius:5px;color:#fff;background:rgba(190,18,18,.95);border:1px solid rgba(255,220,220,.82);font-size:12px;line-height:16px;white-space:nowrap;pointer-events:none;box-shadow:0 4px 12px rgba(0,0,0,.28);z-index:2147483647}.badge.active{display:block}
|
||||
@keyframes scan{from{top:-20%}to{top:100%}}@keyframes framePulse{0%,100%{border-color:rgba(56,189,248,.24)}50%{border-color:rgba(56,189,248,.58)}}@keyframes tipPulse{0%,100%{opacity:.45}50%{opacity:1}}
|
||||
</style></head><body>
|
||||
<div id=frame><i></i><i></i><i></i><i></i></div><div id=scan><b></b></div><div id=tip>AI 识别中…</div>
|
||||
<script>
|
||||
var paneW=1,paneH=1;
|
||||
function startScan(label){paneW=window.__paneW||document.documentElement.clientWidth;paneH=window.__paneH||document.documentElement.clientHeight;document.getElementById('frame').style.display='block';document.getElementById('scan').style.display='block';document.getElementById('tip').style.display='block';document.getElementById('tip').textContent=label||'AI 识别中…';document.body.classList.add('scanning')}
|
||||
function showCandidates(items){
|
||||
paneW=window.__paneW||document.documentElement.clientWidth;paneH=window.__paneH||document.documentElement.clientHeight;
|
||||
document.getElementById('scan').style.display='none';document.getElementById('frame').style.display='none';document.getElementById('tip').style.display='block';
|
||||
document.getElementById('tip').textContent='AI 识别完成：移动鼠标高亮；单击选中，再次单击或空格回填；Esc 取消';
|
||||
document.body.classList.remove('scanning');document.body.style.cursor='pointer';
|
||||
window.__hover=null;window.__selected=null;window.__confirmed=null;window.__readyAt=Date.now()+600;
|
||||
items.sort(function(a,b){return a.width*a.height-b.width*b.height}).forEach(function(c,index){
|
||||
var box=document.createElement('div');box.className='candidate';box.__index=index;
|
||||
box.style.left=(c.x/paneW*100)+'%';box.style.top=(c.y/paneH*100)+'%';
|
||||
box.style.width=(c.width/paneW*100)+'%';box.style.height=(c.height/paneH*100)+'%';
|
||||
function activate(){
|
||||
Array.prototype.forEach.call(document.querySelectorAll('.candidate,.badge'),function(n){n.classList.remove('active')});
|
||||
box.classList.add('active');var label=document.getElementById('badge-'+index);if(label)label.classList.add('active');
|
||||
window.__hover=index;window.__hoverCandidate=c;
|
||||
}
|
||||
window.confirmCandidate=function(index,c){
|
||||
var tip=document.getElementById('tip');
|
||||
tip.textContent='已选中：'+(c.name||'控件')+'；再次单击或空格回填，Esc 取消';
|
||||
window.__confirmed=index;window.__selected=index;
|
||||
};
|
||||
function deactivate(){
|
||||
box.classList.remove('active');var label=document.getElementById('badge-'+index);if(label)label.classList.remove('active');
|
||||
if(window.__hover===index){window.__hover=null;window.__hoverCandidate=null}
|
||||
}
|
||||
box.addEventListener('mouseenter',activate);box.addEventListener('mousemove',activate);box.addEventListener('mouseleave',deactivate);
|
||||
box.addEventListener('mousedown',function(e){
|
||||
e.preventDefault();e.stopPropagation();activate();
|
||||
if(Date.now()<(window.__readyAt||0)){console.log('pane-ai ignore early click',index);return}
|
||||
if(window.__confirmed!==index){console.log('pane-ai first select',index);window.confirmCandidate(index,c);return}
|
||||
console.log('pane-ai confirm click',index);window.__selected=index;window.__done=true;
|
||||
});
|
||||
document.body.appendChild(box);
|
||||
var yTop=c.y/paneH*100,yBottom=(c.y+c.height)/paneH*100,leftPct=Math.max(0,Math.min(72,c.x/paneW*100));
|
||||
var badge=document.createElement('div');badge.id='badge-'+index;badge.className='badge';
|
||||
badge.textContent='AI识别 · '+(c.name||'控件');
|
||||
badge.style.left=leftPct+'%';
|
||||
badge.style.top=(yTop<9?'calc('+yBottom+'% + 8px)':'calc('+yTop+'% - 32px)');
|
||||
document.body.appendChild(badge);
|
||||
});
|
||||
}
|
||||
document.addEventListener('keydown',function(e){
|
||||
if(e.key==='Escape'){window.__cancelled=true}
|
||||
if(e.key===' '&&window.__hoverCandidate&&!window.__done){
|
||||
e.preventDefault();
|
||||
if(Date.now()<(window.__readyAt||0)){console.log('pane-ai ignore early space');return}
|
||||
if(window.__confirmed!==window.__hover){window.confirmCandidate(window.__hover,window.__hoverCandidate);return}
|
||||
window.__selected=window.__hover;window.__done=true;
|
||||
}
|
||||
});
|
||||
</script></body></html>`;
|
||||
}
|
||||
|
||||
function paneAiRect(context: UiaPickOutput) {
|
||||
const x = Math.round(Number(context.paneX));
|
||||
const y = Math.round(Number(context.paneY));
|
||||
const w = Math.round(Number(context.paneW));
|
||||
const h = Math.round(Number(context.paneH));
|
||||
if (![x, y, w, h].every(Number.isFinite) || w <= 4 || h <= 4) {
|
||||
throw new RuntimeError("ai_pane_invalid", "Pane 区域无效");
|
||||
}
|
||||
return { x, y, w, h };
|
||||
}
|
||||
|
||||
async function createPaneAiOverlay(rect: { x: number; y: number; w: number; h: number }): Promise<BrowserWindow> {
|
||||
const dipPoint = screen.screenToDipPoint({ x: rect.x, y: rect.y });
|
||||
const display = displayForPoint(dipPoint.x, dipPoint.y);
|
||||
const scale = display.scaleFactor || 1;
|
||||
const win = new BrowserWindow({
|
||||
x: Math.round(dipPoint.x),
|
||||
y: Math.round(dipPoint.y),
|
||||
width: Math.max(24, Math.round(rect.w / scale)),
|
||||
height: Math.max(24, Math.round(rect.h / scale)),
|
||||
frame: false,
|
||||
transparent: true,
|
||||
resizable: false,
|
||||
movable: false,
|
||||
minimizable: false,
|
||||
maximizable: false,
|
||||
hasShadow: false,
|
||||
alwaysOnTop: true,
|
||||
skipTaskbar: true,
|
||||
show: false,
|
||||
webPreferences: { contextIsolation: true, nodeIntegration: false, sandbox: true }
|
||||
});
|
||||
win.setAlwaysOnTop(true, "screen-saver");
|
||||
win.webContents.on("console-message", (_event, level, message, line, sourceId) => {
|
||||
if (message.includes("pane-ai")) console.log("[pane-ai]", message, sourceId + ":" + line);
|
||||
});
|
||||
await win.loadURL("data:text/html;charset=utf-8," + encodeURIComponent(paneAiOverlayHtml()));
|
||||
await win.webContents.executeJavaScript(
|
||||
"window.__paneW=" + JSON.stringify(rect.w) + ";window.__paneH=" + JSON.stringify(rect.h) +
|
||||
";window.__cancelled=false;window.__done=false;window.__selected=null;window.__hover=null;window.__hoverCandidate=null;startScan();"
|
||||
);
|
||||
win.show();
|
||||
win.focus();
|
||||
return win;
|
||||
}
|
||||
|
||||
async function capturePaneForAi(rect: { x: number; y: number; w: number; h: number }): Promise<AiCaptureOutput> {
|
||||
const outDir = path.join(app.getPath("userData"), "ai-picks");
|
||||
fs.mkdirSync(outDir, { recursive: true });
|
||||
const imagePath = path.join(outDir, "ai-pane-" + Date.now() + ".png");
|
||||
const { stdout } = await run(
|
||||
"powershell.exe",
|
||||
[
|
||||
"-NoProfile", "-STA", "-ExecutionPolicy", "Bypass", "-File", pickerScriptPath(),
|
||||
"-Mode", "capture", "-CaptureExact",
|
||||
"-CaptureX", String(rect.x), "-CaptureY", String(rect.y),
|
||||
"-CaptureWidth", String(rect.w), "-CaptureHeight", String(rect.h),
|
||||
"-CapturePath", imagePath
|
||||
],
|
||||
{ timeout: 30000, windowsHide: true, maxBuffer: 2 * 1024 * 1024 }
|
||||
);
|
||||
const parsed = jsonLine<AiCaptureOutput>(stdout, "{");
|
||||
if (!parsed.imagePath || !fs.existsSync(parsed.imagePath)) {
|
||||
throw new RuntimeError("ai_capture_failed", "AI 拾取截图失败");
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function parseAiCandidates(text: string, cropW: number, cropH: number): AiCandidate[] {
|
||||
const fence = String.fromCharCode(96);
|
||||
const fencePattern = new RegExp(fence + "{3}(?:json)?" + fence + "{3}", "gi");
|
||||
const cleaned = text.replace(fencePattern, "").trim();
|
||||
const start = cleaned.indexOf("{");
|
||||
const arrayStart = cleaned.indexOf("[");
|
||||
const objectStart = start >= 0 && (start < arrayStart || arrayStart < 0) ? start : arrayStart;
|
||||
const end = Math.max(cleaned.lastIndexOf("}"), cleaned.lastIndexOf("]"));
|
||||
if (objectStart < 0 || end <= objectStart) throw new RuntimeError("ai_response_invalid", "AI 视觉模型没有返回 JSON");
|
||||
const raw = JSON.parse(cleaned.slice(objectStart, end + 1)) as Record<string, unknown>;
|
||||
const list = Array.isArray(raw) ? raw : Array.isArray(raw.candidates) ? raw.candidates : [];
|
||||
const result: AiCandidate[] = [];
|
||||
for (const item of list.slice(0, 40)) {
|
||||
const record = (item || {}) as Record<string, unknown>;
|
||||
const num = (...keys: string[]): number => {
|
||||
for (const key of keys) {
|
||||
const value = Number(record[key]);
|
||||
if (Number.isFinite(value)) return value;
|
||||
}
|
||||
return NaN;
|
||||
};
|
||||
let x = num("x", "left", "x1");
|
||||
let y = num("y", "top", "y1");
|
||||
let width = num("width", "w");
|
||||
let height = num("height", "h");
|
||||
const x2 = num("x2", "right");
|
||||
const y2 = num("y2", "bottom");
|
||||
if (!Number.isFinite(width) && Number.isFinite(x2)) width = x2 - x;
|
||||
if (!Number.isFinite(height) && Number.isFinite(y2)) height = y2 - y;
|
||||
if (![x, y, width, height].every(Number.isFinite) || width < 6 || height < 6) continue;
|
||||
x = Math.max(0, Math.min(cropW - width, x));
|
||||
y = Math.max(0, Math.min(cropH - height, y));
|
||||
width = Math.min(width, cropW - x);
|
||||
height = Math.min(height, cropH - y);
|
||||
if (width * height >= cropW * cropH * 0.97) continue;
|
||||
const confidence = Number(record.confidence);
|
||||
result.push({
|
||||
x: Math.round(x), y: Math.round(y), width: Math.round(width), height: Math.round(height),
|
||||
name: String(record.name ?? "AI识别控件").slice(0, 80) || "AI识别控件",
|
||||
controlType: String(record.controlType ?? "Control").slice(0, 40) || "Control",
|
||||
confidence: Number.isFinite(confidence) ? Math.max(0, Math.min(1, confidence)) : 0.72
|
||||
});
|
||||
}
|
||||
if (!result.length) throw new RuntimeError("ai_response_invalid", "AI 没有识别到可用候选区域");
|
||||
return result;
|
||||
}
|
||||
|
||||
async function runPaneAiPick(context: UiaPickOutput): Promise<DesktopPickResult> {
|
||||
requireAiConfig({ vision: true });
|
||||
const pane = paneAiRect(context);
|
||||
const win = await createPaneAiOverlay(pane);
|
||||
|
||||
try {
|
||||
// Let the visible scan animation complete over the actual Pane before taking the screenshot.
|
||||
await new Promise(resolve => setTimeout(resolve, 950));
|
||||
win.hide();
|
||||
await new Promise(resolve => setTimeout(resolve, 240));
|
||||
const capture = await capturePaneForAi(pane);
|
||||
win.show();
|
||||
win.focus();
|
||||
await win.webContents.executeJavaScript("startScan('AI 理解中…');");
|
||||
|
||||
const image = await fs.promises.readFile(capture.imagePath);
|
||||
let aiSettled = false;
|
||||
const aiRequest = chatCompletion([
|
||||
{
|
||||
role: "system",
|
||||
content: "你是桌面 RPA 视觉控件分析器。只输出 JSON，不输出解释或 Markdown。"
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: [
|
||||
"这是目标 Pane 区域的截图，尺寸 " + capture.cropW + "x" + capture.cropH + " 像素。",
|
||||
"请把这个 Pane 里的可用控件/内容块拆分成多个候选矩形，最多返回 30 个，按从上到下、从左到右排序。",
|
||||
"优先识别按钮、输入框、列表行、卡片、菜单项、链接和明显可操作区域；不要把整个 Pane 当作候选。",
|
||||
"只返回 JSON：{\"candidates\":[{\"x\":number,\"y\":number,\"width\":number,\"height\":number,\"name\":string,\"controlType\":string,\"confidence\":number}]}",
|
||||
"坐标全部是相对这张截图的整数像素坐标；x/y 是左上角，width/height 也是像素。"
|
||||
].join("\n")
|
||||
},
|
||||
{ type: "image_url", image_url: { url: "data:image/png;base64," + image.toString("base64") } }
|
||||
]
|
||||
}
|
||||
], { vision: true, temperature: 0 }).finally(() => { aiSettled = true; });
|
||||
|
||||
const cancelWatcher = (async () => {
|
||||
while (!aiSettled) {
|
||||
try {
|
||||
const state = await win.webContents.executeJavaScript("JSON.stringify({cancelled:window.__cancelled===true})", true);
|
||||
if (JSON.parse(state).cancelled) throw new RuntimeError("pick_cancelled", "已取消 AI 拾取");
|
||||
} catch (error) {
|
||||
if (error instanceof RuntimeError && error.code === "pick_cancelled") throw error;
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
}
|
||||
})();
|
||||
|
||||
let aiText: string;
|
||||
try {
|
||||
aiText = await Promise.race([aiRequest, cancelWatcher]) as string;
|
||||
} finally {
|
||||
aiSettled = true;
|
||||
}
|
||||
|
||||
const candidates = parseAiCandidates(aiText, capture.cropW, capture.cropH);
|
||||
const payload = JSON.stringify(candidates);
|
||||
console.log("[pane-ai] candidates", candidates.length, candidates.map(item => ({ name: item.name, x: item.x, y: item.y, width: item.width, height: item.height })));
|
||||
await win.webContents.executeJavaScript("window.__done=false;window.__cancelled=false;window.__selected=null;window.__hover=null;window.__hoverCandidate=null;window.__confirmed=null;window.__readyAt=Date.now()+600;showCandidates(" + payload + ");");
|
||||
|
||||
const deadline = Date.now() + 180000;
|
||||
let selectedIndex: number | null = null;
|
||||
while (Date.now() < deadline) {
|
||||
await new Promise(resolve => setTimeout(resolve, 80));
|
||||
const state = await win.webContents.executeJavaScript(
|
||||
"JSON.stringify({done:window.__done===true,cancelled:window.__cancelled===true,selected:window.__selected})", true
|
||||
);
|
||||
const parsed = JSON.parse(state) as { done: boolean; cancelled: boolean; selected: number | null };
|
||||
if (parsed.cancelled) throw new RuntimeError("pick_cancelled", "已取消 AI 拾取");
|
||||
if (parsed.done && Number.isInteger(parsed.selected)) {
|
||||
const confirmedIndex = Number(parsed.selected);
|
||||
console.log("[pane-ai] confirmed", confirmedIndex, candidates[confirmedIndex]);
|
||||
selectedIndex = confirmedIndex;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (selectedIndex === null || !candidates[selectedIndex]) throw new RuntimeError("pick_cancelled", "AI 拾取超时");
|
||||
|
||||
const box = candidates[selectedIndex];
|
||||
const x = capture.cropX + box.x;
|
||||
const y = capture.cropY + box.y;
|
||||
const selectorParts = [
|
||||
"screen=" + [x, y, box.width, box.height].map(Math.round).join(","),
|
||||
context.window ? "window=" + safeSelectorText(context.window) : "",
|
||||
"name=" + safeSelectorText(box.name),
|
||||
"type=" + safeSelectorText(box.controlType),
|
||||
"ai=1"
|
||||
].filter(Boolean);
|
||||
return {
|
||||
source: "ai",
|
||||
window: context.window,
|
||||
name: box.name,
|
||||
controlType: box.controlType,
|
||||
x, y, w: box.width, h: box.height,
|
||||
confidence: box.confidence,
|
||||
screenshotPath: capture.imagePath,
|
||||
selector: selectorParts.join(";")
|
||||
};
|
||||
} finally {
|
||||
if (!win.isDestroyed()) win.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
function aiOverlayHtml(): string {
|
||||
return `<!DOCTYPE html><html><head><meta charset='utf-8'><style>
|
||||
html,body{margin:0;width:100%;height:100%;overflow:hidden;background:rgba(3,20,45,.12);cursor:crosshair;font-family:'Microsoft YaHei',Arial,sans-serif}
|
||||
#tip{position:fixed;top:28px;left:50%;transform:translateX(-50%);padding:10px 20px;border-radius:8px;color:#f8fafc;background:rgba(2,14,34,.82);border:1px solid rgba(96,165,250,.55);font-size:15px;box-shadow:0 8px 30px rgba(0,0,0,.25)}
|
||||
#scan{position:absolute;left:0;right:0;top:-240px;height:240px;display:none;background:linear-gradient(to bottom,rgba(59,130,246,0),rgba(59,130,246,.12),rgba(147,197,253,.46),rgba(59,130,246,.10),rgba(59,130,246,0));box-shadow:0 0 38px rgba(96,165,250,.88);animation:scan 1.7s linear infinite}
|
||||
#point{position:absolute;width:46px;height:46px;border:2px solid #38bdf8;border-radius:8px;display:none;box-shadow:0 0 20px rgba(56,189,248,.9)}
|
||||
body.scanning{background:rgba(3,20,45,.03)}
|
||||
body.scanning #scan{display:block}
|
||||
body.scanning #tip{font-size:13px}
|
||||
@keyframes scan{from{top:-240px}to{top:100%}}
|
||||
</style></head><body><div id=scan></div><div id=point></div><div id=tip>AI 拾取：单击要识别的控件；Esc 取消</div><script>
|
||||
document.addEventListener('mousedown', function(e) {
|
||||
if (document.body.classList.contains('scanning')) return;
|
||||
var p=document.getElementById('point');
|
||||
p.style.left=(e.clientX-23)+'px';p.style.top=(e.clientY-23)+'px';p.style.display='block';
|
||||
document.getElementById('tip').textContent='AI 视觉识别中…自上而下扫描画面';
|
||||
document.body.classList.add('scanning');
|
||||
window.__point={x:e.clientX,y:e.clientY};window.__done=true;
|
||||
});
|
||||
document.addEventListener('keydown',function(e){if(e.key==='Escape'){window.__cancelled=true;}});
|
||||
</script></body></html>`;
|
||||
}
|
||||
|
||||
function displayForPoint(x: number, y: number) {
|
||||
return screen.getAllDisplays().find(display => {
|
||||
const b = display.bounds;
|
||||
return x >= b.x && x < b.x + b.width && y >= b.y && y < b.y + b.height;
|
||||
}) || screen.getPrimaryDisplay();
|
||||
}
|
||||
|
||||
async function createAiOverlay(): Promise<{ win: BrowserWindow; display: ReturnType<typeof screen.getPrimaryDisplay> }> {
|
||||
const cursor = screen.getCursorScreenPoint();
|
||||
const display = displayForPoint(cursor.x, cursor.y);
|
||||
const win = new BrowserWindow({
|
||||
x: display.bounds.x,
|
||||
y: display.bounds.y,
|
||||
width: display.bounds.width,
|
||||
height: display.bounds.height,
|
||||
frame: false,
|
||||
transparent: true,
|
||||
resizable: false,
|
||||
movable: false,
|
||||
minimizable: false,
|
||||
maximizable: false,
|
||||
alwaysOnTop: true,
|
||||
skipTaskbar: true,
|
||||
show: false,
|
||||
webPreferences: { contextIsolation: true, nodeIntegration: false, sandbox: true }
|
||||
});
|
||||
win.setAlwaysOnTop(true, "screen-saver");
|
||||
await win.loadURL("data:text/html;charset=utf-8," + encodeURIComponent(aiOverlayHtml()));
|
||||
win.show();
|
||||
win.focus();
|
||||
return { win, display };
|
||||
}
|
||||
|
||||
async function captureAiAtPoint(physicalX: number, physicalY: number): Promise<AiCaptureOutput> {
|
||||
const outDir = path.join(app.getPath("userData"), "ai-picks");
|
||||
fs.mkdirSync(outDir, { recursive: true });
|
||||
const imagePath = path.join(outDir, "ai-pick-" + Date.now() + ".png");
|
||||
const { stdout } = await run(
|
||||
"powershell.exe",
|
||||
[
|
||||
"-NoProfile", "-STA", "-ExecutionPolicy", "Bypass", "-File", pickerScriptPath(),
|
||||
"-Mode", "capture", "-CaptureX", String(physicalX), "-CaptureY", String(physicalY),
|
||||
"-CaptureWidth", "1000", "-CaptureHeight", "700", "-CapturePath", imagePath
|
||||
],
|
||||
{ timeout: 30000, windowsHide: true, maxBuffer: 2 * 1024 * 1024 }
|
||||
);
|
||||
const parsed = jsonLine<AiCaptureOutput>(stdout, "{");
|
||||
if (!parsed.imagePath || !fs.existsSync(parsed.imagePath)) {
|
||||
throw new RuntimeError("ai_capture_failed", "AI 拾取截图失败");
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function parseAiBox(text: string, cropW: number, cropH: number) {
|
||||
const fence = String.fromCharCode(96);
|
||||
const fencePattern = new RegExp(fence + "{3}(?:json)?" + fence + "{3}", "gi");
|
||||
const cleaned = text.replace(fencePattern, "").trim();
|
||||
const start = cleaned.indexOf("{");
|
||||
const end = cleaned.lastIndexOf("}");
|
||||
if (start < 0 || end <= start) throw new RuntimeError("ai_response_invalid", "AI 视觉模型没有返回 JSON");
|
||||
const raw = JSON.parse(cleaned.slice(start, end + 1)) as Record<string, unknown>;
|
||||
const num = (...keys: string[]): number => {
|
||||
for (const key of keys) {
|
||||
const value = Number(raw[key]);
|
||||
if (Number.isFinite(value)) return value;
|
||||
}
|
||||
return NaN;
|
||||
};
|
||||
const x1 = num("x", "left", "x1");
|
||||
const y1 = num("y", "top", "y1");
|
||||
let width = num("width", "w");
|
||||
let height = num("height", "h");
|
||||
const x2 = num("x2", "right");
|
||||
const y2 = num("y2", "bottom");
|
||||
if (!Number.isFinite(width) && Number.isFinite(x2)) width = x2 - x1;
|
||||
if (!Number.isFinite(height) && Number.isFinite(y2)) height = y2 - y1;
|
||||
if (![x1, y1].every(Number.isFinite)) throw new RuntimeError("ai_response_invalid", "AI 视觉模型缺少目标坐标");
|
||||
if (!Number.isFinite(width) || width <= 0) width = 24;
|
||||
if (!Number.isFinite(height) || height <= 0) height = 18;
|
||||
width = Math.max(4, Math.min(cropW, width));
|
||||
height = Math.max(4, Math.min(cropH, height));
|
||||
const localX = Math.max(0, Math.min(cropW - width, x1));
|
||||
const localY = Math.max(0, Math.min(cropH - height, y1));
|
||||
const confidence = Number(raw.confidence);
|
||||
return {
|
||||
x: localX,
|
||||
y: localY,
|
||||
width,
|
||||
height,
|
||||
name: String(raw.name ?? "AI识别控件"),
|
||||
controlType: String(raw.controlType ?? "Control"),
|
||||
confidence: Number.isFinite(confidence) ? Math.max(0, Math.min(1, confidence)) : 0.7
|
||||
};
|
||||
}
|
||||
|
||||
async function runAiPick(context: UiaPickOutput = {}): Promise<DesktopPickResult> {
|
||||
requireAiConfig({ vision: true });
|
||||
const { win, display } = await createAiOverlay();
|
||||
try {
|
||||
let point: { x: number; y: number } | null = null;
|
||||
let physicalX = 0;
|
||||
let physicalY = 0;
|
||||
const autoAi = context.aiRequested && Number.isFinite(Number(context.cursorX)) && Number.isFinite(Number(context.cursorY));
|
||||
if (autoAi) {
|
||||
const cursor = screen.getCursorScreenPoint();
|
||||
point = { x: cursor.x - display.bounds.x, y: cursor.y - display.bounds.y };
|
||||
physicalX = Math.round(Number(context.cursorX));
|
||||
physicalY = Math.round(Number(context.cursorY));
|
||||
await win.webContents.executeJavaScript(
|
||||
"var p=document.getElementById('point');p.style.left=(window.__autoX=" + point.x + ")-23+'px';p.style.top=(window.__autoY=" + point.y + ")-23+'px';p.style.display='block';document.getElementById('tip').textContent='AI 视觉识别中…自上而下扫描画面';document.body.classList.add('scanning');window.__done=true;"
|
||||
);
|
||||
await new Promise(resolve => setTimeout(resolve, 120));
|
||||
}
|
||||
const deadline = Date.now() + 120000;
|
||||
while (!autoAi && Date.now() < deadline) {
|
||||
await new Promise(resolve => setTimeout(resolve, 80));
|
||||
try {
|
||||
const state = await win.webContents.executeJavaScript(
|
||||
"JSON.stringify({done:window.__done===true,cancelled:window.__cancelled===true,point:window.__point||null})", true
|
||||
);
|
||||
const parsed = JSON.parse(state) as { done: boolean; cancelled: boolean; point: { x: number; y: number } | null };
|
||||
if (parsed.cancelled) throw new RuntimeError("pick_cancelled", "已取消 AI 拾取");
|
||||
if (parsed.done && parsed.point) {
|
||||
point = parsed.point;
|
||||
break;
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof RuntimeError && error.code === "pick_cancelled") throw error;
|
||||
}
|
||||
}
|
||||
if (!point) throw new RuntimeError("pick_cancelled", "AI 拾取超时");
|
||||
|
||||
// Hide the effect layer so the screen capture contains the real application only.
|
||||
win.hide();
|
||||
await new Promise(resolve => setTimeout(resolve, 180));
|
||||
if (!autoAi) {
|
||||
const scale = display.scaleFactor || 1;
|
||||
physicalX = Math.round((display.bounds.x + point.x) * scale);
|
||||
physicalY = Math.round((display.bounds.y + point.y) * scale);
|
||||
}
|
||||
const capture = await captureAiAtPoint(physicalX, physicalY);
|
||||
win.showInactive();
|
||||
await win.webContents.executeJavaScript(
|
||||
"document.getElementById('tip').textContent='AI 视觉识别中…请稍候';document.body.classList.add('scanning');"
|
||||
);
|
||||
|
||||
const image = await fs.promises.readFile(capture.imagePath);
|
||||
let aiSettled = false;
|
||||
const aiRequest = chatCompletion([
|
||||
{
|
||||
role: "system",
|
||||
content: "你是桌面 RPA 视觉定位器。只输出 JSON，不输出解释或 Markdown。"
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: [
|
||||
"截图尺寸 " + capture.cropW + "x" + capture.cropH + " 像素。",
|
||||
"红黑色十字中心位于 (" + capture.markerX + ", " + capture.markerY + ")。",
|
||||
"请定位十字中心处或半径 35 像素内最小的可交互 UI 控件；不要返回整个窗口或大容器。",
|
||||
"只返回 JSON：{\"x\":number,\"y\":number,\"width\":number,\"height\":number,\"name\":string,\"controlType\":string,\"confidence\":number}",
|
||||
"x/y 是控件左上角在截图内的像素坐标，width/height 单位也是像素。"
|
||||
].join("\n")
|
||||
},
|
||||
{ type: "image_url", image_url: { url: "data:image/png;base64," + image.toString("base64") } }
|
||||
]
|
||||
}
|
||||
], { vision: true, temperature: 0 }).finally(() => { aiSettled = true; });
|
||||
const cancelWatcher = (async () => {
|
||||
while (!aiSettled) {
|
||||
try {
|
||||
const state = await win.webContents.executeJavaScript("JSON.stringify({cancelled:window.__cancelled===true})", true);
|
||||
if (JSON.parse(state).cancelled) throw new RuntimeError("pick_cancelled", "已取消 AI 拾取");
|
||||
} catch (error) {
|
||||
if (error instanceof RuntimeError && error.code === "pick_cancelled") throw error;
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
}
|
||||
})();
|
||||
try {
|
||||
var result = (await Promise.race([aiRequest, cancelWatcher])) as string;
|
||||
} finally {
|
||||
aiSettled = true;
|
||||
}
|
||||
|
||||
const box = parseAiBox(result, capture.cropW, capture.cropH);
|
||||
const x = capture.cropX + box.x;
|
||||
const y = capture.cropY + box.y;
|
||||
const w = box.width;
|
||||
const h = box.height;
|
||||
const parts = [
|
||||
"screen=" + [x, y, w, h].map(Math.round).join(","),
|
||||
context.window || capture.window ? "window=" + safeSelectorText(context.window || capture.window) : "",
|
||||
"name=" + safeSelectorText(box.name),
|
||||
"type=" + safeSelectorText(box.controlType || "Control"),
|
||||
"ai=1"
|
||||
].filter(Boolean);
|
||||
return {
|
||||
source: "ai",
|
||||
window: context.window || capture.window,
|
||||
name: box.name,
|
||||
controlType: box.controlType,
|
||||
x,
|
||||
y,
|
||||
w,
|
||||
h,
|
||||
confidence: box.confidence,
|
||||
screenshotPath: capture.imagePath,
|
||||
selector: parts.join(";")
|
||||
};
|
||||
} finally {
|
||||
if (!win.isDestroyed()) win.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
export async function listVisibleWindows(): Promise<Array<{ title: string; process: string }>> {
|
||||
const { stdout } = await run(
|
||||
"powershell.exe",
|
||||
["-NoProfile", "-ExecutionPolicy", "Bypass", "-File", pickerScriptPath(), "-Mode", "windows"],
|
||||
{ timeout: 30000, windowsHide: true, maxBuffer: 4 * 1024 * 1024 }
|
||||
);
|
||||
return jsonLine<Array<{ title: string; process: string }>>(stdout, "[");
|
||||
}
|
||||
|
||||
export async function pickScreenRegion(): Promise<{ x: number; y: number; width: number; height: number; imagePath: string }> {
|
||||
const appWindows = BrowserWindow.getAllWindows().filter(w => !w.isDestroyed());
|
||||
for (const w of appWindows) w.minimize();
|
||||
@ -80,6 +687,7 @@ export async function pickScreenRegion(): Promise<{ x: number; y: number; width:
|
||||
"</script></body></html>"
|
||||
];
|
||||
void win.loadURL("data:text/html;charset=utf-8," + encodeURIComponent(htmlParts.join("")));
|
||||
win.show();
|
||||
const deadline = Date.now() + 60000;
|
||||
let region: { x: number; y: number; width: number; height: number } | null = null;
|
||||
while (Date.now() < deadline) {
|
||||
|
||||
@ -113,7 +113,7 @@ function registerIpc(): void {
|
||||
}
|
||||
return session.pickElement();
|
||||
});
|
||||
ipcMain.handle("desktop:pick-element", () => pickDesktopElement());
|
||||
ipcMain.handle("desktop:pick-element", (_event, options?: { ai?: boolean }) => pickDesktopElement(options));
|
||||
ipcMain.handle("desktop:pick-window-list", () => listVisibleWindows());
|
||||
ipcMain.handle("desktop:pick-region", () => pickScreenRegion());
|
||||
ipcMain.handle("flows:list", () => listFlows());
|
||||
|
||||
@ -22,14 +22,14 @@ describe("app settings", () => {
|
||||
});
|
||||
|
||||
it("returns the default element wait when no settings file exists", () => {
|
||||
expect(loadSettings()).toEqual({ browserElementWaitMs: 5000, aiConfig: { baseUrl: "", apiKey: "", model: "" } });
|
||||
expect(loadSettings()).toEqual({ browserElementWaitMs: 5000, aiConfig: { baseUrl: "", apiKey: "", model: "", visionModel: "" } });
|
||||
});
|
||||
|
||||
it("persists and clamps a user-adjustable element wait", async () => {
|
||||
const saved = updateSettings({ browserElementWaitMs: 12500 });
|
||||
expect(saved).toEqual({ browserElementWaitMs: 12500, aiConfig: { baseUrl: "", apiKey: "", model: "" } });
|
||||
expect(saved).toEqual({ browserElementWaitMs: 12500, aiConfig: { baseUrl: "", apiKey: "", model: "", visionModel: "" } });
|
||||
expect(JSON.parse(await readFile(path.join(electronState.userData, "settings.json"), "utf8"))).toEqual(saved);
|
||||
expect(updateSettings({ browserElementWaitMs: -5 })).toEqual({ browserElementWaitMs: 0, aiConfig: { baseUrl: "", apiKey: "", model: "" } });
|
||||
expect(loadSettings()).toEqual({ browserElementWaitMs: 0, aiConfig: { baseUrl: "", apiKey: "", model: "" } });
|
||||
expect(updateSettings({ browserElementWaitMs: -5 })).toEqual({ browserElementWaitMs: 0, aiConfig: { baseUrl: "", apiKey: "", model: "", visionModel: "" } });
|
||||
expect(loadSettings()).toEqual({ browserElementWaitMs: 0, aiConfig: { baseUrl: "", apiKey: "", model: "", visionModel: "" } });
|
||||
});
|
||||
});
|
||||
|
||||
@ -5,7 +5,7 @@ import type { AppSettings } from "../shared/types";
|
||||
|
||||
export const DEFAULT_APP_SETTINGS: AppSettings = {
|
||||
browserElementWaitMs: 5000,
|
||||
aiConfig: { baseUrl: "", apiKey: "", model: "" }
|
||||
aiConfig: { baseUrl: "", apiKey: "", model: "", visionModel: "" }
|
||||
};
|
||||
|
||||
let cache: AppSettings | null = null;
|
||||
@ -25,7 +25,8 @@ function normalizeSettings(value: unknown): AppSettings {
|
||||
aiConfig: {
|
||||
baseUrl: String(ai.baseUrl ?? ""),
|
||||
apiKey: String(ai.apiKey ?? ""),
|
||||
model: String(ai.model ?? "")
|
||||
model: String(ai.model ?? ""),
|
||||
visionModel: String(ai.visionModel ?? "")
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@ -9,7 +9,7 @@ const api = {
|
||||
get: (): Promise<Catalog> => ipcRenderer.invoke("catalog:get")
|
||||
},
|
||||
desktop: {
|
||||
pickElement: () => ipcRenderer.invoke('desktop:pick-element'),
|
||||
pickElement: (options?: { ai?: boolean }) => ipcRenderer.invoke("desktop:pick-element", options || {}),
|
||||
pickWindowList: () => ipcRenderer.invoke('desktop:pick-window-list'),
|
||||
pickRegion: () => ipcRenderer.invoke('desktop:pick-region')
|
||||
},
|
||||
|
||||
@ -14,7 +14,7 @@ export function SettingsPage({ catalog }: { catalog: Catalog }) {
|
||||
const [waitDraft, setWaitDraft] = useState("5000");
|
||||
const [waitSaving, setWaitSaving] = useState(false);
|
||||
const [waitMessage, setWaitMessage] = useState<string | null>(null);
|
||||
const [aiDraft, setAiDraft] = useState({ baseUrl: "", apiKey: "", model: "" });
|
||||
const [aiDraft, setAiDraft] = useState({ baseUrl: "", apiKey: "", model: "", visionModel: "" });
|
||||
const [aiSaving, setAiSaving] = useState(false);
|
||||
const [aiMessage, setAiMessage] = useState<string | null>(null);
|
||||
|
||||
@ -24,7 +24,12 @@ export function SettingsPage({ catalog }: { catalog: Catalog }) {
|
||||
void window.rpa.settings.get().then((value) => {
|
||||
setSettings(value);
|
||||
setWaitDraft(String(value.browserElementWaitMs));
|
||||
setAiDraft(value.aiConfig ?? { baseUrl: "", apiKey: "", model: "" });
|
||||
setAiDraft({
|
||||
baseUrl: value.aiConfig?.baseUrl ?? "",
|
||||
apiKey: value.aiConfig?.apiKey ?? "",
|
||||
model: value.aiConfig?.model ?? "",
|
||||
visionModel: value.aiConfig?.visionModel ?? ""
|
||||
});
|
||||
}).catch(() => setSettings(null));
|
||||
}, []);
|
||||
|
||||
@ -152,6 +157,11 @@ export function SettingsPage({ catalog }: { catalog: Catalog }) {
|
||||
<label className="field-label" htmlFor="ai-model">模型名称</label>
|
||||
<input id="ai-model" className="text-input" value={aiDraft.model} onChange={(event) => setAiDraft({ ...aiDraft, model: event.target.value })} placeholder="gpt-4o-mini / deepseek-chat / qwen-plus" />
|
||||
</div>
|
||||
<div className="field">
|
||||
<label className="field-label" htmlFor="ai-vision-model">视觉模型名称</label>
|
||||
<input id="ai-vision-model" className="text-input" value={aiDraft.visionModel ?? ""} onChange={(event) => setAiDraft({ ...aiDraft, visionModel: event.target.value })} placeholder="gpt-4o-mini / qwen-vl-plus / 留空复用模型名称" />
|
||||
<span className="field-hint">桌面 AI 拾取必须使用多模态视觉模型，并会上传鼠标附近的局部截图。</span>
|
||||
</div>
|
||||
<div className="field-row">
|
||||
<button type="button" className="btn btn-primary" onClick={() => void saveAiConfig()} disabled={aiSaving}>
|
||||
<Save className="icon" />
|
||||
|
||||
@ -471,10 +471,10 @@ function FieldControl({
|
||||
onChange(field.key, current === "" || known ? expression : `${textOf(raw)}${expression}`);
|
||||
};
|
||||
|
||||
const pickDesktop = async () => {
|
||||
const pickDesktop = async (ai = false) => {
|
||||
setPicking(true);
|
||||
try {
|
||||
const picked = await window.rpa.desktop.pickElement();
|
||||
const picked = await window.rpa.desktop.pickElement({ ai });
|
||||
if (picked.cancelled) {
|
||||
onToast("已取消拾取");
|
||||
return;
|
||||
@ -653,9 +653,14 @@ function FieldControl({
|
||||
onChange={(event) => onChange(field.key, event.target.value)}
|
||||
/>
|
||||
{isGroundTarget ? (
|
||||
<button type="button" className="mini-btn" disabled={picking} onClick={() => void pickDesktop()} title="从桌面拾取元素（移到目标按空格）">
|
||||
{picking ? "桌面拾取中" : "桌面拾取"}
|
||||
</button>
|
||||
<>
|
||||
<button type="button" className="mini-btn" disabled={picking} onClick={() => void pickDesktop(false)} title="从桌面拾取元素（移到目标按空格）">
|
||||
{picking ? "桌面拾取中" : "桌面拾取"}
|
||||
</button>
|
||||
<button type="button" className="mini-btn" disabled={picking} onClick={() => void pickDesktop(true)} title="AI 视觉拾取；UIA 类型为 Pane 时也可在桌面拾取中按 F2">
|
||||
{picking ? "AI识别中" : "AI拾取"}
|
||||
</button>
|
||||
</>
|
||||
) : (
|
||||
<button type="button" className="mini-btn" disabled={picking} onClick={() => void pickElement()} title="从运行浏览器拾取元素">
|
||||
{picking ? "拾取中" : "拾取"}
|
||||
|
||||
@ -184,6 +184,8 @@ export interface AiServiceConfig {
|
||||
baseUrl: string;
|
||||
apiKey: string;
|
||||
model: string;
|
||||
/** 桌面 AI 拾取使用的视觉模型；留空时复用 model。 */
|
||||
visionModel?: string;
|
||||
}
|
||||
|
||||
export interface AppSettings {
|
||||
@ -191,10 +193,6 @@ export interface AppSettings {
|
||||
aiConfig: AiServiceConfig;
|
||||
}
|
||||
|
||||
export interface AppSettings {
|
||||
browserElementWaitMs: number;
|
||||
}
|
||||
|
||||
export interface ElementPickResult {
|
||||
selector: string;
|
||||
tag: string;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user