From 85dcb940561b9a0bbc7c7bbfbe377a9cc592b04c Mon Sep 17 00:00:00 2001 From: yu <1919727607@qq.com> Date: Fri, 30 Oct 2026 18:02:46 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=E6=A1=8C=E9=9D=A2AI?= =?UTF-8?q?=E6=8B=BE=E5=8F=96=E5=8A=9F=E8=83=BD=EF=BC=8C=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E8=A7=86=E8=A7=89=E8=AF=86=E5=88=AB=E6=8E=A7=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- resources/desktop-picker.ps1 | 106 +++- src/main/capabilities/ai-service.ts | 88 ++- src/main/capabilities/desktop-uia.test.ts | 10 +- src/main/capabilities/desktop-uia.ts | 77 ++- src/main/desktop-pick.ts | 630 +++++++++++++++++++++- src/main/index.ts | 2 +- src/main/settings.test.ts | 8 +- src/main/settings.ts | 5 +- src/preload/index.ts | 2 +- src/renderer/src/pages/SettingsPage.tsx | 14 +- src/renderer/src/panels/Properties.tsx | 15 +- src/shared/types.ts | 6 +- 12 files changed, 910 insertions(+), 53 deletions(-) diff --git a/resources/desktop-picker.ps1 b/resources/desktop-picker.ps1 index 41c5b4e..43c5def 100644 --- a/resources/desktop-picker.ps1 +++ b/resources/desktop-picker.ps1 @@ -1,4 +1,12 @@ -param([string]$Mode = "element") +param( + [string]$Mode = "element", + [int]$CaptureX = 0, + [int]$CaptureY = 0, + [int]$CaptureWidth = 1000, + [int]$CaptureHeight = 700, + [string]$CapturePath = "", + [switch]$CaptureExact +) $ErrorActionPreference = "Continue"; [Console]::OutputEncoding = [System.Text.Encoding]::UTF8; $logPath = "D:/RPA/test-output/desktop-picker.log"; @@ -14,6 +22,80 @@ Add-Type -AssemblyName System.Windows.Forms Write-PickLog "assemblies ok" Add-Type -MemberDefinition '[DllImport("user32.dll")] public static extern short GetAsyncKeyState(int key); [DllImport("user32.dll", CharSet = CharSet.Unicode)] public static extern int GetWindowTextW(IntPtr h, System.Text.StringBuilder sb, int max); [DllImport("user32.dll")] public static extern int GetWindowLong(IntPtr hWnd, int nIndex); [DllImport("user32.dll")] public static extern int SetWindowLong(IntPtr hWnd, int nIndex, int dwNewLong);' -Name PickW32 -Namespace RpaPick | Out-Null Write-PickLog "w32 ok" +if ($Mode -eq "capture") { + try { [RpaPick.PickDpi]::SetProcessDpiAwarenessContext([IntPtr](-4)) | Out-Null } catch {} + Add-Type -AssemblyName System.Drawing + Add-Type -MemberDefinition '[DllImport("user32.dll")] public static extern int GetSystemMetrics(int nIndex);' -Name PickMetrics -Namespace RpaPickCapture | Out-Null + $vsX = [RpaPickCapture.PickMetrics]::GetSystemMetrics(76) + $vsY = [RpaPickCapture.PickMetrics]::GetSystemMetrics(77) + $vsW = [RpaPickCapture.PickMetrics]::GetSystemMetrics(78) + $vsH = [RpaPickCapture.PickMetrics]::GetSystemMetrics(79) + $cropW = [Math]::Min($CaptureWidth, $vsW) + $cropH = [Math]::Min($CaptureHeight, $vsH) + if ($CaptureExact) { + $cropX = [Math]::Max($vsX, [Math]::Min($CaptureX, ($vsX + $vsW - $cropW))) + $cropY = [Math]::Max($vsY, [Math]::Min($CaptureY, ($vsY + $vsH - $cropH))) + } else { + $cropX = [Math]::Max($vsX, [Math]::Min($CaptureX - [int]($cropW / 2), ($vsX + $vsW - $cropW))) + $cropY = [Math]::Max($vsY, [Math]::Min($CaptureY - [int]($cropH / 2), ($vsY + $vsH - $cropH))) + } + $markerX = $CaptureX - $cropX + $markerY = $CaptureY - $cropY + if ($CapturePath) { $imagePath = $CapturePath } else { $imagePath = Join-Path $env:TEMP ("ai-pick-" + [Guid]::NewGuid().ToString("N") + ".png") } + $imageDir = Split-Path -Parent $imagePath + if (-not (Test-Path -LiteralPath $imageDir)) { New-Item -ItemType Directory -Path $imageDir -Force | Out-Null } + $bitmap = New-Object System.Drawing.Bitmap $cropW, $cropH + $graphics = [System.Drawing.Graphics]::FromImage($bitmap) + $graphics.CopyFromScreen($cropX, $cropY, 0, 0, $bitmap.Size) + $pen = New-Object System.Drawing.Pen([System.Drawing.Color]::Red, 3) + try { + $graphics.SmoothingMode = [System.Drawing.Drawing2D.SmoothingMode]::AntiAlias + if (-not $CaptureExact) { + $graphics.DrawLine($pen, ($markerX - 20), $markerY, ($markerX + 20), $markerY) + $graphics.DrawLine($pen, $markerX, ($markerY - 20), $markerX, ($markerY + 20)) + $graphics.DrawRectangle($pen, ($markerX - 9), ($markerY - 9), 18, 18) + } + } finally { + $pen.Dispose(); $graphics.Dispose() + } + $bitmap.Save($imagePath, [System.Drawing.Imaging.ImageFormat]::Png) + $bitmap.Dispose() + $windowTitle = "" + $uiaName = "" + $uiaType = "" + try { + $pt = New-Object System.Windows.Point($CaptureX, $CaptureY) + $ae = [System.Windows.Automation.AutomationElement]::FromPoint($pt) + if ($ae) { + $ac = $ae.Current + $uiaName = [string]$ac.Name + $uiaType = $ac.ControlType.ProgrammaticName.Replace("ControlType.", "") + $walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker + $cur = $ae + while ($cur -and -not $cur.Equals([System.Windows.Automation.AutomationElement]::RootElement)) { + $windowTitle = [string]$cur.Current.Name + $cur = $walker.GetParent($cur) + } + } + } catch {} + $result = @{ + aiCapture = $true + cursorX = $CaptureX + cursorY = $CaptureY + cropX = $cropX + cropY = $cropY + cropW = $cropW + cropH = $cropH + markerX = $markerX + markerY = $markerY + window = $windowTitle + uiaName = $uiaName + uiaType = $uiaType + imagePath = $imagePath + } + Write-Output (ConvertTo-Json $result -Compress) + exit 0 +} if ($Mode -eq "windows") { $wins = @() foreach ($p in Get-Process) { if ($p.MainWindowHandle -ne [IntPtr]::Zero -and $p.MainWindowTitle) { $wins += @{ title = $p.MainWindowTitle; process = $p.ProcessName } } } @@ -104,6 +186,7 @@ $selIdx = -1 $prevSpace = $false $prevPgUp = $false $prevPgDn = $false +$prevF2 = $false Write-PickLog "loop start" while ($true) { try { @@ -111,6 +194,7 @@ while ($true) { $esc = (([RpaPick.PickW32]::GetAsyncKeyState(27) -band 0x8001) -ne 0) $pgUp = (([RpaPick.PickW32]::GetAsyncKeyState(33) -band 0x8001) -ne 0) $pgDn = (([RpaPick.PickW32]::GetAsyncKeyState(34) -band 0x8001) -ne 0) + $f2 = (([RpaPick.PickW32]::GetAsyncKeyState(113) -band 0x8001) -ne 0) if ($esc) { Write-PickLog "esc"; break } if ($stage -eq "hover") { $ptX = [System.Windows.Forms.Cursor]::Position.X @@ -119,9 +203,24 @@ while ($true) { if ($el) { $c = $el.Current $ctrlName = $c.ControlType.ProgrammaticName.Replace("ControlType.", "") - $r = $c.BoundingRectangle; Set-HighlightBox ([int]$r.X) ([int]$r.Y) ([int]$r.Width) ([int]$r.Height) - $infoLabel.Text = "名称: " + $c.Name + " 类型: " + $ctrlName + " [空格]锁定 [Esc]取消" + $r = $c.BoundingRectangle + $paneX = [int]$r.X; $paneY = [int]$r.Y; $paneW = [int]$r.Width; $paneH = [int]$r.Height + Set-HighlightBox $paneX $paneY $paneW $paneH + $aiHint = if ($ctrlName -eq "Pane") { " [F2]AI识别" } else { "" } + $infoLabel.Text = "名称: " + $c.Name + " 类型: " + $ctrlName + " [空格]锁定" + $aiHint + " [Esc]取消" } else { Hide-Highlight } + if ($f2 -and -not $prevF2 -and $el -and $ctrlName -eq "Pane") { + Write-PickLog "f2 ai" + $walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker + $cur = $el + $topTitle = "" + while ($cur -and -not $cur.Equals([System.Windows.Automation.AutomationElement]::RootElement)) { + $topTitle = [string]$cur.Current.Name + $cur = $walker.GetParent($cur) + } + $result = @{ aiRequested = $true; cursorX = $ptX; cursorY = $ptY; window = $topTitle; name = [string]$c.Name; controlType = $ctrlName; paneX = $paneX; paneY = $paneY; paneW = $paneW; paneH = $paneH } + break + } if ($space -and -not $prevSpace -and $el) { Write-PickLog "space lock" $stage = "locked" @@ -162,6 +261,7 @@ while ($true) { $prevSpace = $space $prevPgUp = $pgUp $prevPgDn = $pgDn + $prevF2 = $f2 Start-Sleep -Milliseconds 60 } $info.Close() diff --git a/src/main/capabilities/ai-service.ts b/src/main/capabilities/ai-service.ts index c1f0e3d..2dcb5a6 100644 --- a/src/main/capabilities/ai-service.ts +++ b/src/main/capabilities/ai-service.ts @@ -1,33 +1,87 @@ import { loadSettings } from "../settings"; import { RuntimeError } from "../engine/types"; -export interface ChatMessage { role: "system" | "user" | "assistant"; content: string } +export type ChatContentPart = + | { type: "text"; text: string } + | { type: "image_url"; image_url: { url: string } }; -export function getAiConfig(): { baseUrl: string; apiKey: string; model: string } { - return loadSettings().aiConfig; +export interface ChatMessage { + role: "system" | "user" | "assistant"; + content: string | ChatContentPart[]; } -export function requireAiConfig() { +export interface ResolvedAiConfig { + baseUrl: string; + apiKey: string; + model: string; + visionModel: string; +} + +export function getAiConfig(): ResolvedAiConfig { + const config = loadSettings().aiConfig; + return { ...config, visionModel: config.visionModel || config.model }; +} + +export function requireAiConfig(options: { vision?: boolean } = {}): ResolvedAiConfig { const config = getAiConfig(); - if (!config.baseUrl || !config.model) { - throw new RuntimeError("ai_not_configured", "AI 服务未配置", { recommendation: "到 设置 → AI 服务 填写 Base URL 与模型名（兼容 OpenAI 协议）" }); + if (!config.baseUrl || !config.model || (options.vision && !config.visionModel)) { + throw new RuntimeError("ai_not_configured", "AI 服务未配置", { + recommendation: options.vision + ? "到 设置 → AI 服务 填写 Base URL 与多模态视觉模型（兼容 OpenAI 协议）" + : "到 设置 → AI 服务 填写 Base URL 与模型名（兼容 OpenAI 协议）" + }); } return config; } -export async function chatCompletion(messages: ChatMessage[], options: { temperature?: number } = {}): Promise { - const config = requireAiConfig(); +export async function chatCompletion(messages: ChatMessage[], options: { temperature?: number; vision?: boolean } = {}): Promise { + const config = requireAiConfig({ vision: options.vision }); const base = config.baseUrl.replace(/\/+$/, ""); - const response = await fetch(base + "/chat/completions", { - signal: AbortSignal.timeout(170000), - method: "POST", - headers: { "Content-Type": "application/json", Authorization: "Bearer " + config.apiKey }, - body: JSON.stringify({ model: config.model, messages, temperature: options.temperature ?? 0.3 }) - }); - if (!response.ok) { + const maxAttempts = 3; + + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + let response: Response; + try { + response = await fetch(base + "/chat/completions", { + signal: AbortSignal.timeout(170000), + method: "POST", + headers: { "Content-Type": "application/json", Authorization: "Bearer " + config.apiKey }, + body: JSON.stringify({ + model: options.vision ? config.visionModel : config.model, + messages, + temperature: options.temperature ?? 0.3 + }) + }); + } catch (error) { + if (attempt === maxAttempts) throw error; + await new Promise(resolve => setTimeout(resolve, 1200 * attempt)); + continue; + } + + if (response.ok) { + const data = (await response.json()) as { choices?: Array<{ message?: { content?: string } }> }; + return data.choices?.[0]?.message?.content ?? ""; + } + const detail = (await response.text()).slice(0, 300); + const retryable = response.status === 408 || response.status === 429 || response.status >= 500; + if (retryable && attempt < maxAttempts) { + const retryAfter = Number(response.headers.get("retry-after")); + const fallbackDelay = 1400 * attempt; + const delay = Number.isFinite(retryAfter) && retryAfter >= 0 && retryAfter <= 15 + ? retryAfter * 1000 + : fallbackDelay; + await new Promise(resolve => setTimeout(resolve, delay)); + continue; + } + + if (response.status === 429) { + const suffix = detail ? " 详情：" + detail.slice(0, 180) : ""; + throw new RuntimeError("ai_rate_limited", "AI 服务限流或配额不足（HTTP 429），请稍后重试或更换可用视觉模型。" + suffix); + } + throw new RuntimeError("ai_request_failed", "AI 请求失败（HTTP " + response.status + "）：" + detail); } - const data = (await response.json()) as { choices?: Array<{ message?: { content?: string } }> }; - return data.choices?.[0]?.message?.content ?? ""; + + throw new RuntimeError("ai_request_failed", "AI 请求失败：已重试仍未成功"); } diff --git a/src/main/capabilities/desktop-uia.test.ts b/src/main/capabilities/desktop-uia.test.ts index bc0031b..793c632 100644 --- a/src/main/capabilities/desktop-uia.test.ts +++ b/src/main/capabilities/desktop-uia.test.ts @@ -33,8 +33,14 @@ describe("desktop UIA ground capabilities", () => { expect(s.name).toBe("确定"); }); - it("rejects selector without any condition", () => { - expect(() => parseUiaSelector("window=只有窗口")).toThrow(); + it("parses AI screen coordinate selector", () => { + const s = parseUiaSelector("screen=100,200,320,80;window= MyApp;ai=1"); + expect(s.screen).toEqual({ x: 100, y: 200, w: 320, h: 80 }); + expect(s.window).toBe("MyApp"); + }); + + it("rejects invalid selector", () => { expect(() => parseUiaSelector("")).toThrow(); + expect(() => parseUiaSelector("screen=1,2,3")).toThrow(); }); }); diff --git a/src/main/capabilities/desktop-uia.ts b/src/main/capabilities/desktop-uia.ts index 7626b41..d379ef3 100644 --- a/src/main/capabilities/desktop-uia.ts +++ b/src/main/capabilities/desktop-uia.ts @@ -8,6 +8,7 @@ interface UiaSelector { id?: string; type?: string; className?: string; + screen?: { x: number; y: number; w: number; h: number }; index: number; } @@ -41,12 +42,19 @@ export function parseUiaSelector(raw: unknown): UiaSelector { else if (key === "id" || key === "automationid") selector.id = value; else if (key === "type" || key === "controltype") selector.type = value.charAt(0).toUpperCase() + value.slice(1).toLowerCase(); else if (key === "class" || key === "classname") selector.className = value; + else if (key === "screen") { + const nums = value.split(/[,，]/).map(item => Number(item.trim())); + if (nums.length !== 4 || nums.some(n => !Number.isFinite(n))) { + throw new RuntimeError("validation_error", "screen 坐标格式无效：" + value, { recommendation: "使用 screen=x,y,width,height" }); + } + selector.screen = { x: nums[0], y: nums[1], w: nums[2], h: nums[3] }; + } else if (key === "index") { const n = Number(value); if (Number.isFinite(n) && n >= 1) selector.index = Math.floor(n); } } - if (!selector.name && !selector.id && !selector.className && !selector.type) { + if (!selector.name && !selector.id && !selector.className && !selector.type && !selector.screen) { throw new RuntimeError("validation_error", `桌面元素选择器缺少定位条件：${text}`, { recommendation: "至少提供 name= / id= / type= 之一" }); } return selector; @@ -119,6 +127,73 @@ function target(args: Record, node: { id: string; componentId: return { selector: parseUiaSelector(raw), args, node }; } +const SCREEN_USER32 = "Add-Type -MemberDefinition '[DllImport(\"user32.dll\")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value); [DllImport(\"user32.dll\")] public static extern bool SetCursorPos(int X, int Y); [DllImport(\"user32.dll\")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra);' -Name RpaScreenInput -Namespace RpaScreen | Out-Null; try { [RpaScreen.RpaScreenInput]::SetProcessDpiAwarenessContext([IntPtr](-4)) | Out-Null } catch {}"; + +function screenCenter(selector: UiaSelector): { x: number; y: number; w: number; h: number } { + return selector.screen!; +} + +function clickScreenScript(t: UiaTarget): string { + const rect = screenCenter(t.selector); + const x = Math.round(rect.x + rect.w / 2); + const y = Math.round(rect.y + rect.h / 2); + const clickway = String(t.args.clickway ?? "0"); + const doubleClick = clickway === "1" || /double/i.test(clickway); + return `${SCREEN_USER32} +if (-not [RpaScreen.RpaScreenInput]::SetCursorPos(${x}, ${y})) { throw "cursor_move_failed" } +Start-Sleep -Milliseconds 100 +[RpaScreen.RpaScreenInput]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero) +Start-Sleep -Milliseconds 40 +[RpaScreen.RpaScreenInput]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero) +if (${doubleClick}) { + Start-Sleep -Milliseconds 70 + [RpaScreen.RpaScreenInput]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero) + Start-Sleep -Milliseconds 40 + [RpaScreen.RpaScreenInput]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero) +} +Write-Output (@{ clicked = $true; method = 'screen_coordinate'; x = ${x}; y = ${y} } | ConvertTo-Json -Compress)`; +} + +function setValueScreenScript(t: UiaTarget, value: string): string { + const rect = screenCenter(t.selector); + const x = Math.round(rect.x + rect.w / 2); + const y = Math.round(rect.y + rect.h / 2); + return `${SCREEN_USER32} +Add-Type -AssemblyName System.Windows.Forms +[RpaScreen.RpaScreenInput]::SetCursorPos(${x}, ${y}) | Out-Null +Start-Sleep -Milliseconds 120 +[RpaScreen.RpaScreenInput]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero) +Start-Sleep -Milliseconds 40 +[RpaScreen.RpaScreenInput]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero) +Start-Sleep -Milliseconds 220 +$saved = $null +try { $saved = [System.Windows.Forms.Clipboard]::GetText() } catch {} +try { [System.Windows.Forms.Clipboard]::SetText(${psString(value)}) } catch { throw "clipboard_failed" } +Start-Sleep -Milliseconds 80 +[System.Windows.Forms.SendKeys]::SendWait('^a') +Start-Sleep -Milliseconds 60 +[System.Windows.Forms.SendKeys]::SendWait('^v') +Start-Sleep -Milliseconds 280 +try { + if ($saved) { [System.Windows.Forms.Clipboard]::SetText($saved) } else { [System.Windows.Forms.Clipboard]::Clear() } +} catch {} +Write-Output (@{ set = $true; method = 'screen_coordinate_paste'; x = ${x}; y = ${y} } | ConvertTo-Json -Compress)`; +} + +function getValueScreenScript(t: UiaTarget): string { + const rect = screenCenter(t.selector); + const x = Math.round(rect.x + rect.w / 2); + const y = Math.round(rect.y + rect.h / 2); + return `${UIA_USING} +Add-Type -AssemblyName WindowsBase +$pt = New-Object System.Windows.Point(${x}, ${y}) +$el = [System.Windows.Automation.AutomationElement]::FromPoint($pt) +if ($null -eq $el) { Write-Output 'JSON_NOT_FOUND'; exit 0 } +$value = $el.Current.Name +try { $vp = $el.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern); $value = $vp.Value } catch {} +Write-Output (@{ value = $value; name = $el.Current.Name } | ConvertTo-Json -Compress)`; +} + function findScript(t: UiaTarget, timeoutMs: number, gapMs: number): string { const s = t.selector; return `${UIA_USING} diff --git a/src/main/desktop-pick.ts b/src/main/desktop-pick.ts index ebf8efc..a5d9b37 100644 --- a/src/main/desktop-pick.ts +++ b/src/main/desktop-pick.ts @@ -5,6 +5,7 @@ import fs from "node:fs"; import path from "node:path"; import { app, BrowserWindow, screen } from "electron"; import { RuntimeError } from "./engine/types"; +import { chatCompletion, requireAiConfig } from "./capabilities/ai-service"; const run = promisify(execFile); @@ -17,6 +18,7 @@ function pickerScriptPath(): string { if (!found) throw new RuntimeError("picker_unavailable", "桌面拾取脚本缺失", { recommendation: "重新安装应用" }); return found; } + export interface DesktopPickResult { cancelled?: boolean; window?: string; @@ -24,36 +26,641 @@ export interface DesktopPickResult { automationId?: string; controlType?: string; className?: string; + source?: "uia" | "ai"; + x?: number; + y?: number; + w?: number; + h?: number; + confidence?: number; + screenshotPath?: string; + aiRequested?: boolean; selector: string; } -export async function pickDesktopElement(): Promise { +interface UiaPickOutput { + cancelled?: boolean; + aiRequested?: boolean; + cursorX?: number; + cursorY?: number; + paneX?: number; + paneY?: number; + paneW?: number; + paneH?: number; + window?: string; + name?: string; + automationId?: string; + controlType?: string; + className?: string; + x?: number; + y?: number; + w?: number; + h?: number; +} + +interface AiCandidate { + x: number; + y: number; + width: number; + height: number; + name: string; + controlType: string; + confidence: number; +} + +interface AiCaptureOutput { + cursorX: number; + cursorY: number; + cropX: number; + cropY: number; + cropW: number; + cropH: number; + markerX: number; + markerY: number; + window?: string; + uiaName?: string; + uiaType?: string; + imagePath: string; +} + +function jsonLine(stdout: string, startsWith: "{" | "["): T { + const line = stdout.trim().split(/\r?\n/).reverse().find(item => item.trim().startsWith(startsWith)); + if (!line) throw new RuntimeError("picker_failed", "桌面拾取未返回结果"); + return JSON.parse(line) as T; +} + +function safeSelectorText(value: string | undefined): string { + return String(value || "").replace(/[;\r\n]+/g, " ").trim(); +} + +export async function pickDesktopElement(options: { ai?: boolean } = {}): Promise { + if (options.ai) requireAiConfig({ vision: true }); const appWindows = BrowserWindow.getAllWindows().filter(w => !w.isDestroyed()); for (const w of appWindows) w.minimize(); try { - const { stdout } = await run("powershell.exe", ["-NoProfile", "-STA", "-ExecutionPolicy", "Bypass", "-File", pickerScriptPath()], { timeout: 180000, windowsHide: true, maxBuffer: 4 * 1024 * 1024 }); - const jsonLine = stdout.trim().split(/\r?\n/).reverse().find(l => l.trim().startsWith("{")); - if (!jsonLine) throw new RuntimeError("picker_failed", "拾取未返回结果"); - const parsed = JSON.parse(jsonLine) as Omit; + if (options.ai) return await runAiPick(); + const { stdout } = await run( + "powershell.exe", + ["-NoProfile", "-STA", "-ExecutionPolicy", "Bypass", "-File", pickerScriptPath()], + { timeout: 180000, windowsHide: true, maxBuffer: 4 * 1024 * 1024 } + ); + const parsed = jsonLine(stdout, "{"); if (parsed.cancelled) return { ...parsed, selector: "" }; + const paneAiReady = [parsed.paneX, parsed.paneY, parsed.paneW, parsed.paneH] + .map(value => Number(value)) + .every(value => Number.isFinite(value)) && Number(parsed.paneW) > 4 && Number(parsed.paneH) > 4; + if (parsed.aiRequested && paneAiReady) return await runPaneAiPick(parsed); + if (parsed.aiRequested) return await runAiPick(parsed); const parts: string[] = []; if (parsed.window) parts.push("window=" + parsed.window); if (parsed.name) parts.push("name=" + parsed.name); if (parsed.controlType) parts.push("type=" + parsed.controlType); if (parsed.automationId) parts.push("id=" + parsed.automationId); - return { ...parsed, selector: parts.join(";") }; + return { ...parsed, source: "uia", selector: parts.join(";") }; } finally { for (const w of appWindows) { if (!w.isDestroyed()) w.restore(); } } } -export async function listVisibleWindows(): Promise> { - const { stdout } = await run("powershell.exe", ["-NoProfile", "-ExecutionPolicy", "Bypass", "-File", pickerScriptPath(), "-Mode", "windows"], { timeout: 30000, windowsHide: true, maxBuffer: 4 * 1024 * 1024 }); - const jsonLine = stdout.trim().split(/\r?\n/).reverse().find(l => l.trim().startsWith("[")); - if (!jsonLine) return []; - return JSON.parse(jsonLine) as Array<{ title: string; process: string }>; + +function paneAiOverlayHtml(): string { + return ` +
AI 识别中…
+`; +} + +function paneAiRect(context: UiaPickOutput) { + const x = Math.round(Number(context.paneX)); + const y = Math.round(Number(context.paneY)); + const w = Math.round(Number(context.paneW)); + const h = Math.round(Number(context.paneH)); + if (![x, y, w, h].every(Number.isFinite) || w <= 4 || h <= 4) { + throw new RuntimeError("ai_pane_invalid", "Pane 区域无效"); + } + return { x, y, w, h }; +} + +async function createPaneAiOverlay(rect: { x: number; y: number; w: number; h: number }): Promise { + const dipPoint = screen.screenToDipPoint({ x: rect.x, y: rect.y }); + const display = displayForPoint(dipPoint.x, dipPoint.y); + const scale = display.scaleFactor || 1; + const win = new BrowserWindow({ + x: Math.round(dipPoint.x), + y: Math.round(dipPoint.y), + width: Math.max(24, Math.round(rect.w / scale)), + height: Math.max(24, Math.round(rect.h / scale)), + frame: false, + transparent: true, + resizable: false, + movable: false, + minimizable: false, + maximizable: false, + hasShadow: false, + alwaysOnTop: true, + skipTaskbar: true, + show: false, + webPreferences: { contextIsolation: true, nodeIntegration: false, sandbox: true } + }); + win.setAlwaysOnTop(true, "screen-saver"); + win.webContents.on("console-message", (_event, level, message, line, sourceId) => { + if (message.includes("pane-ai")) console.log("[pane-ai]", message, sourceId + ":" + line); + }); + await win.loadURL("data:text/html;charset=utf-8," + encodeURIComponent(paneAiOverlayHtml())); + await win.webContents.executeJavaScript( + "window.__paneW=" + JSON.stringify(rect.w) + ";window.__paneH=" + JSON.stringify(rect.h) + + ";window.__cancelled=false;window.__done=false;window.__selected=null;window.__hover=null;window.__hoverCandidate=null;startScan();" + ); + win.show(); + win.focus(); + return win; +} + +async function capturePaneForAi(rect: { x: number; y: number; w: number; h: number }): Promise { + const outDir = path.join(app.getPath("userData"), "ai-picks"); + fs.mkdirSync(outDir, { recursive: true }); + const imagePath = path.join(outDir, "ai-pane-" + Date.now() + ".png"); + const { stdout } = await run( + "powershell.exe", + [ + "-NoProfile", "-STA", "-ExecutionPolicy", "Bypass", "-File", pickerScriptPath(), + "-Mode", "capture", "-CaptureExact", + "-CaptureX", String(rect.x), "-CaptureY", String(rect.y), + "-CaptureWidth", String(rect.w), "-CaptureHeight", String(rect.h), + "-CapturePath", imagePath + ], + { timeout: 30000, windowsHide: true, maxBuffer: 2 * 1024 * 1024 } + ); + const parsed = jsonLine(stdout, "{"); + if (!parsed.imagePath || !fs.existsSync(parsed.imagePath)) { + throw new RuntimeError("ai_capture_failed", "AI 拾取截图失败"); + } + return parsed; +} + +function parseAiCandidates(text: string, cropW: number, cropH: number): AiCandidate[] { + const fence = String.fromCharCode(96); + const fencePattern = new RegExp(fence + "{3}(?:json)?" + fence + "{3}", "gi"); + const cleaned = text.replace(fencePattern, "").trim(); + const start = cleaned.indexOf("{"); + const arrayStart = cleaned.indexOf("["); + const objectStart = start >= 0 && (start < arrayStart || arrayStart < 0) ? start : arrayStart; + const end = Math.max(cleaned.lastIndexOf("}"), cleaned.lastIndexOf("]")); + if (objectStart < 0 || end <= objectStart) throw new RuntimeError("ai_response_invalid", "AI 视觉模型没有返回 JSON"); + const raw = JSON.parse(cleaned.slice(objectStart, end + 1)) as Record; + const list = Array.isArray(raw) ? raw : Array.isArray(raw.candidates) ? raw.candidates : []; + const result: AiCandidate[] = []; + for (const item of list.slice(0, 40)) { + const record = (item || {}) as Record; + const num = (...keys: string[]): number => { + for (const key of keys) { + const value = Number(record[key]); + if (Number.isFinite(value)) return value; + } + return NaN; + }; + let x = num("x", "left", "x1"); + let y = num("y", "top", "y1"); + let width = num("width", "w"); + let height = num("height", "h"); + const x2 = num("x2", "right"); + const y2 = num("y2", "bottom"); + if (!Number.isFinite(width) && Number.isFinite(x2)) width = x2 - x; + if (!Number.isFinite(height) && Number.isFinite(y2)) height = y2 - y; + if (![x, y, width, height].every(Number.isFinite) || width < 6 || height < 6) continue; + x = Math.max(0, Math.min(cropW - width, x)); + y = Math.max(0, Math.min(cropH - height, y)); + width = Math.min(width, cropW - x); + height = Math.min(height, cropH - y); + if (width * height >= cropW * cropH * 0.97) continue; + const confidence = Number(record.confidence); + result.push({ + x: Math.round(x), y: Math.round(y), width: Math.round(width), height: Math.round(height), + name: String(record.name ?? "AI识别控件").slice(0, 80) || "AI识别控件", + controlType: String(record.controlType ?? "Control").slice(0, 40) || "Control", + confidence: Number.isFinite(confidence) ? Math.max(0, Math.min(1, confidence)) : 0.72 + }); + } + if (!result.length) throw new RuntimeError("ai_response_invalid", "AI 没有识别到可用候选区域"); + return result; +} + +async function runPaneAiPick(context: UiaPickOutput): Promise { + requireAiConfig({ vision: true }); + const pane = paneAiRect(context); + const win = await createPaneAiOverlay(pane); + + try { + // Let the visible scan animation complete over the actual Pane before taking the screenshot. + await new Promise(resolve => setTimeout(resolve, 950)); + win.hide(); + await new Promise(resolve => setTimeout(resolve, 240)); + const capture = await capturePaneForAi(pane); + win.show(); + win.focus(); + await win.webContents.executeJavaScript("startScan('AI 理解中…');"); + + const image = await fs.promises.readFile(capture.imagePath); + let aiSettled = false; + const aiRequest = chatCompletion([ + { + role: "system", + content: "你是桌面 RPA 视觉控件分析器。只输出 JSON，不输出解释或 Markdown。" + }, + { + role: "user", + content: [ + { + type: "text", + text: [ + "这是目标 Pane 区域的截图，尺寸 " + capture.cropW + "x" + capture.cropH + " 像素。", + "请把这个 Pane 里的可用控件/内容块拆分成多个候选矩形，最多返回 30 个，按从上到下、从左到右排序。", + "优先识别按钮、输入框、列表行、卡片、菜单项、链接和明显可操作区域；不要把整个 Pane 当作候选。", + "只返回 JSON：{\"candidates\":[{\"x\":number,\"y\":number,\"width\":number,\"height\":number,\"name\":string,\"controlType\":string,\"confidence\":number}]}", + "坐标全部是相对这张截图的整数像素坐标；x/y 是左上角，width/height 也是像素。" + ].join("\n") + }, + { type: "image_url", image_url: { url: "data:image/png;base64," + image.toString("base64") } } + ] + } + ], { vision: true, temperature: 0 }).finally(() => { aiSettled = true; }); + + const cancelWatcher = (async () => { + while (!aiSettled) { + try { + const state = await win.webContents.executeJavaScript("JSON.stringify({cancelled:window.__cancelled===true})", true); + if (JSON.parse(state).cancelled) throw new RuntimeError("pick_cancelled", "已取消 AI 拾取"); + } catch (error) { + if (error instanceof RuntimeError && error.code === "pick_cancelled") throw error; + } + await new Promise(resolve => setTimeout(resolve, 100)); + } + })(); + + let aiText: string; + try { + aiText = await Promise.race([aiRequest, cancelWatcher]) as string; + } finally { + aiSettled = true; + } + + const candidates = parseAiCandidates(aiText, capture.cropW, capture.cropH); + const payload = JSON.stringify(candidates); + console.log("[pane-ai] candidates", candidates.length, candidates.map(item => ({ name: item.name, x: item.x, y: item.y, width: item.width, height: item.height }))); + await win.webContents.executeJavaScript("window.__done=false;window.__cancelled=false;window.__selected=null;window.__hover=null;window.__hoverCandidate=null;window.__confirmed=null;window.__readyAt=Date.now()+600;showCandidates(" + payload + ");"); + + const deadline = Date.now() + 180000; + let selectedIndex: number | null = null; + while (Date.now() < deadline) { + await new Promise(resolve => setTimeout(resolve, 80)); + const state = await win.webContents.executeJavaScript( + "JSON.stringify({done:window.__done===true,cancelled:window.__cancelled===true,selected:window.__selected})", true + ); + const parsed = JSON.parse(state) as { done: boolean; cancelled: boolean; selected: number | null }; + if (parsed.cancelled) throw new RuntimeError("pick_cancelled", "已取消 AI 拾取"); + if (parsed.done && Number.isInteger(parsed.selected)) { + const confirmedIndex = Number(parsed.selected); + console.log("[pane-ai] confirmed", confirmedIndex, candidates[confirmedIndex]); + selectedIndex = confirmedIndex; + break; + } + } + if (selectedIndex === null || !candidates[selectedIndex]) throw new RuntimeError("pick_cancelled", "AI 拾取超时"); + + const box = candidates[selectedIndex]; + const x = capture.cropX + box.x; + const y = capture.cropY + box.y; + const selectorParts = [ + "screen=" + [x, y, box.width, box.height].map(Math.round).join(","), + context.window ? "window=" + safeSelectorText(context.window) : "", + "name=" + safeSelectorText(box.name), + "type=" + safeSelectorText(box.controlType), + "ai=1" + ].filter(Boolean); + return { + source: "ai", + window: context.window, + name: box.name, + controlType: box.controlType, + x, y, w: box.width, h: box.height, + confidence: box.confidence, + screenshotPath: capture.imagePath, + selector: selectorParts.join(";") + }; + } finally { + if (!win.isDestroyed()) win.destroy(); + } +} + +function aiOverlayHtml(): string { + return `
AI 拾取：单击要识别的控件；Esc 取消
`; +} + +function displayForPoint(x: number, y: number) { + return screen.getAllDisplays().find(display => { + const b = display.bounds; + return x >= b.x && x < b.x + b.width && y >= b.y && y < b.y + b.height; + }) || screen.getPrimaryDisplay(); +} + +async function createAiOverlay(): Promise<{ win: BrowserWindow; display: ReturnType }> { + const cursor = screen.getCursorScreenPoint(); + const display = displayForPoint(cursor.x, cursor.y); + const win = new BrowserWindow({ + x: display.bounds.x, + y: display.bounds.y, + width: display.bounds.width, + height: display.bounds.height, + frame: false, + transparent: true, + resizable: false, + movable: false, + minimizable: false, + maximizable: false, + alwaysOnTop: true, + skipTaskbar: true, + show: false, + webPreferences: { contextIsolation: true, nodeIntegration: false, sandbox: true } + }); + win.setAlwaysOnTop(true, "screen-saver"); + await win.loadURL("data:text/html;charset=utf-8," + encodeURIComponent(aiOverlayHtml())); + win.show(); + win.focus(); + return { win, display }; +} + +async function captureAiAtPoint(physicalX: number, physicalY: number): Promise { + const outDir = path.join(app.getPath("userData"), "ai-picks"); + fs.mkdirSync(outDir, { recursive: true }); + const imagePath = path.join(outDir, "ai-pick-" + Date.now() + ".png"); + const { stdout } = await run( + "powershell.exe", + [ + "-NoProfile", "-STA", "-ExecutionPolicy", "Bypass", "-File", pickerScriptPath(), + "-Mode", "capture", "-CaptureX", String(physicalX), "-CaptureY", String(physicalY), + "-CaptureWidth", "1000", "-CaptureHeight", "700", "-CapturePath", imagePath + ], + { timeout: 30000, windowsHide: true, maxBuffer: 2 * 1024 * 1024 } + ); + const parsed = jsonLine(stdout, "{"); + if (!parsed.imagePath || !fs.existsSync(parsed.imagePath)) { + throw new RuntimeError("ai_capture_failed", "AI 拾取截图失败"); + } + return parsed; +} + +function parseAiBox(text: string, cropW: number, cropH: number) { + const fence = String.fromCharCode(96); + const fencePattern = new RegExp(fence + "{3}(?:json)?" + fence + "{3}", "gi"); + const cleaned = text.replace(fencePattern, "").trim(); + const start = cleaned.indexOf("{"); + const end = cleaned.lastIndexOf("}"); + if (start < 0 || end <= start) throw new RuntimeError("ai_response_invalid", "AI 视觉模型没有返回 JSON"); + const raw = JSON.parse(cleaned.slice(start, end + 1)) as Record; + const num = (...keys: string[]): number => { + for (const key of keys) { + const value = Number(raw[key]); + if (Number.isFinite(value)) return value; + } + return NaN; + }; + const x1 = num("x", "left", "x1"); + const y1 = num("y", "top", "y1"); + let width = num("width", "w"); + let height = num("height", "h"); + const x2 = num("x2", "right"); + const y2 = num("y2", "bottom"); + if (!Number.isFinite(width) && Number.isFinite(x2)) width = x2 - x1; + if (!Number.isFinite(height) && Number.isFinite(y2)) height = y2 - y1; + if (![x1, y1].every(Number.isFinite)) throw new RuntimeError("ai_response_invalid", "AI 视觉模型缺少目标坐标"); + if (!Number.isFinite(width) || width <= 0) width = 24; + if (!Number.isFinite(height) || height <= 0) height = 18; + width = Math.max(4, Math.min(cropW, width)); + height = Math.max(4, Math.min(cropH, height)); + const localX = Math.max(0, Math.min(cropW - width, x1)); + const localY = Math.max(0, Math.min(cropH - height, y1)); + const confidence = Number(raw.confidence); + return { + x: localX, + y: localY, + width, + height, + name: String(raw.name ?? "AI识别控件"), + controlType: String(raw.controlType ?? "Control"), + confidence: Number.isFinite(confidence) ? Math.max(0, Math.min(1, confidence)) : 0.7 + }; +} + +async function runAiPick(context: UiaPickOutput = {}): Promise { + requireAiConfig({ vision: true }); + const { win, display } = await createAiOverlay(); + try { + let point: { x: number; y: number } | null = null; + let physicalX = 0; + let physicalY = 0; + const autoAi = context.aiRequested && Number.isFinite(Number(context.cursorX)) && Number.isFinite(Number(context.cursorY)); + if (autoAi) { + const cursor = screen.getCursorScreenPoint(); + point = { x: cursor.x - display.bounds.x, y: cursor.y - display.bounds.y }; + physicalX = Math.round(Number(context.cursorX)); + physicalY = Math.round(Number(context.cursorY)); + await win.webContents.executeJavaScript( + "var p=document.getElementById('point');p.style.left=(window.__autoX=" + point.x + ")-23+'px';p.style.top=(window.__autoY=" + point.y + ")-23+'px';p.style.display='block';document.getElementById('tip').textContent='AI 视觉识别中…自上而下扫描画面';document.body.classList.add('scanning');window.__done=true;" + ); + await new Promise(resolve => setTimeout(resolve, 120)); + } + const deadline = Date.now() + 120000; + while (!autoAi && Date.now() < deadline) { + await new Promise(resolve => setTimeout(resolve, 80)); + try { + const state = await win.webContents.executeJavaScript( + "JSON.stringify({done:window.__done===true,cancelled:window.__cancelled===true,point:window.__point||null})", true + ); + const parsed = JSON.parse(state) as { done: boolean; cancelled: boolean; point: { x: number; y: number } | null }; + if (parsed.cancelled) throw new RuntimeError("pick_cancelled", "已取消 AI 拾取"); + if (parsed.done && parsed.point) { + point = parsed.point; + break; + } + } catch (error) { + if (error instanceof RuntimeError && error.code === "pick_cancelled") throw error; + } + } + if (!point) throw new RuntimeError("pick_cancelled", "AI 拾取超时"); + + // Hide the effect layer so the screen capture contains the real application only. + win.hide(); + await new Promise(resolve => setTimeout(resolve, 180)); + if (!autoAi) { + const scale = display.scaleFactor || 1; + physicalX = Math.round((display.bounds.x + point.x) * scale); + physicalY = Math.round((display.bounds.y + point.y) * scale); + } + const capture = await captureAiAtPoint(physicalX, physicalY); + win.showInactive(); + await win.webContents.executeJavaScript( + "document.getElementById('tip').textContent='AI 视觉识别中…请稍候';document.body.classList.add('scanning');" + ); + + const image = await fs.promises.readFile(capture.imagePath); + let aiSettled = false; + const aiRequest = chatCompletion([ + { + role: "system", + content: "你是桌面 RPA 视觉定位器。只输出 JSON，不输出解释或 Markdown。" + }, + { + role: "user", + content: [ + { + type: "text", + text: [ + "截图尺寸 " + capture.cropW + "x" + capture.cropH + " 像素。", + "红黑色十字中心位于 (" + capture.markerX + ", " + capture.markerY + ")。", + "请定位十字中心处或半径 35 像素内最小的可交互 UI 控件；不要返回整个窗口或大容器。", + "只返回 JSON：{\"x\":number,\"y\":number,\"width\":number,\"height\":number,\"name\":string,\"controlType\":string,\"confidence\":number}", + "x/y 是控件左上角在截图内的像素坐标，width/height 单位也是像素。" + ].join("\n") + }, + { type: "image_url", image_url: { url: "data:image/png;base64," + image.toString("base64") } } + ] + } + ], { vision: true, temperature: 0 }).finally(() => { aiSettled = true; }); + const cancelWatcher = (async () => { + while (!aiSettled) { + try { + const state = await win.webContents.executeJavaScript("JSON.stringify({cancelled:window.__cancelled===true})", true); + if (JSON.parse(state).cancelled) throw new RuntimeError("pick_cancelled", "已取消 AI 拾取"); + } catch (error) { + if (error instanceof RuntimeError && error.code === "pick_cancelled") throw error; + } + await new Promise(resolve => setTimeout(resolve, 100)); + } + })(); + try { + var result = (await Promise.race([aiRequest, cancelWatcher])) as string; + } finally { + aiSettled = true; + } + + const box = parseAiBox(result, capture.cropW, capture.cropH); + const x = capture.cropX + box.x; + const y = capture.cropY + box.y; + const w = box.width; + const h = box.height; + const parts = [ + "screen=" + [x, y, w, h].map(Math.round).join(","), + context.window || capture.window ? "window=" + safeSelectorText(context.window || capture.window) : "", + "name=" + safeSelectorText(box.name), + "type=" + safeSelectorText(box.controlType || "Control"), + "ai=1" + ].filter(Boolean); + return { + source: "ai", + window: context.window || capture.window, + name: box.name, + controlType: box.controlType, + x, + y, + w, + h, + confidence: box.confidence, + screenshotPath: capture.imagePath, + selector: parts.join(";") + }; + } finally { + if (!win.isDestroyed()) win.destroy(); + } +} + +export async function listVisibleWindows(): Promise> { + const { stdout } = await run( + "powershell.exe", + ["-NoProfile", "-ExecutionPolicy", "Bypass", "-File", pickerScriptPath(), "-Mode", "windows"], + { timeout: 30000, windowsHide: true, maxBuffer: 4 * 1024 * 1024 } + ); + return jsonLine>(stdout, "["); +} + export async function pickScreenRegion(): Promise<{ x: number; y: number; width: number; height: number; imagePath: string }> { const appWindows = BrowserWindow.getAllWindows().filter(w => !w.isDestroyed()); for (const w of appWindows) w.minimize(); @@ -80,6 +687,7 @@ export async function pickScreenRegion(): Promise<{ x: number; y: number; width: "" ]; void win.loadURL("data:text/html;charset=utf-8," + encodeURIComponent(htmlParts.join(""))); + win.show(); const deadline = Date.now() + 60000; let region: { x: number; y: number; width: number; height: number } | null = null; while (Date.now() < deadline) { diff --git a/src/main/index.ts b/src/main/index.ts index f612194..90b9c90 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -113,7 +113,7 @@ function registerIpc(): void { } return session.pickElement(); }); - ipcMain.handle("desktop:pick-element", () => pickDesktopElement()); + ipcMain.handle("desktop:pick-element", (_event, options?: { ai?: boolean }) => pickDesktopElement(options)); ipcMain.handle("desktop:pick-window-list", () => listVisibleWindows()); ipcMain.handle("desktop:pick-region", () => pickScreenRegion()); ipcMain.handle("flows:list", () => listFlows()); diff --git a/src/main/settings.test.ts b/src/main/settings.test.ts index 32c8374..0c20176 100644 --- a/src/main/settings.test.ts +++ b/src/main/settings.test.ts @@ -22,14 +22,14 @@ describe("app settings", () => { }); it("returns the default element wait when no settings file exists", () => { - expect(loadSettings()).toEqual({ browserElementWaitMs: 5000, aiConfig: { baseUrl: "", apiKey: "", model: "" } }); + expect(loadSettings()).toEqual({ browserElementWaitMs: 5000, aiConfig: { baseUrl: "", apiKey: "", model: "", visionModel: "" } }); }); it("persists and clamps a user-adjustable element wait", async () => { const saved = updateSettings({ browserElementWaitMs: 12500 }); - expect(saved).toEqual({ browserElementWaitMs: 12500, aiConfig: { baseUrl: "", apiKey: "", model: "" } }); + expect(saved).toEqual({ browserElementWaitMs: 12500, aiConfig: { baseUrl: "", apiKey: "", model: "", visionModel: "" } }); expect(JSON.parse(await readFile(path.join(electronState.userData, "settings.json"), "utf8"))).toEqual(saved); - expect(updateSettings({ browserElementWaitMs: -5 })).toEqual({ browserElementWaitMs: 0, aiConfig: { baseUrl: "", apiKey: "", model: "" } }); - expect(loadSettings()).toEqual({ browserElementWaitMs: 0, aiConfig: { baseUrl: "", apiKey: "", model: "" } }); + expect(updateSettings({ browserElementWaitMs: -5 })).toEqual({ browserElementWaitMs: 0, aiConfig: { baseUrl: "", apiKey: "", model: "", visionModel: "" } }); + expect(loadSettings()).toEqual({ browserElementWaitMs: 0, aiConfig: { baseUrl: "", apiKey: "", model: "", visionModel: "" } }); }); }); diff --git a/src/main/settings.ts b/src/main/settings.ts index 136b5c6..9ca3042 100644 --- a/src/main/settings.ts +++ b/src/main/settings.ts @@ -5,7 +5,7 @@ import type { AppSettings } from "../shared/types"; export const DEFAULT_APP_SETTINGS: AppSettings = { browserElementWaitMs: 5000, - aiConfig: { baseUrl: "", apiKey: "", model: "" } + aiConfig: { baseUrl: "", apiKey: "", model: "", visionModel: "" } }; let cache: AppSettings | null = null; @@ -25,7 +25,8 @@ function normalizeSettings(value: unknown): AppSettings { aiConfig: { baseUrl: String(ai.baseUrl ?? ""), apiKey: String(ai.apiKey ?? ""), - model: String(ai.model ?? "") + model: String(ai.model ?? ""), + visionModel: String(ai.visionModel ?? "") } }; } diff --git a/src/preload/index.ts b/src/preload/index.ts index 9ee136c..77d0cce 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -9,7 +9,7 @@ const api = { get: (): Promise => ipcRenderer.invoke("catalog:get") }, desktop: { - pickElement: () => ipcRenderer.invoke('desktop:pick-element'), + pickElement: (options?: { ai?: boolean }) => ipcRenderer.invoke("desktop:pick-element", options || {}), pickWindowList: () => ipcRenderer.invoke('desktop:pick-window-list'), pickRegion: () => ipcRenderer.invoke('desktop:pick-region') }, diff --git a/src/renderer/src/pages/SettingsPage.tsx b/src/renderer/src/pages/SettingsPage.tsx index 5df35ba..2536087 100644 --- a/src/renderer/src/pages/SettingsPage.tsx +++ b/src/renderer/src/pages/SettingsPage.tsx @@ -14,7 +14,7 @@ export function SettingsPage({ catalog }: { catalog: Catalog }) { const [waitDraft, setWaitDraft] = useState("5000"); const [waitSaving, setWaitSaving] = useState(false); const [waitMessage, setWaitMessage] = useState(null); - const [aiDraft, setAiDraft] = useState({ baseUrl: "", apiKey: "", model: "" }); + const [aiDraft, setAiDraft] = useState({ baseUrl: "", apiKey: "", model: "", visionModel: "" }); const [aiSaving, setAiSaving] = useState(false); const [aiMessage, setAiMessage] = useState(null); @@ -24,7 +24,12 @@ export function SettingsPage({ catalog }: { catalog: Catalog }) { void window.rpa.settings.get().then((value) => { setSettings(value); setWaitDraft(String(value.browserElementWaitMs)); - setAiDraft(value.aiConfig ?? { baseUrl: "", apiKey: "", model: "" }); + setAiDraft({ + baseUrl: value.aiConfig?.baseUrl ?? "", + apiKey: value.aiConfig?.apiKey ?? "", + model: value.aiConfig?.model ?? "", + visionModel: value.aiConfig?.visionModel ?? "" + }); }).catch(() => setSettings(null)); }, []); @@ -152,6 +157,11 @@ export function SettingsPage({ catalog }: { catalog: Catalog }) { setAiDraft({ ...aiDraft, model: event.target.value })} placeholder="gpt-4o-mini / deepseek-chat / qwen-plus" /> +
+ + setAiDraft({ ...aiDraft, visionModel: event.target.value })} placeholder="gpt-4o-mini / qwen-vl-plus / 留空复用模型名称" /> + 桌面 AI 拾取必须使用多模态视觉模型，并会上传鼠标附近的局部截图。 +
+ <> + + + ) : (