feat: 新增桌面AI拾取功能，支持视觉识别控件

resources/desktop-picker.ps1

@@ -1,4 +1,12 @@
-param([string]$Mode = "element")
+param(
+  [string]$Mode = "element",
+  [int]$CaptureX = 0,
+  [int]$CaptureY = 0,
+  [int]$CaptureWidth = 1000,
+  [int]$CaptureHeight = 700,
+  [string]$CapturePath = "",
+  [switch]$CaptureExact
+)
 $ErrorActionPreference = "Continue";
 [Console]::OutputEncoding = [System.Text.Encoding]::UTF8;
 $logPath = "D:/RPA/test-output/desktop-picker.log";
@@ -14,6 +22,80 @@ Add-Type -AssemblyName System.Windows.Forms
 Write-PickLog "assemblies ok"
 Add-Type -MemberDefinition '[DllImport("user32.dll")] public static extern short GetAsyncKeyState(int key); [DllImport("user32.dll", CharSet = CharSet.Unicode)] public static extern int GetWindowTextW(IntPtr h, System.Text.StringBuilder sb, int max); [DllImport("user32.dll")] public static extern int GetWindowLong(IntPtr hWnd, int nIndex); [DllImport("user32.dll")] public static extern int SetWindowLong(IntPtr hWnd, int nIndex, int dwNewLong);' -Name PickW32 -Namespace RpaPick | Out-Null
 Write-PickLog "w32 ok"
+if ($Mode -eq "capture") {
+  try { [RpaPick.PickDpi]::SetProcessDpiAwarenessContext([IntPtr](-4)) | Out-Null } catch {}
+  Add-Type -AssemblyName System.Drawing
+  Add-Type -MemberDefinition '[DllImport("user32.dll")] public static extern int GetSystemMetrics(int nIndex);' -Name PickMetrics -Namespace RpaPickCapture | Out-Null
+  $vsX = [RpaPickCapture.PickMetrics]::GetSystemMetrics(76)
+  $vsY = [RpaPickCapture.PickMetrics]::GetSystemMetrics(77)
+  $vsW = [RpaPickCapture.PickMetrics]::GetSystemMetrics(78)
+  $vsH = [RpaPickCapture.PickMetrics]::GetSystemMetrics(79)
+  $cropW = [Math]::Min($CaptureWidth, $vsW)
+  $cropH = [Math]::Min($CaptureHeight, $vsH)
+  if ($CaptureExact) {
+    $cropX = [Math]::Max($vsX, [Math]::Min($CaptureX, ($vsX + $vsW - $cropW)))
+    $cropY = [Math]::Max($vsY, [Math]::Min($CaptureY, ($vsY + $vsH - $cropH)))
+  } else {
+    $cropX = [Math]::Max($vsX, [Math]::Min($CaptureX - [int]($cropW / 2), ($vsX + $vsW - $cropW)))
+    $cropY = [Math]::Max($vsY, [Math]::Min($CaptureY - [int]($cropH / 2), ($vsY + $vsH - $cropH)))
+  }
+  $markerX = $CaptureX - $cropX
+  $markerY = $CaptureY - $cropY
+  if ($CapturePath) { $imagePath = $CapturePath } else { $imagePath = Join-Path $env:TEMP ("ai-pick-" + [Guid]::NewGuid().ToString("N") + ".png") }
+  $imageDir = Split-Path -Parent $imagePath
+  if (-not (Test-Path -LiteralPath $imageDir)) { New-Item -ItemType Directory -Path $imageDir -Force | Out-Null }
+  $bitmap = New-Object System.Drawing.Bitmap $cropW, $cropH
+  $graphics = [System.Drawing.Graphics]::FromImage($bitmap)
+  $graphics.CopyFromScreen($cropX, $cropY, 0, 0, $bitmap.Size)
+  $pen = New-Object System.Drawing.Pen([System.Drawing.Color]::Red, 3)
+  try {
+    $graphics.SmoothingMode = [System.Drawing.Drawing2D.SmoothingMode]::AntiAlias
+    if (-not $CaptureExact) {
+      $graphics.DrawLine($pen, ($markerX - 20), $markerY, ($markerX + 20), $markerY)
+      $graphics.DrawLine($pen, $markerX, ($markerY - 20), $markerX, ($markerY + 20))
+      $graphics.DrawRectangle($pen, ($markerX - 9), ($markerY - 9), 18, 18)
+    }
+  } finally {
+    $pen.Dispose(); $graphics.Dispose()
+  }
+  $bitmap.Save($imagePath, [System.Drawing.Imaging.ImageFormat]::Png)
+  $bitmap.Dispose()
+  $windowTitle = ""
+  $uiaName = ""
+  $uiaType = ""
+  try {
+    $pt = New-Object System.Windows.Point($CaptureX, $CaptureY)
+    $ae = [System.Windows.Automation.AutomationElement]::FromPoint($pt)
+    if ($ae) {
+      $ac = $ae.Current
+      $uiaName = [string]$ac.Name
+      $uiaType = $ac.ControlType.ProgrammaticName.Replace("ControlType.", "")
+      $walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker
+      $cur = $ae
+      while ($cur -and -not $cur.Equals([System.Windows.Automation.AutomationElement]::RootElement)) {
+        $windowTitle = [string]$cur.Current.Name
+        $cur = $walker.GetParent($cur)
+      }
+    }
+  } catch {}
+  $result = @{
+    aiCapture = $true
+    cursorX = $CaptureX
+    cursorY = $CaptureY
+    cropX = $cropX
+    cropY = $cropY
+    cropW = $cropW
+    cropH = $cropH
+    markerX = $markerX
+    markerY = $markerY
+    window = $windowTitle
+    uiaName = $uiaName
+    uiaType = $uiaType
+    imagePath = $imagePath
+  }
+  Write-Output (ConvertTo-Json $result -Compress)
+  exit 0
+}
 if ($Mode -eq "windows") {
   $wins = @()
   foreach ($p in Get-Process) { if ($p.MainWindowHandle -ne [IntPtr]::Zero -and $p.MainWindowTitle) { $wins += @{ title = $p.MainWindowTitle; process = $p.ProcessName } } }
@@ -104,6 +186,7 @@ $selIdx = -1
 $prevSpace = $false
 $prevPgUp = $false
 $prevPgDn = $false
+$prevF2 = $false
 Write-PickLog "loop start"
 while ($true) {
   try {
@@ -111,6 +194,7 @@ while ($true) {
     $esc = (([RpaPick.PickW32]::GetAsyncKeyState(27) -band 0x8001) -ne 0)
     $pgUp = (([RpaPick.PickW32]::GetAsyncKeyState(33) -band 0x8001) -ne 0)
     $pgDn = (([RpaPick.PickW32]::GetAsyncKeyState(34) -band 0x8001) -ne 0)
+    $f2 = (([RpaPick.PickW32]::GetAsyncKeyState(113) -band 0x8001) -ne 0)
     if ($esc) { Write-PickLog "esc"; break }
     if ($stage -eq "hover") {
       $ptX = [System.Windows.Forms.Cursor]::Position.X
@@ -119,9 +203,24 @@ while ($true) {
       if ($el) {
         $c = $el.Current
         $ctrlName = $c.ControlType.ProgrammaticName.Replace("ControlType.", "")
-        $r = $c.BoundingRectangle; Set-HighlightBox ([int]$r.X) ([int]$r.Y) ([int]$r.Width) ([int]$r.Height)
+        $r = $c.BoundingRectangle
-        $infoLabel.Text = "名称: " + $c.Name + "   类型: " + $ctrlName + "   [空格]锁定   [Esc]取消"
+        $paneX = [int]$r.X; $paneY = [int]$r.Y; $paneW = [int]$r.Width; $paneH = [int]$r.Height
+        Set-HighlightBox $paneX $paneY $paneW $paneH
+        $aiHint = if ($ctrlName -eq "Pane") { "   [F2]AI识别" } else { "" }
+        $infoLabel.Text = "名称: " + $c.Name + "   类型: " + $ctrlName + "   [空格]锁定" + $aiHint + "   [Esc]取消"
       } else { Hide-Highlight }
+      if ($f2 -and -not $prevF2 -and $el -and $ctrlName -eq "Pane") {
+        Write-PickLog "f2 ai"
+        $walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker
+        $cur = $el
+        $topTitle = ""
+        while ($cur -and -not $cur.Equals([System.Windows.Automation.AutomationElement]::RootElement)) {
+          $topTitle = [string]$cur.Current.Name
+          $cur = $walker.GetParent($cur)
+        }
+        $result = @{ aiRequested = $true; cursorX = $ptX; cursorY = $ptY; window = $topTitle; name = [string]$c.Name; controlType = $ctrlName; paneX = $paneX; paneY = $paneY; paneW = $paneW; paneH = $paneH }
+        break
+      }
       if ($space -and -not $prevSpace -and $el) {
         Write-PickLog "space lock"
         $stage = "locked"
@@ -162,6 +261,7 @@ while ($true) {
   $prevSpace = $space
   $prevPgUp = $pgUp
   $prevPgDn = $pgDn
+  $prevF2 = $f2
   Start-Sleep -Milliseconds 60
 }
 $info.Close()

src/main/capabilities/ai-service.ts

@@ -1,33 +1,87 @@
 import { loadSettings } from "../settings";
 import { RuntimeError } from "../engine/types";
 
-export interface ChatMessage { role: "system" | "user" | "assistant"; content: string }
+export type ChatContentPart =
+  | { type: "text"; text: string }
+  | { type: "image_url"; image_url: { url: string } };
 
-export function getAiConfig(): { baseUrl: string; apiKey: string; model: string } {
+export interface ChatMessage {
-  return loadSettings().aiConfig;
+  role: "system" | "user" | "assistant";
+  content: string | ChatContentPart[];
 }
 
-export function requireAiConfig() {
+export interface ResolvedAiConfig {
+  baseUrl: string;
+  apiKey: string;
+  model: string;
+  visionModel: string;
+}
+
+export function getAiConfig(): ResolvedAiConfig {
+  const config = loadSettings().aiConfig;
+  return { ...config, visionModel: config.visionModel || config.model };
+}
+
+export function requireAiConfig(options: { vision?: boolean } = {}): ResolvedAiConfig {
   const config = getAiConfig();
-  if (!config.baseUrl || !config.model) {
+  if (!config.baseUrl || !config.model || (options.vision && !config.visionModel)) {
-    throw new RuntimeError("ai_not_configured", "AI 服务未配置", { recommendation: "到 设置 → AI 服务 填写 Base URL 与模型名（兼容 OpenAI 协议）" });
+    throw new RuntimeError("ai_not_configured", "AI 服务未配置", {
+      recommendation: options.vision
+        ? "到 设置 → AI 服务 填写 Base URL 与多模态视觉模型（兼容 OpenAI 协议）"
+        : "到 设置 → AI 服务 填写 Base URL 与模型名（兼容 OpenAI 协议）"
+    });
   }
   return config;
 }
 
-export async function chatCompletion(messages: ChatMessage[], options: { temperature?: number } = {}): Promise<string> {
+export async function chatCompletion(messages: ChatMessage[], options: { temperature?: number; vision?: boolean } = {}): Promise<string> {
-  const config = requireAiConfig();
+  const config = requireAiConfig({ vision: options.vision });
   const base = config.baseUrl.replace(/\/+$/, "");
-  const response = await fetch(base + "/chat/completions", {
+  const maxAttempts = 3;
-    signal: AbortSignal.timeout(170000),
+
-    method: "POST",
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
-    headers: { "Content-Type": "application/json", Authorization: "Bearer " + config.apiKey },
+    let response: Response;
-    body: JSON.stringify({ model: config.model, messages, temperature: options.temperature ?? 0.3 })
+    try {
-  });
+      response = await fetch(base + "/chat/completions", {
-  if (!response.ok) {
+        signal: AbortSignal.timeout(170000),
+        method: "POST",
+        headers: { "Content-Type": "application/json", Authorization: "Bearer " + config.apiKey },
+        body: JSON.stringify({
+          model: options.vision ? config.visionModel : config.model,
+          messages,
+          temperature: options.temperature ?? 0.3
+        })
+      });
+    } catch (error) {
+      if (attempt === maxAttempts) throw error;
+      await new Promise(resolve => setTimeout(resolve, 1200 * attempt));
+      continue;
+    }
+
+    if (response.ok) {
+      const data = (await response.json()) as { choices?: Array<{ message?: { content?: string } }> };
+      return data.choices?.[0]?.message?.content ?? "";
+    }
+
     const detail = (await response.text()).slice(0, 300);
+    const retryable = response.status === 408 || response.status === 429 || response.status >= 500;
+    if (retryable && attempt < maxAttempts) {
+      const retryAfter = Number(response.headers.get("retry-after"));
+      const fallbackDelay = 1400 * attempt;
+      const delay = Number.isFinite(retryAfter) && retryAfter >= 0 && retryAfter <= 15
+        ? retryAfter * 1000
+        : fallbackDelay;
+      await new Promise(resolve => setTimeout(resolve, delay));
+      continue;
+    }
+
+    if (response.status === 429) {
+      const suffix = detail ? " 详情：" + detail.slice(0, 180) : "";
+      throw new RuntimeError("ai_rate_limited", "AI 服务限流或配额不足（HTTP 429），请稍后重试或更换可用视觉模型。" + suffix);
+    }
+
     throw new RuntimeError("ai_request_failed", "AI 请求失败（HTTP " + response.status + "）：" + detail);
   }
-  const data = (await response.json()) as { choices?: Array<{ message?: { content?: string } }> };
+
-  return data.choices?.[0]?.message?.content ?? "";
+  throw new RuntimeError("ai_request_failed", "AI 请求失败：已重试仍未成功");
 }

src/main/capabilities/desktop-uia.test.ts

@@ -33,8 +33,14 @@ describe("desktop UIA ground capabilities", () => {
     expect(s.name).toBe("确定");
   });
 
-  it("rejects selector without any condition", () => {
+  it("parses AI screen coordinate selector", () => {
-    expect(() => parseUiaSelector("window=只有窗口")).toThrow();
+    const s = parseUiaSelector("screen=100,200,320,80;window= MyApp;ai=1");
+    expect(s.screen).toEqual({ x: 100, y: 200, w: 320, h: 80 });
+    expect(s.window).toBe("MyApp");
+  });
+
+  it("rejects invalid selector", () => {
     expect(() => parseUiaSelector("")).toThrow();
+    expect(() => parseUiaSelector("screen=1,2,3")).toThrow();
   });
 });

src/main/capabilities/desktop-uia.ts

@@ -8,6 +8,7 @@ interface UiaSelector {
   id?: string;
   type?: string;
   className?: string;
+  screen?: { x: number; y: number; w: number; h: number };
   index: number;
 }
 
@@ -41,12 +42,19 @@ export function parseUiaSelector(raw: unknown): UiaSelector {
     else if (key === "id" || key === "automationid") selector.id = value;
     else if (key === "type" || key === "controltype") selector.type = value.charAt(0).toUpperCase() + value.slice(1).toLowerCase();
     else if (key === "class" || key === "classname") selector.className = value;
+    else if (key === "screen") {
+      const nums = value.split(/[,，]/).map(item => Number(item.trim()));
+      if (nums.length !== 4 || nums.some(n => !Number.isFinite(n))) {
+        throw new RuntimeError("validation_error", "screen 坐标格式无效：" + value, { recommendation: "使用 screen=x,y,width,height" });
+      }
+      selector.screen = { x: nums[0], y: nums[1], w: nums[2], h: nums[3] };
+    }
     else if (key === "index") {
       const n = Number(value);
       if (Number.isFinite(n) && n >= 1) selector.index = Math.floor(n);
     }
   }
-  if (!selector.name && !selector.id && !selector.className && !selector.type) {
+  if (!selector.name && !selector.id && !selector.className && !selector.type && !selector.screen) {
     throw new RuntimeError("validation_error", `桌面元素选择器缺少定位条件：${text}`, { recommendation: "至少提供 name= / id= / type= 之一" });
   }
   return selector;
@@ -119,6 +127,73 @@ function target(args: Record<string, unknown>, node: { id: string; componentId:
   return { selector: parseUiaSelector(raw), args, node };
 }
 
+const SCREEN_USER32 = "Add-Type -MemberDefinition '[DllImport(\"user32.dll\")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value); [DllImport(\"user32.dll\")] public static extern bool SetCursorPos(int X, int Y); [DllImport(\"user32.dll\")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra);' -Name RpaScreenInput -Namespace RpaScreen | Out-Null; try { [RpaScreen.RpaScreenInput]::SetProcessDpiAwarenessContext([IntPtr](-4)) | Out-Null } catch {}";
+
+function screenCenter(selector: UiaSelector): { x: number; y: number; w: number; h: number } {
+  return selector.screen!;
+}
+
+function clickScreenScript(t: UiaTarget): string {
+  const rect = screenCenter(t.selector);
+  const x = Math.round(rect.x + rect.w / 2);
+  const y = Math.round(rect.y + rect.h / 2);
+  const clickway = String(t.args.clickway ?? "0");
+  const doubleClick = clickway === "1" || /double/i.test(clickway);
+  return `${SCREEN_USER32}
+if (-not [RpaScreen.RpaScreenInput]::SetCursorPos(${x}, ${y})) { throw "cursor_move_failed" }
+Start-Sleep -Milliseconds 100
+[RpaScreen.RpaScreenInput]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)
+Start-Sleep -Milliseconds 40
+[RpaScreen.RpaScreenInput]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)
+if (${doubleClick}) {
+  Start-Sleep -Milliseconds 70
+  [RpaScreen.RpaScreenInput]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)
+  Start-Sleep -Milliseconds 40
+  [RpaScreen.RpaScreenInput]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)
+}
+Write-Output (@{ clicked = $true; method = 'screen_coordinate'; x = ${x}; y = ${y} } | ConvertTo-Json -Compress)`;
+}
+
+function setValueScreenScript(t: UiaTarget, value: string): string {
+  const rect = screenCenter(t.selector);
+  const x = Math.round(rect.x + rect.w / 2);
+  const y = Math.round(rect.y + rect.h / 2);
+  return `${SCREEN_USER32}
+Add-Type -AssemblyName System.Windows.Forms
+[RpaScreen.RpaScreenInput]::SetCursorPos(${x}, ${y}) | Out-Null
+Start-Sleep -Milliseconds 120
+[RpaScreen.RpaScreenInput]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)
+Start-Sleep -Milliseconds 40
+[RpaScreen.RpaScreenInput]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)
+Start-Sleep -Milliseconds 220
+$saved = $null
+try { $saved = [System.Windows.Forms.Clipboard]::GetText() } catch {}
+try { [System.Windows.Forms.Clipboard]::SetText(${psString(value)}) } catch { throw "clipboard_failed" }
+Start-Sleep -Milliseconds 80
+[System.Windows.Forms.SendKeys]::SendWait('^a')
+Start-Sleep -Milliseconds 60
+[System.Windows.Forms.SendKeys]::SendWait('^v')
+Start-Sleep -Milliseconds 280
+try {
+  if ($saved) { [System.Windows.Forms.Clipboard]::SetText($saved) } else { [System.Windows.Forms.Clipboard]::Clear() }
+} catch {}
+Write-Output (@{ set = $true; method = 'screen_coordinate_paste'; x = ${x}; y = ${y} } | ConvertTo-Json -Compress)`;
+}
+
+function getValueScreenScript(t: UiaTarget): string {
+  const rect = screenCenter(t.selector);
+  const x = Math.round(rect.x + rect.w / 2);
+  const y = Math.round(rect.y + rect.h / 2);
+  return `${UIA_USING}
+Add-Type -AssemblyName WindowsBase
+$pt = New-Object System.Windows.Point(${x}, ${y})
+$el = [System.Windows.Automation.AutomationElement]::FromPoint($pt)
+if ($null -eq $el) { Write-Output 'JSON_NOT_FOUND'; exit 0 }
+$value = $el.Current.Name
+try { $vp = $el.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern); $value = $vp.Value } catch {}
+Write-Output (@{ value = $value; name = $el.Current.Name } | ConvertTo-Json -Compress)`;
+}
+
 function findScript(t: UiaTarget, timeoutMs: number, gapMs: number): string {
   const s = t.selector;
   return `${UIA_USING}

src/main/desktop-pick.ts

@@ -5,6 +5,7 @@ import fs from "node:fs";
 import path from "node:path";
 import { app, BrowserWindow, screen } from "electron";
 import { RuntimeError } from "./engine/types";
+import { chatCompletion, requireAiConfig } from "./capabilities/ai-service";
 
 const run = promisify(execFile);
 
@@ -17,6 +18,7 @@ function pickerScriptPath(): string {
   if (!found) throw new RuntimeError("picker_unavailable", "桌面拾取脚本缺失", { recommendation: "重新安装应用" });
   return found;
 }
+
 export interface DesktopPickResult {
   cancelled?: boolean;
   window?: string;
@@ -24,36 +26,641 @@ export interface DesktopPickResult {
   automationId?: string;
   controlType?: string;
   className?: string;
+  source?: "uia" | "ai";
+  x?: number;
+  y?: number;
+  w?: number;
+  h?: number;
+  confidence?: number;
+  screenshotPath?: string;
+  aiRequested?: boolean;
   selector: string;
 }
 
-export async function pickDesktopElement(): Promise<DesktopPickResult> {
+interface UiaPickOutput {
+  cancelled?: boolean;
+  aiRequested?: boolean;
+  cursorX?: number;
+  cursorY?: number;
+  paneX?: number;
+  paneY?: number;
+  paneW?: number;
+  paneH?: number;
+  window?: string;
+  name?: string;
+  automationId?: string;
+  controlType?: string;
+  className?: string;
+  x?: number;
+  y?: number;
+  w?: number;
+  h?: number;
+}
+
+interface AiCandidate {
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+  name: string;
+  controlType: string;
+  confidence: number;
+}
+
+interface AiCaptureOutput {
+  cursorX: number;
+  cursorY: number;
+  cropX: number;
+  cropY: number;
+  cropW: number;
+  cropH: number;
+  markerX: number;
+  markerY: number;
+  window?: string;
+  uiaName?: string;
+  uiaType?: string;
+  imagePath: string;
+}
+
+function jsonLine<T>(stdout: string, startsWith: "{" | "["): T {
+  const line = stdout.trim().split(/\r?\n/).reverse().find(item => item.trim().startsWith(startsWith));
+  if (!line) throw new RuntimeError("picker_failed", "桌面拾取未返回结果");
+  return JSON.parse(line) as T;
+}
+
+function safeSelectorText(value: string | undefined): string {
+  return String(value || "").replace(/[;\r\n]+/g, " ").trim();
+}
+
+export async function pickDesktopElement(options: { ai?: boolean } = {}): Promise<DesktopPickResult> {
+  if (options.ai) requireAiConfig({ vision: true });
   const appWindows = BrowserWindow.getAllWindows().filter(w => !w.isDestroyed());
   for (const w of appWindows) w.minimize();
   try {
-    const { stdout } = await run("powershell.exe", ["-NoProfile", "-STA", "-ExecutionPolicy", "Bypass", "-File", pickerScriptPath()], { timeout: 180000, windowsHide: true, maxBuffer: 4 * 1024 * 1024 });
+    if (options.ai) return await runAiPick();
-    const jsonLine = stdout.trim().split(/\r?\n/).reverse().find(l => l.trim().startsWith("{"));
+    const { stdout } = await run(
-    if (!jsonLine) throw new RuntimeError("picker_failed", "拾取未返回结果");
+      "powershell.exe",
-    const parsed = JSON.parse(jsonLine) as Omit<DesktopPickResult, "selector">;
+      ["-NoProfile", "-STA", "-ExecutionPolicy", "Bypass", "-File", pickerScriptPath()],
+      { timeout: 180000, windowsHide: true, maxBuffer: 4 * 1024 * 1024 }
+    );
+    const parsed = jsonLine<UiaPickOutput>(stdout, "{");
     if (parsed.cancelled) return { ...parsed, selector: "" };
+    const paneAiReady = [parsed.paneX, parsed.paneY, parsed.paneW, parsed.paneH]
+      .map(value => Number(value))
+      .every(value => Number.isFinite(value)) && Number(parsed.paneW) > 4 && Number(parsed.paneH) > 4;
+    if (parsed.aiRequested && paneAiReady) return await runPaneAiPick(parsed);
+    if (parsed.aiRequested) return await runAiPick(parsed);
     const parts: string[] = [];
     if (parsed.window) parts.push("window=" + parsed.window);
     if (parsed.name) parts.push("name=" + parsed.name);
     if (parsed.controlType) parts.push("type=" + parsed.controlType);
     if (parsed.automationId) parts.push("id=" + parsed.automationId);
-    return { ...parsed, selector: parts.join(";") };
+    return { ...parsed, source: "uia", selector: parts.join(";") };
   } finally {
     for (const w of appWindows) {
       if (!w.isDestroyed()) w.restore();
     }
   }
 }
-export async function listVisibleWindows(): Promise<Array<{ title: string; process: string }>> {
+
-  const { stdout } = await run("powershell.exe", ["-NoProfile", "-ExecutionPolicy", "Bypass", "-File", pickerScriptPath(), "-Mode", "windows"], { timeout: 30000, windowsHide: true, maxBuffer: 4 * 1024 * 1024 });
+function paneAiOverlayHtml(): string {
-  const jsonLine = stdout.trim().split(/\r?\n/).reverse().find(l => l.trim().startsWith("["));
+  return `<!DOCTYPE html><html><head><meta charset='utf-8'><style>
-  if (!jsonLine) return [];
+html,body{margin:0;width:100%;height:100%;overflow:hidden;background:transparent;cursor:default;font-family:'Microsoft YaHei',Arial,sans-serif}
-  return JSON.parse(jsonLine) as Array<{ title: string; process: string }>;
+#frame{position:absolute;inset:0;display:none;border:1px solid rgba(56,189,248,.30);animation:framePulse 1.9s ease-in-out infinite;z-index:4}#frame i{position:absolute;width:30px;height:30px;border:3px solid #38bdf8;filter:drop-shadow(0 0 7px rgba(56,189,248,.85))}#frame i:nth-child(1){left:2px;top:2px;border-right:0;border-bottom:0;border-radius:10px 0 0 0}#frame i:nth-child(2){right:2px;top:2px;border-left:0;border-bottom:0;border-radius:0 10px 0 0}#frame i:nth-child(3){left:2px;bottom:2px;border-right:0;border-top:0;border-radius:0 0 0 10px}#frame i:nth-child(4){right:2px;bottom:2px;border-left:0;border-top:0;border-radius:0 0 10px 0}
+#scan{position:absolute;left:0;right:0;top:-20%;height:19%;display:none;background:linear-gradient(to bottom,rgba(56,189,248,0),rgba(56,189,248,.06) 48%,rgba(125,211,252,.20) 82%,rgba(224,242,254,.16));z-index:5;animation:scan 1.65s linear infinite;will-change:top}#scan b{position:absolute;left:3%;right:3%;bottom:-2px;height:3px;border-radius:999px;background:linear-gradient(90deg,rgba(56,189,248,0),#7dd3fc 14%,#fff 50%,#7dd3fc 86%,rgba(56,189,248,0));box-shadow:0 0 10px #38bdf8,0 0 28px rgba(56,189,248,.78),0 0 54px rgba(56,189,248,.38)}#scan b:after{content:'';position:absolute;left:0;right:0;top:2px;height:9px;background:linear-gradient(to bottom,rgba(56,189,248,.18),rgba(56,189,248,0))}
+#tip{position:absolute;left:50%;top:18px;transform:translateX(-50%);display:none;padding:8px 15px;border-radius:999px;color:#f8fafc;background:rgba(2,20,40,.78);border:1px solid rgba(125,211,252,.42);box-shadow:0 8px 28px rgba(0,0,0,.28),inset 0 1px 0 rgba(255,255,255,.09);font-size:13px;line-height:1;white-space:nowrap;z-index:2147483647}#tip:before{content:'';display:inline-block;width:7px;height:7px;margin-right:8px;border-radius:50%;background:#38bdf8;box-shadow:0 0 8px #38bdf8;vertical-align:1px}body.scanning #tip:before{animation:tipPulse 1.1s ease-in-out infinite}
+.candidate{position:absolute;border:1px solid rgba(255,255,255,.28);background:transparent;cursor:pointer;box-sizing:border-box}
+.candidate:hover,.candidate.active{border:3px solid #ff2d2d;box-shadow:0 0 0 1px rgba(255,255,255,.45);z-index:2147483646}
+.badge{position:absolute;display:none;padding:3px 7px;border-radius:5px;color:#fff;background:rgba(190,18,18,.95);border:1px solid rgba(255,220,220,.82);font-size:12px;line-height:16px;white-space:nowrap;pointer-events:none;box-shadow:0 4px 12px rgba(0,0,0,.28);z-index:2147483647}.badge.active{display:block}
+@keyframes scan{from{top:-20%}to{top:100%}}@keyframes framePulse{0%,100%{border-color:rgba(56,189,248,.24)}50%{border-color:rgba(56,189,248,.58)}}@keyframes tipPulse{0%,100%{opacity:.45}50%{opacity:1}}
+</style></head><body>
+<div id=frame><i></i><i></i><i></i><i></i></div><div id=scan><b></b></div><div id=tip>AI 识别中…</div>
+<script>
+var paneW=1,paneH=1;
+function startScan(label){paneW=window.__paneW||document.documentElement.clientWidth;paneH=window.__paneH||document.documentElement.clientHeight;document.getElementById('frame').style.display='block';document.getElementById('scan').style.display='block';document.getElementById('tip').style.display='block';document.getElementById('tip').textContent=label||'AI 识别中…';document.body.classList.add('scanning')}
+function showCandidates(items){
+  paneW=window.__paneW||document.documentElement.clientWidth;paneH=window.__paneH||document.documentElement.clientHeight;
+  document.getElementById('scan').style.display='none';document.getElementById('frame').style.display='none';document.getElementById('tip').style.display='block';
+  document.getElementById('tip').textContent='AI 识别完成：移动鼠标高亮；单击选中，再次单击或空格回填；Esc 取消';
+  document.body.classList.remove('scanning');document.body.style.cursor='pointer';
+  window.__hover=null;window.__selected=null;window.__confirmed=null;window.__readyAt=Date.now()+600;
+  items.sort(function(a,b){return a.width*a.height-b.width*b.height}).forEach(function(c,index){
+    var box=document.createElement('div');box.className='candidate';box.__index=index;
+    box.style.left=(c.x/paneW*100)+'%';box.style.top=(c.y/paneH*100)+'%';
+    box.style.width=(c.width/paneW*100)+'%';box.style.height=(c.height/paneH*100)+'%';
+    function activate(){
+      Array.prototype.forEach.call(document.querySelectorAll('.candidate,.badge'),function(n){n.classList.remove('active')});
+      box.classList.add('active');var label=document.getElementById('badge-'+index);if(label)label.classList.add('active');
+      window.__hover=index;window.__hoverCandidate=c;
+    }
+    window.confirmCandidate=function(index,c){
+      var tip=document.getElementById('tip');
+      tip.textContent='已选中：'+(c.name||'控件')+'；再次单击或空格回填，Esc 取消';
+      window.__confirmed=index;window.__selected=index;
+    };
+    function deactivate(){
+      box.classList.remove('active');var label=document.getElementById('badge-'+index);if(label)label.classList.remove('active');
+      if(window.__hover===index){window.__hover=null;window.__hoverCandidate=null}
+    }
+    box.addEventListener('mouseenter',activate);box.addEventListener('mousemove',activate);box.addEventListener('mouseleave',deactivate);
+    box.addEventListener('mousedown',function(e){
+      e.preventDefault();e.stopPropagation();activate();
+      if(Date.now()<(window.__readyAt||0)){console.log('pane-ai ignore early click',index);return}
+      if(window.__confirmed!==index){console.log('pane-ai first select',index);window.confirmCandidate(index,c);return}
+      console.log('pane-ai confirm click',index);window.__selected=index;window.__done=true;
+    });
+    document.body.appendChild(box);
+    var yTop=c.y/paneH*100,yBottom=(c.y+c.height)/paneH*100,leftPct=Math.max(0,Math.min(72,c.x/paneW*100));
+    var badge=document.createElement('div');badge.id='badge-'+index;badge.className='badge';
+    badge.textContent='AI识别 · '+(c.name||'控件');
+    badge.style.left=leftPct+'%';
+    badge.style.top=(yTop<9?'calc('+yBottom+'% + 8px)':'calc('+yTop+'% - 32px)');
+    document.body.appendChild(badge);
+  });
 }
+document.addEventListener('keydown',function(e){
+  if(e.key==='Escape'){window.__cancelled=true}
+  if(e.key===' '&&window.__hoverCandidate&&!window.__done){
+    e.preventDefault();
+    if(Date.now()<(window.__readyAt||0)){console.log('pane-ai ignore early space');return}
+    if(window.__confirmed!==window.__hover){window.confirmCandidate(window.__hover,window.__hoverCandidate);return}
+    window.__selected=window.__hover;window.__done=true;
+  }
+});
+</script></body></html>`;
+}
+
+function paneAiRect(context: UiaPickOutput) {
+  const x = Math.round(Number(context.paneX));
+  const y = Math.round(Number(context.paneY));
+  const w = Math.round(Number(context.paneW));
+  const h = Math.round(Number(context.paneH));
+  if (![x, y, w, h].every(Number.isFinite) || w <= 4 || h <= 4) {
+    throw new RuntimeError("ai_pane_invalid", "Pane 区域无效");
+  }
+  return { x, y, w, h };
+}
+
+async function createPaneAiOverlay(rect: { x: number; y: number; w: number; h: number }): Promise<BrowserWindow> {
+  const dipPoint = screen.screenToDipPoint({ x: rect.x, y: rect.y });
+  const display = displayForPoint(dipPoint.x, dipPoint.y);
+  const scale = display.scaleFactor || 1;
+  const win = new BrowserWindow({
+    x: Math.round(dipPoint.x),
+    y: Math.round(dipPoint.y),
+    width: Math.max(24, Math.round(rect.w / scale)),
+    height: Math.max(24, Math.round(rect.h / scale)),
+    frame: false,
+    transparent: true,
+    resizable: false,
+    movable: false,
+    minimizable: false,
+    maximizable: false,
+    hasShadow: false,
+    alwaysOnTop: true,
+    skipTaskbar: true,
+    show: false,
+    webPreferences: { contextIsolation: true, nodeIntegration: false, sandbox: true }
+  });
+  win.setAlwaysOnTop(true, "screen-saver");
+  win.webContents.on("console-message", (_event, level, message, line, sourceId) => {
+    if (message.includes("pane-ai")) console.log("[pane-ai]", message, sourceId + ":" + line);
+  });
+  await win.loadURL("data:text/html;charset=utf-8," + encodeURIComponent(paneAiOverlayHtml()));
+  await win.webContents.executeJavaScript(
+    "window.__paneW=" + JSON.stringify(rect.w) + ";window.__paneH=" + JSON.stringify(rect.h) +
+    ";window.__cancelled=false;window.__done=false;window.__selected=null;window.__hover=null;window.__hoverCandidate=null;startScan();"
+  );
+  win.show();
+  win.focus();
+  return win;
+}
+
+async function capturePaneForAi(rect: { x: number; y: number; w: number; h: number }): Promise<AiCaptureOutput> {
+  const outDir = path.join(app.getPath("userData"), "ai-picks");
+  fs.mkdirSync(outDir, { recursive: true });
+  const imagePath = path.join(outDir, "ai-pane-" + Date.now() + ".png");
+  const { stdout } = await run(
+    "powershell.exe",
+    [
+      "-NoProfile", "-STA", "-ExecutionPolicy", "Bypass", "-File", pickerScriptPath(),
+      "-Mode", "capture", "-CaptureExact",
+      "-CaptureX", String(rect.x), "-CaptureY", String(rect.y),
+      "-CaptureWidth", String(rect.w), "-CaptureHeight", String(rect.h),
+      "-CapturePath", imagePath
+    ],
+    { timeout: 30000, windowsHide: true, maxBuffer: 2 * 1024 * 1024 }
+  );
+  const parsed = jsonLine<AiCaptureOutput>(stdout, "{");
+  if (!parsed.imagePath || !fs.existsSync(parsed.imagePath)) {
+    throw new RuntimeError("ai_capture_failed", "AI 拾取截图失败");
+  }
+  return parsed;
+}
+
+function parseAiCandidates(text: string, cropW: number, cropH: number): AiCandidate[] {
+  const fence = String.fromCharCode(96);
+  const fencePattern = new RegExp(fence + "{3}(?:json)?" + fence + "{3}", "gi");
+  const cleaned = text.replace(fencePattern, "").trim();
+  const start = cleaned.indexOf("{");
+  const arrayStart = cleaned.indexOf("[");
+  const objectStart = start >= 0 && (start < arrayStart || arrayStart < 0) ? start : arrayStart;
+  const end = Math.max(cleaned.lastIndexOf("}"), cleaned.lastIndexOf("]"));
+  if (objectStart < 0 || end <= objectStart) throw new RuntimeError("ai_response_invalid", "AI 视觉模型没有返回 JSON");
+  const raw = JSON.parse(cleaned.slice(objectStart, end + 1)) as Record<string, unknown>;
+  const list = Array.isArray(raw) ? raw : Array.isArray(raw.candidates) ? raw.candidates : [];
+  const result: AiCandidate[] = [];
+  for (const item of list.slice(0, 40)) {
+    const record = (item || {}) as Record<string, unknown>;
+    const num = (...keys: string[]): number => {
+      for (const key of keys) {
+        const value = Number(record[key]);
+        if (Number.isFinite(value)) return value;
+      }
+      return NaN;
+    };
+    let x = num("x", "left", "x1");
+    let y = num("y", "top", "y1");
+    let width = num("width", "w");
+    let height = num("height", "h");
+    const x2 = num("x2", "right");
+    const y2 = num("y2", "bottom");
+    if (!Number.isFinite(width) && Number.isFinite(x2)) width = x2 - x;
+    if (!Number.isFinite(height) && Number.isFinite(y2)) height = y2 - y;
+    if (![x, y, width, height].every(Number.isFinite) || width < 6 || height < 6) continue;
+    x = Math.max(0, Math.min(cropW - width, x));
+    y = Math.max(0, Math.min(cropH - height, y));
+    width = Math.min(width, cropW - x);
+    height = Math.min(height, cropH - y);
+    if (width * height >= cropW * cropH * 0.97) continue;
+    const confidence = Number(record.confidence);
+    result.push({
+      x: Math.round(x), y: Math.round(y), width: Math.round(width), height: Math.round(height),
+      name: String(record.name ?? "AI识别控件").slice(0, 80) || "AI识别控件",
+      controlType: String(record.controlType ?? "Control").slice(0, 40) || "Control",
+      confidence: Number.isFinite(confidence) ? Math.max(0, Math.min(1, confidence)) : 0.72
+    });
+  }
+  if (!result.length) throw new RuntimeError("ai_response_invalid", "AI 没有识别到可用候选区域");
+  return result;
+}
+
+async function runPaneAiPick(context: UiaPickOutput): Promise<DesktopPickResult> {
+  requireAiConfig({ vision: true });
+  const pane = paneAiRect(context);
+  const win = await createPaneAiOverlay(pane);
+
+  try {
+    // Let the visible scan animation complete over the actual Pane before taking the screenshot.
+    await new Promise(resolve => setTimeout(resolve, 950));
+    win.hide();
+    await new Promise(resolve => setTimeout(resolve, 240));
+    const capture = await capturePaneForAi(pane);
+    win.show();
+    win.focus();
+    await win.webContents.executeJavaScript("startScan('AI 理解中…');");
+
+    const image = await fs.promises.readFile(capture.imagePath);
+    let aiSettled = false;
+    const aiRequest = chatCompletion([
+      {
+        role: "system",
+        content: "你是桌面 RPA 视觉控件分析器。只输出 JSON，不输出解释或 Markdown。"
+      },
+      {
+        role: "user",
+        content: [
+          {
+            type: "text",
+            text: [
+              "这是目标 Pane 区域的截图，尺寸 " + capture.cropW + "x" + capture.cropH + " 像素。",
+              "请把这个 Pane 里的可用控件/内容块拆分成多个候选矩形，最多返回 30 个，按从上到下、从左到右排序。",
+              "优先识别按钮、输入框、列表行、卡片、菜单项、链接和明显可操作区域；不要把整个 Pane 当作候选。",
+              "只返回 JSON：{\"candidates\":[{\"x\":number,\"y\":number,\"width\":number,\"height\":number,\"name\":string,\"controlType\":string,\"confidence\":number}]}",
+              "坐标全部是相对这张截图的整数像素坐标；x/y 是左上角，width/height 也是像素。"
+            ].join("\n")
+          },
+          { type: "image_url", image_url: { url: "data:image/png;base64," + image.toString("base64") } }
+        ]
+      }
+    ], { vision: true, temperature: 0 }).finally(() => { aiSettled = true; });
+
+    const cancelWatcher = (async () => {
+      while (!aiSettled) {
+        try {
+          const state = await win.webContents.executeJavaScript("JSON.stringify({cancelled:window.__cancelled===true})", true);
+          if (JSON.parse(state).cancelled) throw new RuntimeError("pick_cancelled", "已取消 AI 拾取");
+        } catch (error) {
+          if (error instanceof RuntimeError && error.code === "pick_cancelled") throw error;
+        }
+        await new Promise(resolve => setTimeout(resolve, 100));
+      }
+    })();
+
+    let aiText: string;
+    try {
+      aiText = await Promise.race([aiRequest, cancelWatcher]) as string;
+    } finally {
+      aiSettled = true;
+    }
+
+    const candidates = parseAiCandidates(aiText, capture.cropW, capture.cropH);
+    const payload = JSON.stringify(candidates);
+    console.log("[pane-ai] candidates", candidates.length, candidates.map(item => ({ name: item.name, x: item.x, y: item.y, width: item.width, height: item.height })));
+    await win.webContents.executeJavaScript("window.__done=false;window.__cancelled=false;window.__selected=null;window.__hover=null;window.__hoverCandidate=null;window.__confirmed=null;window.__readyAt=Date.now()+600;showCandidates(" + payload + ");");
+
+    const deadline = Date.now() + 180000;
+    let selectedIndex: number | null = null;
+    while (Date.now() < deadline) {
+      await new Promise(resolve => setTimeout(resolve, 80));
+      const state = await win.webContents.executeJavaScript(
+        "JSON.stringify({done:window.__done===true,cancelled:window.__cancelled===true,selected:window.__selected})", true
+      );
+      const parsed = JSON.parse(state) as { done: boolean; cancelled: boolean; selected: number | null };
+      if (parsed.cancelled) throw new RuntimeError("pick_cancelled", "已取消 AI 拾取");
+      if (parsed.done && Number.isInteger(parsed.selected)) {
+        const confirmedIndex = Number(parsed.selected);
+        console.log("[pane-ai] confirmed", confirmedIndex, candidates[confirmedIndex]);
+        selectedIndex = confirmedIndex;
+        break;
+      }
+    }
+    if (selectedIndex === null || !candidates[selectedIndex]) throw new RuntimeError("pick_cancelled", "AI 拾取超时");
+
+    const box = candidates[selectedIndex];
+    const x = capture.cropX + box.x;
+    const y = capture.cropY + box.y;
+    const selectorParts = [
+      "screen=" + [x, y, box.width, box.height].map(Math.round).join(","),
+      context.window ? "window=" + safeSelectorText(context.window) : "",
+      "name=" + safeSelectorText(box.name),
+      "type=" + safeSelectorText(box.controlType),
+      "ai=1"
+    ].filter(Boolean);
+    return {
+      source: "ai",
+      window: context.window,
+      name: box.name,
+      controlType: box.controlType,
+      x, y, w: box.width, h: box.height,
+      confidence: box.confidence,
+      screenshotPath: capture.imagePath,
+      selector: selectorParts.join(";")
+    };
+  } finally {
+    if (!win.isDestroyed()) win.destroy();
+  }
+}
+
+function aiOverlayHtml(): string {
+  return `<!DOCTYPE html><html><head><meta charset='utf-8'><style>
+html,body{margin:0;width:100%;height:100%;overflow:hidden;background:rgba(3,20,45,.12);cursor:crosshair;font-family:'Microsoft YaHei',Arial,sans-serif}
+#tip{position:fixed;top:28px;left:50%;transform:translateX(-50%);padding:10px 20px;border-radius:8px;color:#f8fafc;background:rgba(2,14,34,.82);border:1px solid rgba(96,165,250,.55);font-size:15px;box-shadow:0 8px 30px rgba(0,0,0,.25)}
+#scan{position:absolute;left:0;right:0;top:-240px;height:240px;display:none;background:linear-gradient(to bottom,rgba(59,130,246,0),rgba(59,130,246,.12),rgba(147,197,253,.46),rgba(59,130,246,.10),rgba(59,130,246,0));box-shadow:0 0 38px rgba(96,165,250,.88);animation:scan 1.7s linear infinite}
+#point{position:absolute;width:46px;height:46px;border:2px solid #38bdf8;border-radius:8px;display:none;box-shadow:0 0 20px rgba(56,189,248,.9)}
+body.scanning{background:rgba(3,20,45,.03)}
+body.scanning #scan{display:block}
+body.scanning #tip{font-size:13px}
+@keyframes scan{from{top:-240px}to{top:100%}}
+</style></head><body><div id=scan></div><div id=point></div><div id=tip>AI 拾取：单击要识别的控件；Esc 取消</div><script>
+document.addEventListener('mousedown', function(e) {
+  if (document.body.classList.contains('scanning')) return;
+  var p=document.getElementById('point');
+  p.style.left=(e.clientX-23)+'px';p.style.top=(e.clientY-23)+'px';p.style.display='block';
+  document.getElementById('tip').textContent='AI 视觉识别中…自上而下扫描画面';
+  document.body.classList.add('scanning');
+  window.__point={x:e.clientX,y:e.clientY};window.__done=true;
+});
+document.addEventListener('keydown',function(e){if(e.key==='Escape'){window.__cancelled=true;}});
+</script></body></html>`;
+}
+
+function displayForPoint(x: number, y: number) {
+  return screen.getAllDisplays().find(display => {
+    const b = display.bounds;
+    return x >= b.x && x < b.x + b.width && y >= b.y && y < b.y + b.height;
+  }) || screen.getPrimaryDisplay();
+}
+
+async function createAiOverlay(): Promise<{ win: BrowserWindow; display: ReturnType<typeof screen.getPrimaryDisplay> }> {
+  const cursor = screen.getCursorScreenPoint();
+  const display = displayForPoint(cursor.x, cursor.y);
+  const win = new BrowserWindow({
+    x: display.bounds.x,
+    y: display.bounds.y,
+    width: display.bounds.width,
+    height: display.bounds.height,
+    frame: false,
+    transparent: true,
+    resizable: false,
+    movable: false,
+    minimizable: false,
+    maximizable: false,
+    alwaysOnTop: true,
+    skipTaskbar: true,
+    show: false,
+    webPreferences: { contextIsolation: true, nodeIntegration: false, sandbox: true }
+  });
+  win.setAlwaysOnTop(true, "screen-saver");
+  await win.loadURL("data:text/html;charset=utf-8," + encodeURIComponent(aiOverlayHtml()));
+  win.show();
+  win.focus();
+  return { win, display };
+}
+
+async function captureAiAtPoint(physicalX: number, physicalY: number): Promise<AiCaptureOutput> {
+  const outDir = path.join(app.getPath("userData"), "ai-picks");
+  fs.mkdirSync(outDir, { recursive: true });
+  const imagePath = path.join(outDir, "ai-pick-" + Date.now() + ".png");
+  const { stdout } = await run(
+    "powershell.exe",
+    [
+      "-NoProfile", "-STA", "-ExecutionPolicy", "Bypass", "-File", pickerScriptPath(),
+      "-Mode", "capture", "-CaptureX", String(physicalX), "-CaptureY", String(physicalY),
+      "-CaptureWidth", "1000", "-CaptureHeight", "700", "-CapturePath", imagePath
+    ],
+    { timeout: 30000, windowsHide: true, maxBuffer: 2 * 1024 * 1024 }
+  );
+  const parsed = jsonLine<AiCaptureOutput>(stdout, "{");
+  if (!parsed.imagePath || !fs.existsSync(parsed.imagePath)) {
+    throw new RuntimeError("ai_capture_failed", "AI 拾取截图失败");
+  }
+  return parsed;
+}
+
+function parseAiBox(text: string, cropW: number, cropH: number) {
+  const fence = String.fromCharCode(96);
+  const fencePattern = new RegExp(fence + "{3}(?:json)?" + fence + "{3}", "gi");
+  const cleaned = text.replace(fencePattern, "").trim();
+  const start = cleaned.indexOf("{");
+  const end = cleaned.lastIndexOf("}");
+  if (start < 0 || end <= start) throw new RuntimeError("ai_response_invalid", "AI 视觉模型没有返回 JSON");
+  const raw = JSON.parse(cleaned.slice(start, end + 1)) as Record<string, unknown>;
+  const num = (...keys: string[]): number => {
+    for (const key of keys) {
+      const value = Number(raw[key]);
+      if (Number.isFinite(value)) return value;
+    }
+    return NaN;
+  };
+  const x1 = num("x", "left", "x1");
+  const y1 = num("y", "top", "y1");
+  let width = num("width", "w");
+  let height = num("height", "h");
+  const x2 = num("x2", "right");
+  const y2 = num("y2", "bottom");
+  if (!Number.isFinite(width) && Number.isFinite(x2)) width = x2 - x1;
+  if (!Number.isFinite(height) && Number.isFinite(y2)) height = y2 - y1;
+  if (![x1, y1].every(Number.isFinite)) throw new RuntimeError("ai_response_invalid", "AI 视觉模型缺少目标坐标");
+  if (!Number.isFinite(width) || width <= 0) width = 24;
+  if (!Number.isFinite(height) || height <= 0) height = 18;
+  width = Math.max(4, Math.min(cropW, width));
+  height = Math.max(4, Math.min(cropH, height));
+  const localX = Math.max(0, Math.min(cropW - width, x1));
+  const localY = Math.max(0, Math.min(cropH - height, y1));
+  const confidence = Number(raw.confidence);
+  return {
+    x: localX,
+    y: localY,
+    width,
+    height,
+    name: String(raw.name ?? "AI识别控件"),
+    controlType: String(raw.controlType ?? "Control"),
+    confidence: Number.isFinite(confidence) ? Math.max(0, Math.min(1, confidence)) : 0.7
+  };
+}
+
+async function runAiPick(context: UiaPickOutput = {}): Promise<DesktopPickResult> {
+  requireAiConfig({ vision: true });
+  const { win, display } = await createAiOverlay();
+  try {
+    let point: { x: number; y: number } | null = null;
+    let physicalX = 0;
+    let physicalY = 0;
+    const autoAi = context.aiRequested && Number.isFinite(Number(context.cursorX)) && Number.isFinite(Number(context.cursorY));
+    if (autoAi) {
+      const cursor = screen.getCursorScreenPoint();
+      point = { x: cursor.x - display.bounds.x, y: cursor.y - display.bounds.y };
+      physicalX = Math.round(Number(context.cursorX));
+      physicalY = Math.round(Number(context.cursorY));
+      await win.webContents.executeJavaScript(
+        "var p=document.getElementById('point');p.style.left=(window.__autoX=" + point.x + ")-23+'px';p.style.top=(window.__autoY=" + point.y + ")-23+'px';p.style.display='block';document.getElementById('tip').textContent='AI 视觉识别中…自上而下扫描画面';document.body.classList.add('scanning');window.__done=true;"
+      );
+      await new Promise(resolve => setTimeout(resolve, 120));
+    }
+    const deadline = Date.now() + 120000;
+    while (!autoAi && Date.now() < deadline) {
+      await new Promise(resolve => setTimeout(resolve, 80));
+      try {
+        const state = await win.webContents.executeJavaScript(
+          "JSON.stringify({done:window.__done===true,cancelled:window.__cancelled===true,point:window.__point||null})", true
+        );
+        const parsed = JSON.parse(state) as { done: boolean; cancelled: boolean; point: { x: number; y: number } | null };
+        if (parsed.cancelled) throw new RuntimeError("pick_cancelled", "已取消 AI 拾取");
+        if (parsed.done && parsed.point) {
+          point = parsed.point;
+          break;
+        }
+      } catch (error) {
+        if (error instanceof RuntimeError && error.code === "pick_cancelled") throw error;
+      }
+    }
+    if (!point) throw new RuntimeError("pick_cancelled", "AI 拾取超时");
+
+    // Hide the effect layer so the screen capture contains the real application only.
+    win.hide();
+    await new Promise(resolve => setTimeout(resolve, 180));
+    if (!autoAi) {
+      const scale = display.scaleFactor || 1;
+      physicalX = Math.round((display.bounds.x + point.x) * scale);
+      physicalY = Math.round((display.bounds.y + point.y) * scale);
+    }
+    const capture = await captureAiAtPoint(physicalX, physicalY);
+    win.showInactive();
+    await win.webContents.executeJavaScript(
+      "document.getElementById('tip').textContent='AI 视觉识别中…请稍候';document.body.classList.add('scanning');"
+    );
+
+    const image = await fs.promises.readFile(capture.imagePath);
+    let aiSettled = false;
+    const aiRequest = chatCompletion([
+      {
+        role: "system",
+        content: "你是桌面 RPA 视觉定位器。只输出 JSON，不输出解释或 Markdown。"
+      },
+      {
+        role: "user",
+        content: [
+          {
+            type: "text",
+            text: [
+              "截图尺寸 " + capture.cropW + "x" + capture.cropH + " 像素。",
+              "红黑色十字中心位于 (" + capture.markerX + ", " + capture.markerY + ")。",
+              "请定位十字中心处或半径 35 像素内最小的可交互 UI 控件；不要返回整个窗口或大容器。",
+              "只返回 JSON：{\"x\":number,\"y\":number,\"width\":number,\"height\":number,\"name\":string,\"controlType\":string,\"confidence\":number}",
+              "x/y 是控件左上角在截图内的像素坐标，width/height 单位也是像素。"
+            ].join("\n")
+          },
+          { type: "image_url", image_url: { url: "data:image/png;base64," + image.toString("base64") } }
+        ]
+      }
+    ], { vision: true, temperature: 0 }).finally(() => { aiSettled = true; });
+    const cancelWatcher = (async () => {
+      while (!aiSettled) {
+        try {
+          const state = await win.webContents.executeJavaScript("JSON.stringify({cancelled:window.__cancelled===true})", true);
+          if (JSON.parse(state).cancelled) throw new RuntimeError("pick_cancelled", "已取消 AI 拾取");
+        } catch (error) {
+          if (error instanceof RuntimeError && error.code === "pick_cancelled") throw error;
+        }
+        await new Promise(resolve => setTimeout(resolve, 100));
+      }
+    })();
+    try {
+      var result = (await Promise.race([aiRequest, cancelWatcher])) as string;
+    } finally {
+      aiSettled = true;
+    }
+
+    const box = parseAiBox(result, capture.cropW, capture.cropH);
+    const x = capture.cropX + box.x;
+    const y = capture.cropY + box.y;
+    const w = box.width;
+    const h = box.height;
+    const parts = [
+      "screen=" + [x, y, w, h].map(Math.round).join(","),
+      context.window || capture.window ? "window=" + safeSelectorText(context.window || capture.window) : "",
+      "name=" + safeSelectorText(box.name),
+      "type=" + safeSelectorText(box.controlType || "Control"),
+      "ai=1"
+    ].filter(Boolean);
+    return {
+      source: "ai",
+      window: context.window || capture.window,
+      name: box.name,
+      controlType: box.controlType,
+      x,
+      y,
+      w,
+      h,
+      confidence: box.confidence,
+      screenshotPath: capture.imagePath,
+      selector: parts.join(";")
+    };
+  } finally {
+    if (!win.isDestroyed()) win.destroy();
+  }
+}
+
+export async function listVisibleWindows(): Promise<Array<{ title: string; process: string }>> {
+  const { stdout } = await run(
+    "powershell.exe",
+    ["-NoProfile", "-ExecutionPolicy", "Bypass", "-File", pickerScriptPath(), "-Mode", "windows"],
+    { timeout: 30000, windowsHide: true, maxBuffer: 4 * 1024 * 1024 }
+  );
+  return jsonLine<Array<{ title: string; process: string }>>(stdout, "[");
+}
+
 export async function pickScreenRegion(): Promise<{ x: number; y: number; width: number; height: number; imagePath: string }> {
   const appWindows = BrowserWindow.getAllWindows().filter(w => !w.isDestroyed());
   for (const w of appWindows) w.minimize();
@@ -80,6 +687,7 @@ export async function pickScreenRegion(): Promise<{ x: number; y: number; width:
     "</script></body></html>"
   ];
   void win.loadURL("data:text/html;charset=utf-8," + encodeURIComponent(htmlParts.join("")));
+  win.show();
   const deadline = Date.now() + 60000;
   let region: { x: number; y: number; width: number; height: number } | null = null;
   while (Date.now() < deadline) {

src/main/index.ts

@@ -113,7 +113,7 @@ function registerIpc(): void {
     }
     return session.pickElement();
   });
-  ipcMain.handle("desktop:pick-element", () => pickDesktopElement());
+  ipcMain.handle("desktop:pick-element", (_event, options?: { ai?: boolean }) => pickDesktopElement(options));
   ipcMain.handle("desktop:pick-window-list", () => listVisibleWindows());
   ipcMain.handle("desktop:pick-region", () => pickScreenRegion());
   ipcMain.handle("flows:list", () => listFlows());

src/main/settings.test.ts

@@ -22,14 +22,14 @@ describe("app settings", () => {
   });
 
   it("returns the default element wait when no settings file exists", () => {
-    expect(loadSettings()).toEqual({ browserElementWaitMs: 5000, aiConfig: { baseUrl: "", apiKey: "", model: "" } });
+    expect(loadSettings()).toEqual({ browserElementWaitMs: 5000, aiConfig: { baseUrl: "", apiKey: "", model: "", visionModel: "" } });
   });
 
   it("persists and clamps a user-adjustable element wait", async () => {
     const saved = updateSettings({ browserElementWaitMs: 12500 });
-    expect(saved).toEqual({ browserElementWaitMs: 12500, aiConfig: { baseUrl: "", apiKey: "", model: "" } });
+    expect(saved).toEqual({ browserElementWaitMs: 12500, aiConfig: { baseUrl: "", apiKey: "", model: "", visionModel: "" } });
     expect(JSON.parse(await readFile(path.join(electronState.userData, "settings.json"), "utf8"))).toEqual(saved);
-    expect(updateSettings({ browserElementWaitMs: -5 })).toEqual({ browserElementWaitMs: 0, aiConfig: { baseUrl: "", apiKey: "", model: "" } });
+    expect(updateSettings({ browserElementWaitMs: -5 })).toEqual({ browserElementWaitMs: 0, aiConfig: { baseUrl: "", apiKey: "", model: "", visionModel: "" } });
-    expect(loadSettings()).toEqual({ browserElementWaitMs: 0, aiConfig: { baseUrl: "", apiKey: "", model: "" } });
+    expect(loadSettings()).toEqual({ browserElementWaitMs: 0, aiConfig: { baseUrl: "", apiKey: "", model: "", visionModel: "" } });
   });
 });

src/main/settings.ts

@@ -5,7 +5,7 @@ import type { AppSettings } from "../shared/types";
 
 export const DEFAULT_APP_SETTINGS: AppSettings = {
   browserElementWaitMs: 5000,
-  aiConfig: { baseUrl: "", apiKey: "", model: "" }
+  aiConfig: { baseUrl: "", apiKey: "", model: "", visionModel: "" }
 };
 
 let cache: AppSettings | null = null;
@@ -25,7 +25,8 @@ function normalizeSettings(value: unknown): AppSettings {
     aiConfig: {
       baseUrl: String(ai.baseUrl ?? ""),
       apiKey: String(ai.apiKey ?? ""),
-      model: String(ai.model ?? "")
+      model: String(ai.model ?? ""),
+      visionModel: String(ai.visionModel ?? "")
     }
   };
 }

src/preload/index.ts

@@ -9,7 +9,7 @@ const api = {
     get: (): Promise<Catalog> => ipcRenderer.invoke("catalog:get")
   },
   desktop: {
-    pickElement: () => ipcRenderer.invoke('desktop:pick-element'),
+    pickElement: (options?: { ai?: boolean }) => ipcRenderer.invoke("desktop:pick-element", options || {}),
     pickWindowList: () => ipcRenderer.invoke('desktop:pick-window-list'),
     pickRegion: () => ipcRenderer.invoke('desktop:pick-region')
   },

src/renderer/src/pages/SettingsPage.tsx

@@ -14,7 +14,7 @@ export function SettingsPage({ catalog }: { catalog: Catalog }) {
   const [waitDraft, setWaitDraft] = useState("5000");
   const [waitSaving, setWaitSaving] = useState(false);
   const [waitMessage, setWaitMessage] = useState<string | null>(null);
-  const [aiDraft, setAiDraft] = useState({ baseUrl: "", apiKey: "", model: "" });
+  const [aiDraft, setAiDraft] = useState({ baseUrl: "", apiKey: "", model: "", visionModel: "" });
   const [aiSaving, setAiSaving] = useState(false);
   const [aiMessage, setAiMessage] = useState<string | null>(null);
 
@@ -24,7 +24,12 @@ export function SettingsPage({ catalog }: { catalog: Catalog }) {
     void window.rpa.settings.get().then((value) => {
       setSettings(value);
       setWaitDraft(String(value.browserElementWaitMs));
-      setAiDraft(value.aiConfig ?? { baseUrl: "", apiKey: "", model: "" });
+      setAiDraft({
+          baseUrl: value.aiConfig?.baseUrl ?? "",
+          apiKey: value.aiConfig?.apiKey ?? "",
+          model: value.aiConfig?.model ?? "",
+          visionModel: value.aiConfig?.visionModel ?? ""
+        });
     }).catch(() => setSettings(null));
   }, []);
 
@@ -152,6 +157,11 @@ export function SettingsPage({ catalog }: { catalog: Catalog }) {
                 <label className="field-label" htmlFor="ai-model">模型名称</label>
                 <input id="ai-model" className="text-input" value={aiDraft.model} onChange={(event) => setAiDraft({ ...aiDraft, model: event.target.value })} placeholder="gpt-4o-mini / deepseek-chat / qwen-plus" />
               </div>
+              <div className="field">
+                <label className="field-label" htmlFor="ai-vision-model">视觉模型名称</label>
+                <input id="ai-vision-model" className="text-input" value={aiDraft.visionModel ?? ""} onChange={(event) => setAiDraft({ ...aiDraft, visionModel: event.target.value })} placeholder="gpt-4o-mini / qwen-vl-plus / 留空复用模型名称" />
+                <span className="field-hint">桌面 AI 拾取必须使用多模态视觉模型，并会上传鼠标附近的局部截图。</span>
+              </div>
               <div className="field-row">
                 <button type="button" className="btn btn-primary" onClick={() => void saveAiConfig()} disabled={aiSaving}>
                   <Save className="icon" />

src/renderer/src/panels/Properties.tsx

@@ -471,10 +471,10 @@ function FieldControl({
     onChange(field.key, current === "" || known ? expression : `${textOf(raw)}${expression}`);
   };
 
-  const pickDesktop = async () => {
+  const pickDesktop = async (ai = false) => {
     setPicking(true);
     try {
-      const picked = await window.rpa.desktop.pickElement();
+      const picked = await window.rpa.desktop.pickElement({ ai });
       if (picked.cancelled) {
         onToast("已取消拾取");
         return;
@@ -653,9 +653,14 @@ function FieldControl({
             onChange={(event) => onChange(field.key, event.target.value)}
           />
           {isGroundTarget ? (
-            <button type="button" className="mini-btn" disabled={picking} onClick={() => void pickDesktop()} title="从桌面拾取元素（移到目标按空格）">
+            <>
-            {picking ? "桌面拾取中" : "桌面拾取"}
+              <button type="button" className="mini-btn" disabled={picking} onClick={() => void pickDesktop(false)} title="从桌面拾取元素（移到目标按空格）">
-            </button>
+                {picking ? "桌面拾取中" : "桌面拾取"}
+              </button>
+              <button type="button" className="mini-btn" disabled={picking} onClick={() => void pickDesktop(true)} title="AI 视觉拾取；UIA 类型为 Pane 时也可在桌面拾取中按 F2">
+                {picking ? "AI识别中" : "AI拾取"}
+              </button>
+            </>
           ) : (
             <button type="button" className="mini-btn" disabled={picking} onClick={() => void pickElement()} title="从运行浏览器拾取元素">
             {picking ? "拾取中" : "拾取"}

src/shared/types.ts

@@ -184,6 +184,8 @@ export interface AiServiceConfig {
   baseUrl: string;
   apiKey: string;
   model: string;
+  /** 桌面 AI 拾取使用的视觉模型；留空时复用 model。 */
+  visionModel?: string;
 }
 
 export interface AppSettings {
@@ -191,10 +193,6 @@ export interface AppSettings {
   aiConfig: AiServiceConfig;
 }
 
-export interface AppSettings {
-  browserElementWaitMs: number;
-}
-
 export interface ElementPickResult {
   selector: string;
   tag: string;