diff --git a/.gitignore b/.gitignore index f6d70ea..ba1e6e9 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ test-output/ web-capability-audit.txt *.traineddata + +temp/ diff --git a/electron-builder.config.mjs b/electron-builder.config.mjs index 24716dc..14c707d 100644 --- a/electron-builder.config.mjs +++ b/electron-builder.config.mjs @@ -23,6 +23,8 @@ export default { extraResources: [ { from: path.join(root, "resources", "catalog.json"), to: "catalog.json" }, { from: path.join(root, "resources", "excluded.json"), to: "excluded.json" }, + { from: path.join(root, "resources", "desktop-picker.ps1"), to: "desktop-picker.ps1" }, + { from: path.join(root, "resources", "models"), to: "models" }, { from: path.join(root, "resources", "fixture"), to: "fixture" } ], asar: true, diff --git a/electron.vite.config.ts b/electron.vite.config.ts index 9d4947b..ef3260b 100644 --- a/electron.vite.config.ts +++ b/electron.vite.config.ts @@ -12,7 +12,7 @@ export default defineConfig({ }, build: { rollupOptions: { - input: { index: resolve("src/main/index.ts") } + input: { index: resolve("src/main/index.ts"), "local-vision/ui-detect-worker": resolve("src/main/local-vision/ui-detect-worker.ts") } } } }, diff --git a/package-lock.json b/package-lock.json index 96e6e25..d7b6aca 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,6 +22,7 @@ "mssql": "^12.7.13", "mysql2": "^3.24.6", "nodemailer": "^6.9.16", + "onnxruntime-node": "^1.31.1", "pdf-lib": "^1.17.1", "pdfjs-dist": "^4.10.38", "pg": "^8.23.1", @@ -2362,7 +2363,7 @@ }, "node_modules/@napi-rs/canvas/node_modules/@napi-rs/canvas-linux-riscv64-gnu": { "version": "1.1.0", - "resolved": "https://registry.npmmirror.com/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-1.1.0.tgz", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-1.1.0.tgz", "integrity": "sha512-kJ7m5n2VffxWXM5QGa5lfq8ZN4LdOJEoz3aAJAmIqYvKiZw1gRdXqpucnRXsF6D9nLYiiIlevaAtAclR7c5ilg==", "cpu": [ "riscv64" @@ -8452,7 +8453,7 @@ }, "node_modules/onnxruntime-node": { "version": "1.31.1", - "resolved": "https://registry.npmmirror.com/onnxruntime-node/-/onnxruntime-node-1.31.1.tgz", + "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.31.1.tgz", "integrity": "sha512-ijnSghbDHVWaQ2nDPmVccPlCHI8kH1Mx1aJ4a/qzRP16bBcVs4E3EqZQKZCkhWnbqN1tuQWxagimFtDpbPYYQw==", "hasInstallScript": true, "license": "MIT", diff --git a/package.json b/package.json index d5d0e35..d73ed34 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "mssql": "^12.7.13", "mysql2": "^3.24.6", "nodemailer": "^6.9.16", + "onnxruntime-node": "^1.31.1", "pdf-lib": "^1.17.1", "pdfjs-dist": "^4.10.38", "pg": "^8.23.1", diff --git a/resources/desktop-picker.ps1 b/resources/desktop-picker.ps1 index 43c5def..4f45246 100644 --- a/resources/desktop-picker.ps1 +++ b/resources/desktop-picker.ps1 @@ -150,10 +150,15 @@ function Set-ClickThrough($form) { } function Set-HighlightBox([int]$x, [int]$y, [int]$w, [int]$h) { - $hl.Location = New-Object System.Drawing.Point(($x - 2), ($y - 2)) - $hl.Size = New-Object System.Drawing.Size(($w + 4), ($h + 4)) - $hl.Show() - Set-ClickThrough $hl + $bounds = New-Object System.Drawing.Rectangle(($x - 2), ($y - 2), ($w + 4), ($h + 4)) + if (-not $hl.Visible) { + $hl.Bounds = $bounds + $hl.Show() + Set-ClickThrough $hl + } elseif ($hl.Bounds.X -ne $bounds.X -or $hl.Bounds.Y -ne $bounds.Y -or $hl.Bounds.Width -ne $bounds.Width -or $hl.Bounds.Height -ne $bounds.Height) { + # 不重设隐藏;同一元素内移动时不重复改窗口,避免 WinForms 分层窗体重绘闪烁 + $hl.Bounds = $bounds + } } function Hide-Highlight { $hl.Hide() } function Get-DeepestAtPoint([double]$px, [double]$py) { @@ -187,6 +192,19 @@ $prevSpace = $false $prevPgUp = $false $prevPgDn = $false $prevF2 = $false +$cacheX = [int]::MinValue +$cacheY = [int]::MinValue +$cacheEl = $null +$cacheCtrlName = "" +$cacheName = "" +$cachePaneX = 0 +$cachePaneY = 0 +$cachePaneW = 0 +$cachePaneH = 0 +$cacheAt = [DateTime]::MinValue +$lastQueryAt = [DateTime]::MinValue +$queryGapMs = 45 +$cacheTtlMs = 1800 Write-PickLog "loop start" while ($true) { try { @@ -199,17 +217,49 @@ while ($true) { if ($stage -eq "hover") { $ptX = [System.Windows.Forms.Cursor]::Position.X $ptY = [System.Windows.Forms.Cursor]::Position.Y - $el = Get-DeepestAtPoint $ptX $ptY + $now = [DateTime]::UtcNow + $samePoint = ($ptX -eq $cacheX -and $ptY -eq $cacheY) + $insideCache = $cacheEl -ne $null -and $ptX -ge $cachePaneX -and $ptX -lt ($cachePaneX + $cachePaneW) -and $ptY -ge $cachePaneY -and $ptY -lt ($cachePaneY + $cachePaneH) + $cacheAlive = ($null -ne $cacheEl -and ($now - $cacheAt).TotalMilliseconds -le $cacheTtlMs) + # 鼠标还在当前元素矩形内部时不丢弃缓存;UIA 查询完成后才切换,避免移动过程先隐藏再显示 + if (-not $samePoint -and -not $insideCache) { $cacheEl = $null } + if (-not $cacheAlive -and ($now - $lastQueryAt).TotalMilliseconds -ge $queryGapMs) { + $queryX = $ptX + $queryY = $ptY + $lastQueryAt = [DateTime]::UtcNow + try { $queried = Get-DeepestAtPoint $queryX $queryY } catch { $queried = $null; Write-PickLog ("QUERY ERR: " + $_.Exception.Message) } + $afterX = [System.Windows.Forms.Cursor]::Position.X + $afterY = [System.Windows.Forms.Cursor]::Position.Y + if ($afterX -ne $queryX -or $afterY -ne $queryY) { + $queried = $null + Write-PickLog "query stale: cursor moved" + } + if ($null -ne $queried) { + $qc = $queried.Current + $qr = $qc.BoundingRectangle + $cacheEl = $queried + $cacheX = $queryX + $cacheY = $queryY + $cacheAt = [DateTime]::UtcNow + $cacheCtrlName = $qc.ControlType.ProgrammaticName.Replace("ControlType.", "") + $cacheName = [string]$qc.Name + $cachePaneX = [int]$qr.X + $cachePaneY = [int]$qr.Y + $cachePaneW = [int]$qr.Width + $cachePaneH = [int]$qr.Height + } else { + $cacheEl = $null + $cacheCtrlName = "" + $cacheName = "" + } + } + $el = $cacheEl if ($el) { - $c = $el.Current - $ctrlName = $c.ControlType.ProgrammaticName.Replace("ControlType.", "") - $r = $c.BoundingRectangle - $paneX = [int]$r.X; $paneY = [int]$r.Y; $paneW = [int]$r.Width; $paneH = [int]$r.Height - Set-HighlightBox $paneX $paneY $paneW $paneH - $aiHint = if ($ctrlName -eq "Pane") { " [F2]AI识别" } else { "" } - $infoLabel.Text = "名称: " + $c.Name + " 类型: " + $ctrlName + " [空格]锁定" + $aiHint + " [Esc]取消" + Set-HighlightBox $cachePaneX $cachePaneY $cachePaneW $cachePaneH + $aiHint = if ($cacheCtrlName -eq "Pane") { " [F2]AI识别" } else { "" } + $infoLabel.Text = "名称: " + $cacheName + " 类型: " + $cacheCtrlName + " [空格]锁定" + $aiHint + " [Esc]取消" } else { Hide-Highlight } - if ($f2 -and -not $prevF2 -and $el -and $ctrlName -eq "Pane") { + if ($f2 -and -not $prevF2 -and $el -and $cacheCtrlName -eq "Pane") { Write-PickLog "f2 ai" $walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker $cur = $el @@ -218,7 +268,7 @@ while ($true) { $topTitle = [string]$cur.Current.Name $cur = $walker.GetParent($cur) } - $result = @{ aiRequested = $true; cursorX = $ptX; cursorY = $ptY; window = $topTitle; name = [string]$c.Name; controlType = $ctrlName; paneX = $paneX; paneY = $paneY; paneW = $paneW; paneH = $paneH } + $result = @{ aiRequested = $true; cursorX = $ptX; cursorY = $ptY; window = $topTitle; name = $cacheName; controlType = $cacheCtrlName; paneX = $cachePaneX; paneY = $cachePaneY; paneW = $cachePaneW; paneH = $cachePaneH } break } if ($space -and -not $prevSpace -and $el) { @@ -226,7 +276,7 @@ while ($true) { $stage = "locked" $chain = New-Object System.Collections.ArrayList $walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker - $cur = $el + $cur = $cacheEl while ($cur -and -not $cur.Equals([System.Windows.Automation.AutomationElement]::RootElement)) { $c = $cur.Current $r = $c.BoundingRectangle diff --git a/resources/models/ui-icon-detect/model.onnx b/resources/models/ui-icon-detect/model.onnx new file mode 100644 index 0000000..9189fde Binary files /dev/null and b/resources/models/ui-icon-detect/model.onnx differ diff --git a/resources/models/ui-icon-detect/preprocessor_config.json b/resources/models/ui-icon-detect/preprocessor_config.json new file mode 100644 index 0000000..4ce8c7b --- /dev/null +++ b/resources/models/ui-icon-detect/preprocessor_config.json @@ -0,0 +1,12 @@ +{ + "do_normalize": false, + "do_rescale": true, + "do_resize": true, + "feature_extractor_type": "ImageFeatureExtractor", + "resample": 2, + "rescale_factor": 0.00392156862745098, + "size_divisor": 16, + "size": { + "longest_edge": 640 + } +} \ No newline at end of file diff --git a/scripts/package.mjs b/scripts/package.mjs index 438166d..d3c9979 100644 --- a/scripts/package.mjs +++ b/scripts/package.mjs @@ -115,9 +115,13 @@ if (!fs.existsSync(path.join(builderUnpacked, "RPA Studio.exe"))) { } const packagedCatalogPath = path.join(builderUnpacked, "resources", "catalog.json"); const packagedFixturePath = path.join(builderUnpacked, "resources", "fixture", "index.html"); +const packagedPickerPath = path.join(builderUnpacked, "resources", "desktop-picker.ps1"); if (!fs.existsSync(packagedCatalogPath) || !fs.existsSync(packagedFixturePath)) { throw new Error("打包产物缺少运行资源:resources/catalog.json 或 resources/fixture/index.html"); } +if (!fs.existsSync(packagedPickerPath)) { + throw new Error("打包产物缺少桌面拾取脚本:resources/desktop-picker.ps1 未随包发布"); +} const packagedCatalog = JSON.parse(fs.readFileSync(packagedCatalogPath, "utf8")); if (!Array.isArray(packagedCatalog.components) || packagedCatalog.components.length === 0) { throw new Error("打包产物 resources/catalog.json 无效:组件目录为空"); @@ -216,6 +220,9 @@ if (!fs.existsSync(path.join(installSmokeTarget, "RPA Studio.exe"))) { if (!fs.existsSync(path.join(installSmokeTarget, "resources", "catalog.json"))) { throw new Error("NSIS 安装包冒烟失败:catalog.json 未安装"); } +if (!fs.existsSync(path.join(installSmokeTarget, "resources", "desktop-picker.ps1"))) { + throw new Error("NSIS 安装包冒烟失败:desktop-picker.ps1 未安装"); +} await verifyLaunch(path.join(installSmokeTarget, "RPA Studio.exe"), "installed"); removeTree(installSmokeRoot); diff --git a/src/main/browser/session.ts b/src/main/browser/session.ts index eabc071..5c383f1 100644 --- a/src/main/browser/session.ts +++ b/src/main/browser/session.ts @@ -233,6 +233,18 @@ export class BrowserSession implements BrowserLike { return !name || accessibleName.includes(name); }) || null; } + if (spec.kind === 'css') { + const segments = String(spec.value).split('>>>').map((item) => item.trim()).filter(Boolean); + let root = document; + for (let index = 0; index < segments.length; index += 1) { + const found = root.querySelector(segments[index]); + if (!found) return null; + if (index === segments.length - 1) return found; + root = found.shadowRoot; + if (!root) return null; + } + return null; + } return document.querySelector(spec.value); }; const el = find(); @@ -269,33 +281,46 @@ export class BrowserSession implements BrowserLike { try { return document.querySelectorAll(candidate).length === 1; } catch { return false; } }; const cssPath = (node) => { - if (node.id) { - const candidate = '#' + cssEscape(node.id); - if (isUnique(candidate)) return candidate; - } - const testId = node.getAttribute('data-testid'); - if (testId) { - const candidate = '[data-testid="' + cssEscape(testId) + '"]'; - if (isUnique(candidate)) return candidate; - } - const parts = []; + const localPath = (start, root) => { + const stable = (item) => { + if (item.id) { + const candidate = '#' + cssEscape(item.id); + try { if (root.querySelectorAll(candidate).length === 1) return candidate; } catch {} + } + const testId = item.getAttribute('data-testid'); + if (testId) { + const candidate = '[data-testid="' + cssEscape(testId) + '"]'; + try { if (root.querySelectorAll(candidate).length === 1) return candidate; } catch {} + } + return null; + }; + const parts = []; + let current = start; + while (current && current.nodeType === 1) { + const stableCandidate = stable(current); + if (stableCandidate) { + parts.unshift(stableCandidate); + return parts.join(' > '); + } + let part = current.tagName.toLowerCase(); + const parent = current.parentElement; + if (parent) { + const siblings = [...parent.children].filter((item) => item.tagName === current.tagName); + if (siblings.length > 1) part += ':nth-of-type(' + (siblings.indexOf(current) + 1) + ')'; + } + parts.unshift(part); + current = parent; + } + return parts.join(' > '); + }; + const trees = []; let current = node; while (current && current.nodeType === 1) { - let part = current.tagName.toLowerCase(); - if (current.id) { - part += '#' + cssEscape(current.id); - parts.unshift(part); - break; - } - const parent = current.parentElement; - if (parent) { - const siblings = [...parent.children].filter((item) => item.tagName === current.tagName); - if (siblings.length > 1) part += ':nth-of-type(' + (siblings.indexOf(current) + 1) + ')'; - } - parts.unshift(part); - current = parent; + const root = current.getRootNode(); + trees.unshift(localPath(current, root)); + current = root instanceof ShadowRoot ? root.host : null; } - return parts.join(' > '); + return trees.join(' >>> '); }; const describe = (node) => ({ kind: 'css', @@ -1143,33 +1168,46 @@ export class BrowserSession implements BrowserLike { try { return document.querySelectorAll(candidate).length === 1; } catch { return false; } }; const cssPath = (node) => { - if (node.id) { - const candidate = '#' + cssEscape(node.id); - if (isUnique(candidate)) return candidate; - } - const testId = node.getAttribute('data-testid'); - if (testId) { - const candidate = '[data-testid="' + cssEscape(testId) + '"]'; - if (isUnique(candidate)) return candidate; - } - const parts = []; + const localPath = (start, root) => { + const stable = (item) => { + if (item.id) { + const candidate = '#' + cssEscape(item.id); + try { if (root.querySelectorAll(candidate).length === 1) return candidate; } catch {} + } + const testId = item.getAttribute('data-testid'); + if (testId) { + const candidate = '[data-testid="' + cssEscape(testId) + '"]'; + try { if (root.querySelectorAll(candidate).length === 1) return candidate; } catch {} + } + return null; + }; + const parts = []; + let current = start; + while (current && current.nodeType === 1) { + const stableCandidate = stable(current); + if (stableCandidate) { + parts.unshift(stableCandidate); + return parts.join(' > '); + } + let part = current.tagName.toLowerCase(); + const parent = current.parentElement; + if (parent) { + const siblings = [...parent.children].filter((item) => item.tagName === current.tagName); + if (siblings.length > 1) part += ':nth-of-type(' + (siblings.indexOf(current) + 1) + ')'; + } + parts.unshift(part); + current = parent; + } + return parts.join(' > '); + }; + const trees = []; let current = node; while (current && current.nodeType === 1) { - let part = current.tagName.toLowerCase(); - if (current.id) { - part += '#' + cssEscape(current.id); - parts.unshift(part); - break; - } - const parent = current.parentElement; - if (parent) { - const siblings = [...parent.children].filter((item) => item.tagName === current.tagName); - if (siblings.length > 1) part += ':nth-of-type(' + (siblings.indexOf(current) + 1) + ')'; - } - parts.unshift(part); - current = parent; + const root = current.getRootNode(); + trees.unshift(localPath(current, root)); + current = root instanceof ShadowRoot ? root.host : null; } - return parts.join(' > '); + return trees.join(' >>> '); }; const describe = (node) => { const rect = node.getBoundingClientRect(); @@ -1200,9 +1238,33 @@ export class BrowserSession implements BrowserLike { badge.style.left = Math.max(0, rect.left) + 'px'; badge.style.top = Math.max(0, rect.top - 30) + 'px'; }; + const interactiveSelector = [ + 'button', 'a', 'input', 'select', 'textarea', 'summary', 'label', 'option', + '[role]', '[onclick]', '[contenteditable]', '[tabindex]' + ].join(','); + const closestInteractive = (start) => { + let node = start; + while (node && node.nodeType === 1) { + try { if (node.matches(interactiveSelector)) return node; } catch {} + if (node.parentElement) { + node = node.parentElement; + continue; + } + const root = node.getRootNode(); + node = root instanceof ShadowRoot ? root.host : null; + } + return start; + }; const firstElement = (event) => { - for (const item of event.composedPath()) if (item instanceof Element) return item; - return event.target instanceof Element ? event.target : null; + let deepest = null; + for (const item of event.composedPath()) { + if (item instanceof Element) { + deepest = item; + break; + } + } + if (!deepest && event.target instanceof Element) deepest = event.target; + return closestInteractive(deepest); }; const move = (event) => { if (finished) return; diff --git a/src/main/capabilities/ai-service.ts b/src/main/capabilities/ai-service.ts index 2dcb5a6..e859c11 100644 --- a/src/main/capabilities/ai-service.ts +++ b/src/main/capabilities/ai-service.ts @@ -34,16 +34,29 @@ export function requireAiConfig(options: { vision?: boolean } = {}): ResolvedAiC return config; } -export async function chatCompletion(messages: ChatMessage[], options: { temperature?: number; vision?: boolean } = {}): Promise { +export interface ChatCompletionOptions { + temperature?: number; + vision?: boolean; + /** 交互式请求(用户正在等待,如 AI 拾取):默认短超时 45s、最多 2 次,超时不重试 */ + interactive?: boolean; + /** 单次请求超时毫秒数;显式指定后覆盖交互式/后台默认值 */ + timeoutMs?: number; + /** 最大尝试次数;显式指定后覆盖交互式/后台默认值 */ + maxAttempts?: number; +} + +export async function chatCompletion(messages: ChatMessage[], options: ChatCompletionOptions = {}): Promise { const config = requireAiConfig({ vision: options.vision }); const base = config.baseUrl.replace(/\/+$/, ""); - const maxAttempts = 3; + const interactive = options.interactive === true; + const timeoutMs = Math.max(1000, Math.round(options.timeoutMs ?? (interactive ? 45000 : 170000))); + const maxAttempts = Math.max(1, Math.round(options.maxAttempts ?? (interactive ? 2 : 3))); for (let attempt = 1; attempt <= maxAttempts; attempt++) { let response: Response; try { response = await fetch(base + "/chat/completions", { - signal: AbortSignal.timeout(170000), + signal: AbortSignal.timeout(timeoutMs), method: "POST", headers: { "Content-Type": "application/json", Authorization: "Bearer " + config.apiKey }, body: JSON.stringify({ @@ -53,6 +66,10 @@ export async function chatCompletion(messages: ChatMessage[], options: { tempera }) }); } catch (error) { + // 超时(AbortSignal.timeout)不重试:重发只会成倍拉长等待 + if ((error as { name?: string }).name === "TimeoutError") { + throw new RuntimeError("ai_request_timeout", `AI 请求超时(${timeoutMs}ms),请检查网络或稍后重试`); + } if (attempt === maxAttempts) throw error; await new Promise(resolve => setTimeout(resolve, 1200 * attempt)); continue; diff --git a/src/main/capabilities/cv-vision.test.ts b/src/main/capabilities/cv-vision.test.ts new file mode 100644 index 0000000..f5ece19 --- /dev/null +++ b/src/main/capabilities/cv-vision.test.ts @@ -0,0 +1,50 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const powershellMock = vi.hoisted(() => vi.fn(async () => '{"ok":true}')); +const detectMock = vi.hoisted(() => vi.fn(async () => ({ + texts: [{ text: "确认保存", box: [[0, 0], [20, 0], [20, 10], [0, 10]] }] +}))); + +vi.mock("./desktop", () => ({ powershell: powershellMock })); +vi.mock("@gutenye/ocr-node", () => ({ + default: { create: vi.fn(async () => ({ detect: detectMock })) } +})); + +import { getHandler } from "../engine/registry"; +import "./cv-vision"; + +describe("CV vision capabilities", () => { + beforeEach(() => { + powershellMock.mockClear(); + detectMock.mockClear(); + }); + + it.each(["CV_GetText", "CV_SetText", "CV_MouseHover", "CV_MouseClick"])("registers %s", (id) => { + expect(getHandler(id), id).toBeTruthy(); + }); + + it("rejects an empty CV selector", async () => { + await expect(getHandler("CV_GetText")!({ log: vi.fn() } as any, { id: "n1", componentId: "CV_GetText" } as any, {}, {} as any)) + .rejects.toThrow("CV 目标为空"); + }); + + it("locates OCR text and clicks its center with offset", async () => { + const handler = getHandler("CV_MouseClick")!; + await expect(handler({ log: vi.fn() } as any, { id: "n1", componentId: "CV_MouseClick" } as any, { + cvSelector: "确认", + offsetX: 5, + offsetY: 2 + }, {} as any)).resolves.toBe(true); + expect(detectMock).toHaveBeenCalledTimes(1); + const script = String((powershellMock.mock.calls as unknown as Array<[string]>)[1]?.[0] ?? ""); + expect(script).toContain("SetCursorPos(15, 7)"); + expect(script).toContain("mouse_event(0x0002"); + }); + + it("returns matched OCR text", async () => { + await expect(getHandler("CV_GetText")!({ log: vi.fn() } as any, { id: "n2", componentId: "CV_GetText" } as any, { + cvSelector: "保存" + }, {} as any)).resolves.toBe("确认保存"); + expect(powershellMock).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/main/capabilities/cv-vision.ts b/src/main/capabilities/cv-vision.ts index 88222ac..d3935ee 100644 --- a/src/main/capabilities/cv-vision.ts +++ b/src/main/capabilities/cv-vision.ts @@ -1,5 +1,5 @@ import { register } from "../engine/registry"; -import { RuntimeError } from "../engine/types"; +import { RuntimeError, type CapabilityHandler } from "../engine/types"; import { powershell } from "./desktop"; import fs from "node:fs"; import os from "node:os"; @@ -57,4 +57,95 @@ export async function locateTextOnScreen(target: string, nodeId: string): Promis const cx = (Math.min(...xs) + Math.max(...xs)) / 2; const cy = (Math.min(...ys) + Math.max(...ys)) / 2; return { target: keyword, text: hit.text, cx, cy }; -} \ No newline at end of file +} +function screenUser32(): string { + return [ + "Add-Type -MemberDefinition '[DllImport(\"user32.dll\")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value); [DllImport(\"user32.dll\")] public static extern bool SetCursorPos(int X, int Y); [DllImport(\"user32.dll\")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra);' -Name RpaCvInput -Namespace RpaCv | Out-Null;", + "try { [RpaCv.RpaCvInput]::SetProcessDpiAwarenessContext([IntPtr](-4)) | Out-Null } catch {}" + ].join("\n"); +} + +function coordinateScript(x: number, y: number, action: "hover" | "click" | "doubleClick" | "paste", value = ""): string { + const lines = [ + screenUser32(), + "if (-not [RpaCv.RpaCvInput]::SetCursorPos(" + x + ", " + y + ")) { throw 'cursor_move_failed' }", + "Start-Sleep -Milliseconds 100" + ]; + if (action !== "hover") { + lines.push( + "[RpaCv.RpaCvInput]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)", + "Start-Sleep -Milliseconds 40", + "[RpaCv.RpaCvInput]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)" + ); + } + if (action === "doubleClick") { + lines.push( + "Start-Sleep -Milliseconds 70", + "[RpaCv.RpaCvInput]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)", + "Start-Sleep -Milliseconds 40", + "[RpaCv.RpaCvInput]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)" + ); + } + if (action === "paste") { + lines.push( + "Add-Type -AssemblyName System.Windows.Forms", + "Start-Sleep -Milliseconds 220", + "$saved = $null", + "try { $saved = [System.Windows.Forms.Clipboard]::GetText() } catch {}", + "try { [System.Windows.Forms.Clipboard]::SetText(" + JSON.stringify(value) + ") } catch { throw 'clipboard_failed' }", + "Start-Sleep -Milliseconds 80", + "[System.Windows.Forms.SendKeys]::SendWait('^a')", + "Start-Sleep -Milliseconds 60", + "[System.Windows.Forms.SendKeys]::SendWait('^v')", + "Start-Sleep -Milliseconds 280", + "try { if ($saved) { [System.Windows.Forms.Clipboard]::SetText($saved) } else { [System.Windows.Forms.Clipboard]::Clear() } } catch {}" + ); + } + lines.push("Write-Output (@{ ok = $true; x = " + x + "; y = " + y + " } | ConvertTo-Json -Compress)"); + return lines.join("\n"); +} + +async function locateFor(args: Record, node: { id: string; componentId: string }) { + const target = cvTarget(args, node.id); + const located = await locateTextOnScreen(target, node.id); + const offsetX = Number(args.offsetX ?? args.offset_x ?? 0) || 0; + const offsetY = Number(args.offsetY ?? args.offset_y ?? 0) || 0; + return { located, x: Math.round(located.cx + offsetX), y: Math.round(located.cy + offsetY) }; +} + +const getTextGround: CapabilityHandler = async (ctx, node, args) => { + const target = cvTarget(args, node.id); + const result = await locateTextOnScreen(target, node.id); + ctx.log("CV_GetText found: " + result.text, "info", node.id); + return result.text; +}; + +const setTextGround: CapabilityHandler = async (ctx, node, args) => { + const value = String(args.value ?? args.text ?? ""); + if (!value) throw new RuntimeError("validation_error", "CV 文本输入内容为空", { nodeId: node.id, componentId: node.componentId }); + const { x, y } = await locateFor(args, node); + await powershell(coordinateScript(x, y, "paste", value), 30000); + ctx.log("CV_SetText input at " + x + "," + y, "info", node.id); + return true; +}; + +const mouseHoverGround: CapabilityHandler = async (ctx, node, args) => { + const { x, y } = await locateFor(args, node); + await powershell(coordinateScript(x, y, "hover"), 15000); + ctx.log("CV_MouseHover hover at " + x + "," + y, "info", node.id); + return true; +}; + +const mouseClickGround: CapabilityHandler = async (ctx, node, args) => { + const { x, y } = await locateFor(args, node); + const clickway = String(args.clickway ?? "0"); + const action = clickway === "1" || /double/i.test(clickway) ? "doubleClick" as const : "click" as const; + await powershell(coordinateScript(x, y, action), 15000); + ctx.log("CV_MouseClick " + action + " at " + x + "," + y, "info", node.id); + return true; +}; + +register("CV_GetText", getTextGround); +register("CV_SetText", setTextGround); +register("CV_MouseHover", mouseHoverGround); +register("CV_MouseClick", mouseClickGround); diff --git a/src/main/capabilities/desktop-uia.test.ts b/src/main/capabilities/desktop-uia.test.ts index c83024b..c4c27a7 100644 --- a/src/main/capabilities/desktop-uia.test.ts +++ b/src/main/capabilities/desktop-uia.test.ts @@ -1,7 +1,16 @@ -import { describe, expect, it } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const powershellMock = vi.hoisted(() => vi.fn(async () => '{"clicked":true,"value":"cached"}')); + +vi.mock("./desktop", () => ({ + powershell: powershellMock, + psString: (value: unknown) => JSON.stringify(String(value ?? "")) +})); + import { getHandler } from "../engine/registry"; import { parseUiaSelector } from "./desktop-uia"; + describe("desktop UIA ground capabilities", () => { it.each([ "Element_ClickGround", @@ -21,7 +30,10 @@ describe("desktop UIA ground capabilities", () => { "Element_MouseToGround", "Element_ScrollIntoViewGround", "Element_ChildrenGround", - "Element_DesktopChildren" + "Element_DesktopChildren", + "Element_RestoreWindowGround", + "Element_ActivateWindowGround", + "Element_MoveWindowGround" ])("registers %s", (id) => { expect(getHandler(id), id).toBeTruthy(); }); @@ -57,3 +69,61 @@ describe("desktop UIA ground capabilities", () => { expect(() => parseUiaSelector("screen=1,2,3")).toThrow(); }); }); + +describe("desktop UIA selector compatibility", () => { + beforeEach(() => { + powershellMock.mockClear(); + powershellMock.mockImplementation(async () => '{"clicked":true,"value":"cached"}'); + }); + + it("maps compound .NET ControlType names explicitly and preserves raw input", () => { + expect(parseUiaSelector("type=combobox")).toMatchObject({ type: "ComboBox", typeRaw: "combobox" }); + expect(parseUiaSelector("type=listitem")).toMatchObject({ type: "ListItem", typeRaw: "listitem" }); + expect(parseUiaSelector("type=datagrid")).toMatchObject({ type: "DataGrid", typeRaw: "datagrid" }); + }); + + it("parses a stable ancestor path", () => { + const selector = parseUiaSelector("window=App;path=Pane>Edit[name=用户名;index=2]>Button[name=确定]"); + expect(selector.path).toEqual([ + { type: "Pane" }, + { type: "Edit", name: "用户名", index: 2 }, + { type: "Button", name: "确定" } + ]); + }); + + const context = { log: vi.fn() } as any; + const node = { id: "n1", componentId: "Element_ClickGround" } as any; + + it("dispatches screen selectors to coordinate execution instead of UIA lookup", async () => { + const args = { + selector: "screen=100,200,320,80;ai=1", + clickway: "1" + }; + await expect(getHandler("Element_ClickGround")!(context, node, args, {} as any)).resolves.toEqual({ clicked: true, value: "cached" }); + const script = String((powershellMock.mock.calls as unknown as Array<[string]>)[0]?.[0] ?? ""); + expect(script).toContain("RpaScreenInput"); + expect(script).toContain("SetCursorPos(260, 240)"); + expect(script).not.toContain("Find-RpaWindow"); + }); + + it("dispatches screen setValue to coordinate paste", async () => { + await getHandler("Element_SetValueGround")!(context, node, { + selector: "screen=10,20,30,40", + value: "hello" + }, {} as any); + const script = String((powershellMock.mock.calls as unknown as Array<[string]>)[0]?.[0] ?? ""); + expect(script).toContain("screen_coordinate_paste"); + expect(script).toContain("hello"); + expect(script).not.toContain("Find-RpaWindow"); + }); + + it("treats a picked screen rectangle as present for exists/wait", async () => { + await expect(getHandler("Element_ExistsGround")!(context, node, { + selector: "screen=1,2,3,4" + }, {} as any)).resolves.toBe(true); + await expect(getHandler("Element_WaitElementGround")!(context, node, { + selector: "screen=1,2,3,4" + }, {} as any)).resolves.toBe(true); + expect(powershellMock).not.toHaveBeenCalled(); + }); +}); diff --git a/src/main/capabilities/desktop-uia.ts b/src/main/capabilities/desktop-uia.ts index e02436a..9ed3f97 100644 --- a/src/main/capabilities/desktop-uia.ts +++ b/src/main/capabilities/desktop-uia.ts @@ -8,12 +8,124 @@ interface UiaSelector { window?: string; name?: string; id?: string; + /** .NET ControlType 字段名(已映射,如 ComboBox/ListItem/DataGrid) */ type?: string; + /** 用户输入的原始类型字符串 */ + typeRaw?: string; className?: string; screen?: { x: number; y: number; w: number; h: number }; + /** 从窗口(或桌面根)到目标元素的祖先路径,逐段用稳定属性定位 */ + path?: Array<{ type?: string; name?: string; id?: string; className?: string; index?: number }>; index: number; } +/** ControlType 显式映射:type= 值 → .NET ControlType 字段名;复合名称不得用首字母大写推导 */ +const UIA_CONTROL_TYPE_MAP: Record = { + button: "Button", + calendar: "Calendar", + checkbox: "CheckBox", + combo: "ComboBox", + combobox: "ComboBox", + custom: "Custom", + datagrid: "DataGrid", + dataitem: "DataItem", + document: "Document", + edit: "Edit", + group: "Group", + header: "Header", + headeritem: "HeaderItem", + hyperlink: "Hyperlink", + image: "Image", + list: "List", + listbox: "List", + listitem: "ListItem", + menu: "Menu", + menubar: "MenuBar", + menuitem: "MenuItem", + pane: "Pane", + progressbar: "ProgressBar", + radio: "RadioButton", + radiobutton: "RadioButton", + scrollbar: "ScrollBar", + separator: "Separator", + slider: "Slider", + spinner: "Spinner", + splitbutton: "SplitButton", + statusbar: "StatusBar", + tab: "Tab", + tabitem: "TabItem", + table: "Table", + text: "Text", + thumb: "Thumb", + titlebar: "TitleBar", + toolbar: "ToolBar", + tooltip: "ToolTip", + tree: "Tree", + treeitem: "TreeItem", + window: "Window" +}; + +function normalizeUiaControlType(raw: string): string { + const key = raw.trim().toLowerCase().replace(/[\s_-]+/g, ""); + return UIA_CONTROL_TYPE_MAP[key] ?? raw.charAt(0).toUpperCase() + raw.slice(1).toLowerCase(); +} + +/** 解析 path= 段:Pane>Edit[name=用户名;index=2],段间用 > 分隔,段内 [prop=value;...] */ +function parseUiaSelectorPath(value: string): UiaSelector["path"] { + // 按括号外的 > 分段,避免 path=Edit[name=a>b] 这类属性值被错误拆开 + const rawSegments: string[] = []; + let buffer = ""; + let bracketDepth = 0; + for (const char of value) { + if (char === "[") bracketDepth += 1; + else if (char === "]") bracketDepth = Math.max(0, bracketDepth - 1); + if (char === ">" && bracketDepth === 0) { + rawSegments.push(buffer); + buffer = ""; + } else { + buffer += char; + } + } + rawSegments.push(buffer); + const segments = rawSegments.map(item => item.trim()).filter(Boolean); + if (!segments.length) return undefined; + const path: NonNullable = []; + for (const segment of segments) { + const match = segment.match(/^([A-Za-z][A-Za-z0-9_]*)?\s*(?:\[(.*)\])?$/); + if (!match) { + throw new RuntimeError("validation_error", `UIA 路径段无效:${segment}`, { recommendation: "使用 Type[name=xx;id=yy;class=zz;index=1] > 下一级" }); + } + const entry: NonNullable[number] = {}; + if (match[1]) entry.type = normalizeUiaControlType(match[1]); + const props = match[2]; + if (props) { + for (const pair of props.split(/[;,,]/).map(item => item.trim()).filter(Boolean)) { + const eq = pair.indexOf("="); + if (eq <= 0) { + throw new RuntimeError("validation_error", `UIA 路径属性无效:${pair}`, { recommendation: "使用 name= / id= / class= / index=" }); + } + const key = pair.slice(0, eq).trim().toLowerCase(); + const val = pair.slice(eq + 1).trim(); + if (key === "name" || key === "n") entry.name = val; + else if (key === "id" || key === "automationid") entry.id = val; + else if (key === "class" || key === "classname") entry.className = val; + else if (key === "type" || key === "controltype") entry.type = normalizeUiaControlType(val); + else if (key === "index") { + const n = Number(val); + if (!Number.isFinite(n) || n < 1) throw new RuntimeError("validation_error", `UIA 路径 index 无效:${val}`); + entry.index = Math.floor(n); + } + else throw new RuntimeError("validation_error", `UIA 路径属性不支持:${key}`); + } + } + if (!entry.type && !entry.name && !entry.id && !entry.className) { + throw new RuntimeError("validation_error", `UIA 路径段缺少定位条件:${segment}`, { recommendation: "每段至少提供 type/name/id/class 之一" }); + } + path.push(entry); + } + return path; +} + export function parseUiaSelector(raw: unknown): UiaSelector { let text = ""; if (raw && typeof raw === "object") { @@ -33,7 +145,22 @@ export function parseUiaSelector(raw: unknown): UiaSelector { } if (text.startsWith("uia:")) text = text.slice(4); const selector: UiaSelector = { index: 1 }; - for (const part of text.split(";")) { + // path 段内的 [name=a;b] 不应被顶层 ; 拆开 + const selectorParts: string[] = []; + let partBuffer = ""; + let selectorBracketDepth = 0; + for (const char of text) { + if (char === "[") selectorBracketDepth += 1; + else if (char === "]") selectorBracketDepth = Math.max(0, selectorBracketDepth - 1); + if (char === ";" && selectorBracketDepth === 0) { + selectorParts.push(partBuffer); + partBuffer = ""; + } else { + partBuffer += char; + } + } + selectorParts.push(partBuffer); + for (const part of selectorParts) { const eq = part.indexOf("="); if (eq <= 0) continue; const key = part.slice(0, eq).trim().toLowerCase(); @@ -42,7 +169,10 @@ export function parseUiaSelector(raw: unknown): UiaSelector { if (key === "window" || key === "win") selector.window = value; else if (key === "name" || key === "n") selector.name = value; else if (key === "id" || key === "automationid") selector.id = value; - else if (key === "type" || key === "controltype") selector.type = value.charAt(0).toUpperCase() + value.slice(1).toLowerCase(); + else if (key === "type" || key === "controltype") { + selector.typeRaw = value; + selector.type = normalizeUiaControlType(value); + } else if (key === "class" || key === "classname") selector.className = value; else if (key === "screen") { const nums = value.split(/[,,]/).map(item => Number(item.trim())); @@ -55,8 +185,11 @@ export function parseUiaSelector(raw: unknown): UiaSelector { const n = Number(value); if (Number.isFinite(n) && n >= 1) selector.index = Math.floor(n); } + else if (key === "path" || key === "ancestors" || key === "ancestorpath") { + selector.path = parseUiaSelectorPath(value); + } } - if (!selector.name && !selector.id && !selector.className && !selector.type && !selector.screen) { + if (!selector.name && !selector.id && !selector.className && !selector.type && !selector.screen && !selector.path) { throw new RuntimeError("validation_error", `桌面元素选择器缺少定位条件:${text}`, { recommendation: "至少提供 name= / id= / type= 之一" }); } return selector; @@ -69,6 +202,7 @@ const UIA_USING = [ const FIND_HELPER = ` function Find-RpaWindow([string]$winName) { + # 只读查找:绝不移动/还原/缩放窗口;还原、激活、移动是独立显式动作(Restore/Activate/Move-RpaWindow) $root = [System.Windows.Automation.AutomationElement]::RootElement if (-not $winName) { return $root } $cond = New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::NameProperty, $winName) @@ -78,41 +212,88 @@ function Find-RpaWindow([string]$winName) { $script:hitHwnd = [IntPtr]::Zero $cb = [WinRpa.RpaUser32Click+EnumProc]{ param($h, $l) $sb = New-Object System.Text.StringBuilder 256 - [WinRpa.RpaUser32Click]::GetWindowTextW($h, $sb, 256) | Out-Null + [void][WinRpa.RpaUser32Click]::GetWindowTextW($h, $sb, 256) if ($sb.ToString() -eq $winName) { $script:hitHwnd = $h; return $false } return $true } - [WinRpa.RpaUser32Click]::EnumWindows($cb, [IntPtr]::Zero) | Out-Null + [void][WinRpa.RpaUser32Click]::EnumWindows($cb, [IntPtr]::Zero) if ($script:hitHwnd -ne [IntPtr]::Zero) { - [WinRpa.RpaUser32Click]::ShowWindow($script:hitHwnd, 9) | Out-Null - Start-Sleep -Milliseconds 500 - $sw2 = [WinRpa.RpaUser32Click]::GetSystemMetrics(0) - $sh2 = [WinRpa.RpaUser32Click]::GetSystemMetrics(1) - $w2 = [Math]::Min(1400, $sw2 - 120) - $h2 = [Math]::Min(820, $sh2 - 120) - [WinRpa.RpaUser32Click]::SetWindowPos($script:hitHwnd, [IntPtr]::Zero, 60, 60, $w2, $h2, 0x0040) | Out-Null - Start-Sleep -Milliseconds 600 - $win = $root.FindFirst([System.Windows.Automation.TreeScope]::Children, $cond) - if ($win) { return $win } + try { $win = [System.Windows.Automation.AutomationElement]::FromHandle($script:hitHwnd); if ($win) { return $win } } catch {} } throw "window_not_found:$winName" } -function Find-RpaElement($parent, [string]$name, [string]$id, [string]$type, [string]$className, [int]$index) { +function Restore-RpaWindow($win) { + # 显式动作:还原窗口(最小化/最大化状态恢复),不用于查询 + Add-RpaUser32 + $hwnd = [IntPtr]$win.Current.NativeWindowHandle + if ($hwnd -eq [IntPtr]::Zero) { throw "window_no_hwnd" } + [void][WinRpa.RpaUser32Click]::ShowWindow($hwnd, 9) + Start-Sleep -Milliseconds 300 + return $hwnd +} +function Activate-RpaWindow($win) { + # 显式动作:把窗口带到前台 + Add-RpaUser32 + $hwnd = [IntPtr]$win.Current.NativeWindowHandle + if ($hwnd -eq [IntPtr]::Zero) { throw "window_no_hwnd" } + $fgHwnd = [WinRpa.RpaUser32Click]::GetForegroundWindow() + if ($fgHwnd -ne $hwnd) { + $fgPid = [uint32]0 + $fgThread = [WinRpa.RpaUser32Click]::GetWindowThreadProcessId($fgHwnd, [ref]$fgPid) + $thisThread = [WinRpa.RpaUser32Click]::GetCurrentThreadId() + [void][WinRpa.RpaUser32Click]::AttachThreadInput($thisThread, $fgThread, $true) + [void][WinRpa.RpaUser32Click]::SetForegroundWindow($hwnd) + [void][WinRpa.RpaUser32Click]::AttachThreadInput($thisThread, $fgThread, $false) + } + Start-Sleep -Milliseconds 200 + return $hwnd +} +function Move-RpaWindow($win, [int]$x, [int]$y, [int]$w, [int]$h) { + # 显式动作:移动/缩放窗口;w/h<=0 时使用安全默认尺寸 + Add-RpaUser32 + $hwnd = [IntPtr]$win.Current.NativeWindowHandle + if ($hwnd -eq [IntPtr]::Zero) { throw "window_no_hwnd" } + $sw = [WinRpa.RpaUser32Click]::GetSystemMetrics(0) + $sh = [WinRpa.RpaUser32Click]::GetSystemMetrics(1) + if ($w -le 0) { $w = [Math]::Min(1400, $sw - 120) } + if ($h -le 0) { $h = [Math]::Min(820, $sh - 120) } + [void][WinRpa.RpaUser32Click]::SetWindowPos($hwnd, [IntPtr]::Zero, $x, $y, $w, $h, 0x0040) + Start-Sleep -Milliseconds 300 + return $hwnd +} +function Find-RpaElement($parent, [string]$name, [string]$id, [string]$type, [string]$className, [int]$index, [switch]$directChild) { $conds = @() if ($name) { $conds += New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::NameProperty, $name) } if ($id) { $conds += New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::AutomationIdProperty, $id) } if ($className) { $conds += New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::ClassNameProperty, $className) } if ($type) { - try { $field = [System.Windows.Automation.ControlType].GetField($type); if ($field) { $conds += New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::ControlTypeProperty, $field.GetValue($null)) } } catch {} - if ($ct) { $conds += New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::ControlTypeProperty, $ct) } + # 不区分大小写匹配 .NET ControlType 字段,兼容 ComboBox/ListItem/DataGrid 等复合名称 + try { + $ctField = [System.Windows.Automation.ControlType].GetFields([System.Reflection.BindingFlags]"Public,Static") | Where-Object { $_.Name -ieq $type } | Select-Object -First 1 + if ($ctField) { $conds += New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::ControlTypeProperty, $ctField.GetValue($null)) } + } catch {} } if ($conds.Count -eq 0) { throw "no_condition" } $final = if ($conds.Count -eq 1) { $conds[0] } else { New-Object System.Windows.Automation.AndCondition($conds) } - $all = $parent.FindAll([System.Windows.Automation.TreeScope]::Descendants, $final) + $scope = if ($directChild) { [System.Windows.Automation.TreeScope]::Children } else { [System.Windows.Automation.TreeScope]::Descendants } + $all = $parent.FindAll($scope, $final) if ($all.Count -eq 0) { return $null } $pick = [Math]::Min([Math]::Max($index, 1), $all.Count) return $all[$pick - 1] } +function Find-RpaElementByPath($parent, [string]$pathJsonBase64) { + # 逐段走直接子级,用稳定属性(id/type/class 优先)解析窗口→目标祖先路径 + $segments = [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String($pathJsonBase64)) | ConvertFrom-Json + $cur = $parent + foreach ($seg in @($segments)) { + $ix = 1 + try { if ($seg.index) { $ix = [int]$seg.index } } catch {} + $next = Find-RpaElement $cur $seg.name $seg.id $seg.type $seg.className $ix -directChild + if ($null -eq $next) { return $null } + $cur = $next + } + return $cur +} function Add-RpaUser32 { Add-Type -MemberDefinition '[DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y); [DllImport("user32.dll")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra); [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd); [DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); [DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr hWnd, int nCmdShow); [DllImport("user32.dll")] public static extern bool SetWindowPos(IntPtr hWnd, IntPtr after, int X, int Y, int cx, int cy, uint flags); [DllImport("user32.dll")] public static extern bool AttachThreadInput(uint idAttach, uint idAttachTo, bool fAttach); [DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint processId); [DllImport("kernel32.dll")] public static extern uint GetCurrentThreadId(); [DllImport("user32.dll")] public static extern int GetSystemMetrics(int nIndex); [DllImport("user32.dll")] public static extern bool EnumWindows(EnumProc cb, IntPtr lp); public delegate bool EnumProc(IntPtr hWnd, IntPtr lp); [DllImport("user32.dll", CharSet = CharSet.Unicode)] public static extern int GetWindowTextW(IntPtr hWnd, System.Text.StringBuilder sb, int max);' -Name RpaUser32Click -Namespace Win32Rpa | Out-Null } @@ -129,6 +310,21 @@ function target(args: Record, node: { id: string; componentId: return { selector: parseUiaSelector(raw), args, node }; } +/** 生成“解析窗口 + 解析元素(路径优先,其次平铺属性)”的 PowerShell 语句,结果写入 $el */ +function elementResolveLines(s: UiaSelector): string { + const parentLine = "$parent = Find-RpaWindow " + psString(s.window ?? ""); + if (s.path?.length) { + const pathB64 = Buffer.from(JSON.stringify(s.path), "utf8").toString("base64"); + return parentLine + "\n$el = Find-RpaElementByPath $parent '" + pathB64 + "'"; + } + return parentLine + + "\n$el = Find-RpaElement $parent " + psString(s.name ?? "") + + " " + psString(s.id ?? "") + + " " + psString(s.type ?? "") + + " " + psString(s.className ?? "") + + " " + s.index; +} + const SCREEN_USER32 = "Add-Type -MemberDefinition '[DllImport(\"user32.dll\")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value); [DllImport(\"user32.dll\")] public static extern bool SetCursorPos(int X, int Y); [DllImport(\"user32.dll\")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra);' -Name RpaScreenInput -Namespace RpaScreen | Out-Null; try { [RpaScreen.RpaScreenInput]::SetProcessDpiAwarenessContext([IntPtr](-4)) | Out-Null } catch {}"; function screenCenter(selector: UiaSelector): { x: number; y: number; w: number; h: number } { @@ -204,8 +400,8 @@ $deadline = [DateTime]::UtcNow.AddMilliseconds(${Math.max(0, Math.round(timeoutM $found = $null while ($null -eq $found) { try { - $parent = Find-RpaWindow ${psString(s.window ?? "")} - $found = Find-RpaElement $parent ${psString(s.name ?? "")} ${psString(s.id ?? "")} ${psString(s.type ?? "")} ${psString(s.className ?? "")} ${s.index} + ${elementResolveLines(s)} + $found = $el } catch { $err = $_.Exception.Message if ($err -like "window_not_found:*") { $found = $null } @@ -239,8 +435,7 @@ function clickScript(t: UiaTarget): string { ${FIND_HELPER} $el = $null try { - $parent = Find-RpaWindow ${psString(s.window ?? "")} - $el = Find-RpaElement $parent ${psString(s.name ?? "")} ${psString(s.id ?? "")} ${psString(s.type ?? "")} ${psString(s.className ?? "")} ${s.index} + ${elementResolveLines(s)} } catch { $el = $null } if ($null -eq $el) { Write-Output "JSON_NOT_FOUND"; exit 0 } $rect = $el.Current.BoundingRectangle @@ -335,10 +530,7 @@ export function setValueScript(t: UiaTarget, value: string): string { const s = t.selector; return `${UIA_USING} ${FIND_HELPER} -try { - $parent = Find-RpaWindow ${psString(s.window ?? "")} - $el = Find-RpaElement $parent ${psString(s.name ?? "")} ${psString(s.id ?? "")} ${psString(s.type ?? "")} ${psString(s.className ?? "")} ${s.index} -} catch { $el = $null } +${elementResolveLines(s)} if ($null -eq $el) { Write-Output "JSON_NOT_FOUND"; exit 0 } $value = ${psString(value)} $method = "value_pattern" @@ -372,10 +564,7 @@ function getValueScript(t: UiaTarget): string { const s = t.selector; return `${UIA_USING} ${FIND_HELPER} -try { - $parent = Find-RpaWindow ${psString(s.window ?? "")} - $el = Find-RpaElement $parent ${psString(s.name ?? "")} ${psString(s.id ?? "")} ${psString(s.type ?? "")} ${psString(s.className ?? "")} ${s.index} -} catch { $el = $null } +${elementResolveLines(s)} if ($null -eq $el) { Write-Output "JSON_NOT_FOUND"; exit 0 } $value = $el.Current.Name try { $vp = $el.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern); $value = $vp.Value } catch {} @@ -426,29 +615,37 @@ function requireFound(result: Record | null, t: UiaTarget, acti const clickGround: CapabilityHandler = async (_ctx, node, args) => { const t = target(args, node); + if (t.selector.screen) return runScript(clickScreenScript(t), t); return runWithImplicitWait(t, "click", clickScript(t), runScript, args); }; const setValueGround: CapabilityHandler = async (_ctx, node, args) => { const t = target(args, node); const value = String(args.value ?? args.text ?? ""); + if (t.selector.screen) return runScript(setValueScreenScript(t, value), t); return runWithImplicitWait(t, "setValue", setValueScript(t, value), runScript, args); }; const existsGround: CapabilityHandler = async (_ctx, node, args) => { const t = target(args, node); + if (t.selector.screen) return true; const found = await findElement(t, 1000, 150); return found !== null; }; const getValueGround: CapabilityHandler = async (_ctx, node, args) => { const t = target(args, node); + if (t.selector.screen) { + const result = await runScript(getValueScreenScript(t), t); + return requireFound(result, t, "getValue").value ?? ""; + } const result = await runWithImplicitWait(t, "getValue", getValueScript(t), runScript, args); return result.value ?? ""; }; const waitElementGround: CapabilityHandler = async (_ctx, node, args) => { const t = target(args, node); + if (t.selector.screen) return true; const timeout = Number(args.timeout ?? args.timeoutMs ?? 5000) || 5000; const gap = Number(args.time_gap ?? args.gapMs ?? 200) || 200; const found = await findElement(t, timeout, gap); @@ -459,6 +656,39 @@ register("Element_ClickGround", clickGround); register("Element_SetValueGround", setValueGround); register("Element_ExistsGround", existsGround); register("Element_GetValueGround", getValueGround); +const restoreWindowGround: CapabilityHandler = async (_ctx, node, args) => { + const t = target(args, node); + const lines = ["$hwnd = Restore-RpaWindow $el"]; + if ([args.x, args.y, args.width, args.height].some((value) => value !== undefined && value !== null && String(value).trim() !== "")) { + const x = Math.round(Number(args.x ?? 60)) || 0; + const y = Math.round(Number(args.y ?? 60)) || 0; + const width = Math.round(Number(args.width ?? 0)) || 0; + const height = Math.round(Number(args.height ?? 0)) || 0; + lines.push("$hwnd = Move-RpaWindow $el " + x + " " + y + " " + width + " " + height); + } + return runWithImplicitWait(t, "restoreWindow", propertyValue([...lines, "Write-Output (@{ hwnd = [string]$hwnd; name = $el.Current.Name } | ConvertTo-Json -Compress)"], t), runScript, args); +}; + +const activateWindowGround: CapabilityHandler = async (_ctx, node, args) => { + const t = target(args, node); + return runWithImplicitWait(t, "activateWindow", propertyValue([ + "$hwnd = Activate-RpaWindow $el", + "Write-Output (@{ hwnd = [string]$hwnd; name = $el.Current.Name } | ConvertTo-Json -Compress)" + ], t), runScript, args); +}; + +const moveWindowGround: CapabilityHandler = async (_ctx, node, args) => { + const t = target(args, node); + const x = Math.round(Number(args.x ?? 60)) || 0; + const y = Math.round(Number(args.y ?? 60)) || 0; + const width = Math.round(Number(args.width ?? 0)) || 0; + const height = Math.round(Number(args.height ?? 0)) || 0; + return runWithImplicitWait(t, "moveWindow", propertyValue([ + "$hwnd = Move-RpaWindow $el " + x + " " + y + " " + width + " " + height, + "Write-Output (@{ hwnd = [string]$hwnd; name = $el.Current.Name } | ConvertTo-Json -Compress)" + ], t), runScript, args); +}; + register("Element_WaitElementGround", waitElementGround); @@ -486,8 +716,7 @@ function propertyValue(scriptLines: string[], t: UiaTarget): string { ${FIND_HELPER} $el = $null try { - $parent = Find-RpaWindow ${psString(s.window ?? "")} - $el = Find-RpaElement $parent ${psString(s.name ?? "")} ${psString(s.id ?? "")} ${psString(s.type ?? "")} ${psString(s.className ?? "")} ${s.index} + ${elementResolveLines(s)} } catch { $el = $null } if ($null -eq $el) { Write-Output "JSON_NOT_FOUND"; exit 0 } ${scriptLines.join("\n")}`; @@ -709,3 +938,6 @@ register("Element_MouseToGround", mouseToGround); register("Element_ScrollIntoViewGround", scrollIntoViewGround); register("Element_ChildrenGround", childrenGroundHandler(false)); register("Element_DesktopChildren", childrenGroundHandler(true)); +register("Element_RestoreWindowGround", restoreWindowGround); +register("Element_ActivateWindowGround", activateWindowGround); +register("Element_MoveWindowGround", moveWindowGround); diff --git a/src/main/capabilities/index.ts b/src/main/capabilities/index.ts index 14ca743..ff63b65 100644 --- a/src/main/capabilities/index.ts +++ b/src/main/capabilities/index.ts @@ -24,6 +24,7 @@ import "./word"; import "./word-extended"; import "./message"; import "./vision"; +import "./cv-vision"; import "./extended"; import "./system-extended"; import "./office-file-extended"; diff --git a/src/main/desktop-pick.ts b/src/main/desktop-pick.ts index a5d9b37..4a5c516 100644 --- a/src/main/desktop-pick.ts +++ b/src/main/desktop-pick.ts @@ -3,9 +3,9 @@ import { promisify } from "node:util"; import fs from "node:fs"; import path from "node:path"; -import { app, BrowserWindow, screen } from "electron"; +import { app, BrowserWindow, desktopCapturer, globalShortcut, screen } from "electron"; import { RuntimeError } from "./engine/types"; -import { chatCompletion, requireAiConfig } from "./capabilities/ai-service"; +import { detectUiElements, localUiModelPath } from "./local-vision/ui-detect-service"; const run = promisify(execFile); @@ -58,6 +58,8 @@ interface UiaPickOutput { } interface AiCandidate { + /** 候选稳定身份:覆盖层排序后仍按 ID 回填,禁止用排序后的数组下标当身份 */ + candidateId: string; x: number; y: number; width: number; @@ -92,12 +94,10 @@ function safeSelectorText(value: string | undefined): string { return String(value || "").replace(/[;\r\n]+/g, " ").trim(); } -export async function pickDesktopElement(options: { ai?: boolean } = {}): Promise { - if (options.ai) requireAiConfig({ vision: true }); +export async function pickDesktopElement(): Promise { const appWindows = BrowserWindow.getAllWindows().filter(w => !w.isDestroyed()); for (const w of appWindows) w.minimize(); try { - if (options.ai) return await runAiPick(); const { stdout } = await run( "powershell.exe", ["-NoProfile", "-STA", "-ExecutionPolicy", "Bypass", "-File", pickerScriptPath()], @@ -109,7 +109,9 @@ export async function pickDesktopElement(options: { ai?: boolean } = {}): Promis .map(value => Number(value)) .every(value => Number.isFinite(value)) && Number(parsed.paneW) > 4 && Number(parsed.paneH) > 4; if (parsed.aiRequested && paneAiReady) return await runPaneAiPick(parsed); - if (parsed.aiRequested) return await runAiPick(parsed); + if (parsed.aiRequested) { + throw new RuntimeError("ai_pane_invalid", "AI 兜底识别仅支持包含有效区域的 Pane"); + } const parts: string[] = []; if (parsed.window) parts.push("window=" + parsed.window); if (parsed.name) parts.push("name=" + parsed.name); @@ -125,68 +127,121 @@ export async function pickDesktopElement(options: { ai?: boolean } = {}): Promis function paneAiOverlayHtml(): string { return ` -
AI 识别中…
+
AI 识别中…
`; } @@ -236,160 +291,168 @@ async function createPaneAiOverlay(rect: { x: number; y: number; w: number; h: n return win; } -async function capturePaneForAi(rect: { x: number; y: number; w: number; h: number }): Promise { - const outDir = path.join(app.getPath("userData"), "ai-picks"); - fs.mkdirSync(outDir, { recursive: true }); - const imagePath = path.join(outDir, "ai-pane-" + Date.now() + ".png"); - const { stdout } = await run( - "powershell.exe", - [ - "-NoProfile", "-STA", "-ExecutionPolicy", "Bypass", "-File", pickerScriptPath(), - "-Mode", "capture", "-CaptureExact", - "-CaptureX", String(rect.x), "-CaptureY", String(rect.y), - "-CaptureWidth", String(rect.w), "-CaptureHeight", String(rect.h), - "-CapturePath", imagePath - ], - { timeout: 30000, windowsHide: true, maxBuffer: 2 * 1024 * 1024 } - ); - const parsed = jsonLine(stdout, "{"); - if (!parsed.imagePath || !fs.existsSync(parsed.imagePath)) { - throw new RuntimeError("ai_capture_failed", "AI 拾取截图失败"); - } - return parsed; -} +async function capturePaneForAi(rect: { x: number; y: number; w: number; h: number }, overlay: BrowserWindow): Promise { + const centerDip = screen.screenToDipPoint({ x: rect.x + rect.w / 2, y: rect.y + rect.h / 2 }); + const display = displayForPoint(centerDip.x, centerDip.y); + const scale = display.scaleFactor || 1; + const displayPhysicalX = Math.round(display.bounds.x * scale); + const displayPhysicalY = Math.round(display.bounds.y * scale); + // UIA/Desktop Picker 返回的是物理像素坐标;截图裁剪也必须直接使用物理像素。 + // 这里不能把 cropX/cropY 再乘 actualScale,否则会重复缩放并导致候选框整体偏移。 + const cropX = rect.x - displayPhysicalX; + const cropY = rect.y - displayPhysicalY; + const thumbnailWidth = Math.round(display.bounds.width * scale); + const thumbnailHeight = Math.round(display.bounds.height * scale); -function parseAiCandidates(text: string, cropW: number, cropH: number): AiCandidate[] { - const fence = String.fromCharCode(96); - const fencePattern = new RegExp(fence + "{3}(?:json)?" + fence + "{3}", "gi"); - const cleaned = text.replace(fencePattern, "").trim(); - const start = cleaned.indexOf("{"); - const arrayStart = cleaned.indexOf("["); - const objectStart = start >= 0 && (start < arrayStart || arrayStart < 0) ? start : arrayStart; - const end = Math.max(cleaned.lastIndexOf("}"), cleaned.lastIndexOf("]")); - if (objectStart < 0 || end <= objectStart) throw new RuntimeError("ai_response_invalid", "AI 视觉模型没有返回 JSON"); - const raw = JSON.parse(cleaned.slice(objectStart, end + 1)) as Record; - const list = Array.isArray(raw) ? raw : Array.isArray(raw.candidates) ? raw.candidates : []; - const result: AiCandidate[] = []; - for (const item of list.slice(0, 40)) { - const record = (item || {}) as Record; - const num = (...keys: string[]): number => { - for (const key of keys) { - const value = Number(record[key]); - if (Number.isFinite(value)) return value; - } - return NaN; - }; - let x = num("x", "left", "x1"); - let y = num("y", "top", "y1"); - let width = num("width", "w"); - let height = num("height", "h"); - const x2 = num("x2", "right"); - const y2 = num("y2", "bottom"); - if (!Number.isFinite(width) && Number.isFinite(x2)) width = x2 - x; - if (!Number.isFinite(height) && Number.isFinite(y2)) height = y2 - y; - if (![x, y, width, height].every(Number.isFinite) || width < 6 || height < 6) continue; - x = Math.max(0, Math.min(cropW - width, x)); - y = Math.max(0, Math.min(cropH - height, y)); - width = Math.min(width, cropW - x); - height = Math.min(height, cropH - y); - if (width * height >= cropW * cropH * 0.97) continue; - const confidence = Number(record.confidence); - result.push({ - x: Math.round(x), y: Math.round(y), width: Math.round(width), height: Math.round(height), - name: String(record.name ?? "AI识别控件").slice(0, 80) || "AI识别控件", - controlType: String(record.controlType ?? "Control").slice(0, 40) || "Control", - confidence: Number.isFinite(confidence) ? Math.max(0, Math.min(1, confidence)) : 0.72 + // 只为避开拾取窗口本身;推理和动画继续在后台进行。 + overlay.hide(); + await new Promise(resolve => setTimeout(resolve, 60)); + try { + const sources = await desktopCapturer.getSources({ + types: ["screen"], + thumbnailSize: { width: thumbnailWidth, height: thumbnailHeight } }); + overlay.show(); + const source = sources.find(item => item.display_id === String(display.id)) || sources[0]; + if (!source || source.thumbnail.isEmpty()) { + throw new RuntimeError("ai_capture_failed", "AI 拾取截图失败"); + } + + const thumbnail = source.thumbnail; + const actualScale = thumbnail.getSize().width / Math.max(1, display.bounds.width); + const x = Math.max(0, Math.min(thumbnail.getSize().width - 1, cropX)); + const y = Math.max(0, Math.min(thumbnail.getSize().height - 1, cropY)); + const width = Math.max(1, Math.min(thumbnail.getSize().width - x, Math.round(rect.w * actualScale))); + const height = Math.max(1, Math.min(thumbnail.getSize().height - y, Math.round(rect.h * actualScale))); + const cropped = thumbnail.crop({ x, y, width, height }); + const image = cropped.toPNG(); + const outDir = path.join(app.getPath("userData"), "ai-picks"); + fs.mkdirSync(outDir, { recursive: true }); + const imagePath = path.join(outDir, `ai-pane-${Date.now()}.png`); + await fs.promises.writeFile(imagePath, image); + return { + cursorX: rect.x, + cursorY: rect.y, + cropX: rect.x, + cropY: rect.y, + cropW: width, + cropH: height, + markerX: 0, + markerY: 0, + imagePath + }; + } catch (error) { + overlay.show(); + throw error; } - if (!result.length) throw new RuntimeError("ai_response_invalid", "AI 没有识别到可用候选区域"); - return result; } async function runPaneAiPick(context: UiaPickOutput): Promise { - requireAiConfig({ vision: true }); + // 本地小模型是 Pane AI 的主链路;这里不要求用户配置远程 VLM。 + const localModelPath = localUiModelPath(); + console.log("[pane-ai] local model", localModelPath); const pane = paneAiRect(context); const win = await createPaneAiOverlay(pane); + const startedAt = Date.now(); + // F8 用全局快捷键,避免键盘焦点停在目标应用时 overlay 收不到按键。 + const debugHotkeyRegistered = globalShortcut.register("F8", () => { + void win.webContents.executeJavaScript("window.__toggleDebug && window.__toggleDebug()", true).catch(() => {}); + }); try { - // Let the visible scan animation complete over the actual Pane before taking the screenshot. - await new Promise(resolve => setTimeout(resolve, 950)); - win.hide(); - await new Promise(resolve => setTimeout(resolve, 240)); - const capture = await capturePaneForAi(pane); - win.show(); - win.focus(); - await win.webContents.executeJavaScript("startScan('AI 理解中…');"); + // 稍等 overlay 完成首轮绘制,但不再等一整套“表演动画”。 + await new Promise(resolve => setTimeout(resolve, 350)); + const detection = capturePaneForAi(pane, win) + .then(async capture => { + const image = await fs.promises.readFile(capture.imagePath); + const inferenceStarted = Date.now(); + const boxes = await detectUiElements({ image, width: capture.cropW, height: capture.cropH }); + console.log("[pane-ai] local inference", boxes.length, Date.now() - inferenceStarted + "ms"); + return { capture, boxes }; + }); - const image = await fs.promises.readFile(capture.imagePath); - let aiSettled = false; - const aiRequest = chatCompletion([ - { - role: "system", - content: "你是桌面 RPA 视觉控件分析器。只输出 JSON,不输出解释或 Markdown。" - }, - { - role: "user", - content: [ - { - type: "text", - text: [ - "这是目标 Pane 区域的截图,尺寸 " + capture.cropW + "x" + capture.cropH + " 像素。", - "请把这个 Pane 里的可用控件/内容块拆分成多个候选矩形,最多返回 30 个,按从上到下、从左到右排序。", - "优先识别按钮、输入框、列表行、卡片、菜单项、链接和明显可操作区域;不要把整个 Pane 当作候选。", - "只返回 JSON:{\"candidates\":[{\"x\":number,\"y\":number,\"width\":number,\"height\":number,\"name\":string,\"controlType\":string,\"confidence\":number}]}", - "坐标全部是相对这张截图的整数像素坐标;x/y 是左上角,width/height 也是像素。" - ].join("\n") - }, - { type: "image_url", image_url: { url: "data:image/png;base64," + image.toString("base64") } } - ] + let settled = false; + const tickStatus = async () => { + const elapsed = Date.now() - startedAt; + const label = elapsed < 2600 + ? "AI 识别中…" + : elapsed < 6000 + ? "AI 理解中…" + : "正在整理候选区域…"; + try { + await win.webContents.executeJavaScript("showStatus(" + JSON.stringify(label) + ");", true); + } catch { + // Overlay 销毁时状态更新可以安全忽略。 } - ], { vision: true, temperature: 0 }).finally(() => { aiSettled = true; }); + }; + const statusTimer = setInterval(() => void tickStatus(), 1500); const cancelWatcher = (async () => { - while (!aiSettled) { + while (!settled) { try { const state = await win.webContents.executeJavaScript("JSON.stringify({cancelled:window.__cancelled===true})", true); if (JSON.parse(state).cancelled) throw new RuntimeError("pick_cancelled", "已取消 AI 拾取"); } catch (error) { if (error instanceof RuntimeError && error.code === "pick_cancelled") throw error; } - await new Promise(resolve => setTimeout(resolve, 100)); + await new Promise(resolve => setTimeout(resolve, 80)); } })(); - let aiText: string; + let detected: { capture: AiCaptureOutput; boxes: Awaited> }; try { - aiText = await Promise.race([aiRequest, cancelWatcher]) as string; + detected = await Promise.race([detection, cancelWatcher]) as { capture: AiCaptureOutput; boxes: Awaited> }; } finally { - aiSettled = true; + settled = true; + clearInterval(statusTimer); + } + + const candidates: AiCandidate[] = detected.boxes.map((box, index) => ({ + candidateId: "local-" + index, + x: box.x, + y: box.y, + width: box.width, + height: box.height, + name: box.category, + controlType: box.category, + confidence: box.confidence + })); + if (!candidates.length) { + throw new RuntimeError("ai_response_invalid", "本地 UI 模型没有识别到可用候选区域"); } - const candidates = parseAiCandidates(aiText, capture.cropW, capture.cropH); const payload = JSON.stringify(candidates); - console.log("[pane-ai] candidates", candidates.length, candidates.map(item => ({ name: item.name, x: item.x, y: item.y, width: item.width, height: item.height }))); - await win.webContents.executeJavaScript("window.__done=false;window.__cancelled=false;window.__selected=null;window.__hover=null;window.__hoverCandidate=null;window.__confirmed=null;window.__readyAt=Date.now()+600;showCandidates(" + payload + ");"); + await win.show(); + win.focus(); + await win.webContents.executeJavaScript( + "window.__done=false;window.__cancelled=false;window.__selectedCid=null;window.__hoverCid=null;window.__hoverCandidate=null;window.__confirmedCid=null;window.__readyAt=Date.now()+500;showCandidates(" + payload + ");", + true + ); const deadline = Date.now() + 180000; - let selectedIndex: number | null = null; + let selectedId: string | null = null; while (Date.now() < deadline) { await new Promise(resolve => setTimeout(resolve, 80)); const state = await win.webContents.executeJavaScript( - "JSON.stringify({done:window.__done===true,cancelled:window.__cancelled===true,selected:window.__selected})", true + "JSON.stringify({done:window.__done===true,cancelled:window.__cancelled===true,selectedCid:window.__selectedCid||null})", + true ); - const parsed = JSON.parse(state) as { done: boolean; cancelled: boolean; selected: number | null }; + const parsed = JSON.parse(state) as { done: boolean; cancelled: boolean; selectedCid: string | null }; if (parsed.cancelled) throw new RuntimeError("pick_cancelled", "已取消 AI 拾取"); - if (parsed.done && Number.isInteger(parsed.selected)) { - const confirmedIndex = Number(parsed.selected); - console.log("[pane-ai] confirmed", confirmedIndex, candidates[confirmedIndex]); - selectedIndex = confirmedIndex; - break; + if (parsed.done && parsed.selectedCid) { + const box = candidates.find(item => item.candidateId === parsed.selectedCid); + console.log("[pane-ai] confirmed", parsed.selectedCid, box); + if (box) { + selectedId = box.candidateId; + break; + } } } - if (selectedIndex === null || !candidates[selectedIndex]) throw new RuntimeError("pick_cancelled", "AI 拾取超时"); + if (!selectedId) throw new RuntimeError("pick_cancelled", "AI 拾取超时"); - const box = candidates[selectedIndex]; - const x = capture.cropX + box.x; - const y = capture.cropY + box.y; + const box = candidates.find(item => item.candidateId === selectedId)!; + const x = detected.capture.cropX + box.x; + const y = detected.capture.cropY + box.y; const selectorParts = [ "screen=" + [x, y, box.width, box.height].map(Math.round).join(","), context.window ? "window=" + safeSelectorText(context.window) : "", @@ -404,37 +467,15 @@ async function runPaneAiPick(context: UiaPickOutput): Promise controlType: box.controlType, x, y, w: box.width, h: box.height, confidence: box.confidence, - screenshotPath: capture.imagePath, + screenshotPath: detected.capture.imagePath, selector: selectorParts.join(";") }; } finally { + if (debugHotkeyRegistered) globalShortcut.unregister("F8"); if (!win.isDestroyed()) win.destroy(); } } -function aiOverlayHtml(): string { - return `
AI 拾取:单击要识别的控件;Esc 取消
`; -} - function displayForPoint(x: number, y: number) { return screen.getAllDisplays().find(display => { const b = display.bounds; @@ -442,216 +483,6 @@ function displayForPoint(x: number, y: number) { }) || screen.getPrimaryDisplay(); } -async function createAiOverlay(): Promise<{ win: BrowserWindow; display: ReturnType }> { - const cursor = screen.getCursorScreenPoint(); - const display = displayForPoint(cursor.x, cursor.y); - const win = new BrowserWindow({ - x: display.bounds.x, - y: display.bounds.y, - width: display.bounds.width, - height: display.bounds.height, - frame: false, - transparent: true, - resizable: false, - movable: false, - minimizable: false, - maximizable: false, - alwaysOnTop: true, - skipTaskbar: true, - show: false, - webPreferences: { contextIsolation: true, nodeIntegration: false, sandbox: true } - }); - win.setAlwaysOnTop(true, "screen-saver"); - await win.loadURL("data:text/html;charset=utf-8," + encodeURIComponent(aiOverlayHtml())); - win.show(); - win.focus(); - return { win, display }; -} - -async function captureAiAtPoint(physicalX: number, physicalY: number): Promise { - const outDir = path.join(app.getPath("userData"), "ai-picks"); - fs.mkdirSync(outDir, { recursive: true }); - const imagePath = path.join(outDir, "ai-pick-" + Date.now() + ".png"); - const { stdout } = await run( - "powershell.exe", - [ - "-NoProfile", "-STA", "-ExecutionPolicy", "Bypass", "-File", pickerScriptPath(), - "-Mode", "capture", "-CaptureX", String(physicalX), "-CaptureY", String(physicalY), - "-CaptureWidth", "1000", "-CaptureHeight", "700", "-CapturePath", imagePath - ], - { timeout: 30000, windowsHide: true, maxBuffer: 2 * 1024 * 1024 } - ); - const parsed = jsonLine(stdout, "{"); - if (!parsed.imagePath || !fs.existsSync(parsed.imagePath)) { - throw new RuntimeError("ai_capture_failed", "AI 拾取截图失败"); - } - return parsed; -} - -function parseAiBox(text: string, cropW: number, cropH: number) { - const fence = String.fromCharCode(96); - const fencePattern = new RegExp(fence + "{3}(?:json)?" + fence + "{3}", "gi"); - const cleaned = text.replace(fencePattern, "").trim(); - const start = cleaned.indexOf("{"); - const end = cleaned.lastIndexOf("}"); - if (start < 0 || end <= start) throw new RuntimeError("ai_response_invalid", "AI 视觉模型没有返回 JSON"); - const raw = JSON.parse(cleaned.slice(start, end + 1)) as Record; - const num = (...keys: string[]): number => { - for (const key of keys) { - const value = Number(raw[key]); - if (Number.isFinite(value)) return value; - } - return NaN; - }; - const x1 = num("x", "left", "x1"); - const y1 = num("y", "top", "y1"); - let width = num("width", "w"); - let height = num("height", "h"); - const x2 = num("x2", "right"); - const y2 = num("y2", "bottom"); - if (!Number.isFinite(width) && Number.isFinite(x2)) width = x2 - x1; - if (!Number.isFinite(height) && Number.isFinite(y2)) height = y2 - y1; - if (![x1, y1].every(Number.isFinite)) throw new RuntimeError("ai_response_invalid", "AI 视觉模型缺少目标坐标"); - if (!Number.isFinite(width) || width <= 0) width = 24; - if (!Number.isFinite(height) || height <= 0) height = 18; - width = Math.max(4, Math.min(cropW, width)); - height = Math.max(4, Math.min(cropH, height)); - const localX = Math.max(0, Math.min(cropW - width, x1)); - const localY = Math.max(0, Math.min(cropH - height, y1)); - const confidence = Number(raw.confidence); - return { - x: localX, - y: localY, - width, - height, - name: String(raw.name ?? "AI识别控件"), - controlType: String(raw.controlType ?? "Control"), - confidence: Number.isFinite(confidence) ? Math.max(0, Math.min(1, confidence)) : 0.7 - }; -} - -async function runAiPick(context: UiaPickOutput = {}): Promise { - requireAiConfig({ vision: true }); - const { win, display } = await createAiOverlay(); - try { - let point: { x: number; y: number } | null = null; - let physicalX = 0; - let physicalY = 0; - const autoAi = context.aiRequested && Number.isFinite(Number(context.cursorX)) && Number.isFinite(Number(context.cursorY)); - if (autoAi) { - const cursor = screen.getCursorScreenPoint(); - point = { x: cursor.x - display.bounds.x, y: cursor.y - display.bounds.y }; - physicalX = Math.round(Number(context.cursorX)); - physicalY = Math.round(Number(context.cursorY)); - await win.webContents.executeJavaScript( - "var p=document.getElementById('point');p.style.left=(window.__autoX=" + point.x + ")-23+'px';p.style.top=(window.__autoY=" + point.y + ")-23+'px';p.style.display='block';document.getElementById('tip').textContent='AI 视觉识别中…自上而下扫描画面';document.body.classList.add('scanning');window.__done=true;" - ); - await new Promise(resolve => setTimeout(resolve, 120)); - } - const deadline = Date.now() + 120000; - while (!autoAi && Date.now() < deadline) { - await new Promise(resolve => setTimeout(resolve, 80)); - try { - const state = await win.webContents.executeJavaScript( - "JSON.stringify({done:window.__done===true,cancelled:window.__cancelled===true,point:window.__point||null})", true - ); - const parsed = JSON.parse(state) as { done: boolean; cancelled: boolean; point: { x: number; y: number } | null }; - if (parsed.cancelled) throw new RuntimeError("pick_cancelled", "已取消 AI 拾取"); - if (parsed.done && parsed.point) { - point = parsed.point; - break; - } - } catch (error) { - if (error instanceof RuntimeError && error.code === "pick_cancelled") throw error; - } - } - if (!point) throw new RuntimeError("pick_cancelled", "AI 拾取超时"); - - // Hide the effect layer so the screen capture contains the real application only. - win.hide(); - await new Promise(resolve => setTimeout(resolve, 180)); - if (!autoAi) { - const scale = display.scaleFactor || 1; - physicalX = Math.round((display.bounds.x + point.x) * scale); - physicalY = Math.round((display.bounds.y + point.y) * scale); - } - const capture = await captureAiAtPoint(physicalX, physicalY); - win.showInactive(); - await win.webContents.executeJavaScript( - "document.getElementById('tip').textContent='AI 视觉识别中…请稍候';document.body.classList.add('scanning');" - ); - - const image = await fs.promises.readFile(capture.imagePath); - let aiSettled = false; - const aiRequest = chatCompletion([ - { - role: "system", - content: "你是桌面 RPA 视觉定位器。只输出 JSON,不输出解释或 Markdown。" - }, - { - role: "user", - content: [ - { - type: "text", - text: [ - "截图尺寸 " + capture.cropW + "x" + capture.cropH + " 像素。", - "红黑色十字中心位于 (" + capture.markerX + ", " + capture.markerY + ")。", - "请定位十字中心处或半径 35 像素内最小的可交互 UI 控件;不要返回整个窗口或大容器。", - "只返回 JSON:{\"x\":number,\"y\":number,\"width\":number,\"height\":number,\"name\":string,\"controlType\":string,\"confidence\":number}", - "x/y 是控件左上角在截图内的像素坐标,width/height 单位也是像素。" - ].join("\n") - }, - { type: "image_url", image_url: { url: "data:image/png;base64," + image.toString("base64") } } - ] - } - ], { vision: true, temperature: 0 }).finally(() => { aiSettled = true; }); - const cancelWatcher = (async () => { - while (!aiSettled) { - try { - const state = await win.webContents.executeJavaScript("JSON.stringify({cancelled:window.__cancelled===true})", true); - if (JSON.parse(state).cancelled) throw new RuntimeError("pick_cancelled", "已取消 AI 拾取"); - } catch (error) { - if (error instanceof RuntimeError && error.code === "pick_cancelled") throw error; - } - await new Promise(resolve => setTimeout(resolve, 100)); - } - })(); - try { - var result = (await Promise.race([aiRequest, cancelWatcher])) as string; - } finally { - aiSettled = true; - } - - const box = parseAiBox(result, capture.cropW, capture.cropH); - const x = capture.cropX + box.x; - const y = capture.cropY + box.y; - const w = box.width; - const h = box.height; - const parts = [ - "screen=" + [x, y, w, h].map(Math.round).join(","), - context.window || capture.window ? "window=" + safeSelectorText(context.window || capture.window) : "", - "name=" + safeSelectorText(box.name), - "type=" + safeSelectorText(box.controlType || "Control"), - "ai=1" - ].filter(Boolean); - return { - source: "ai", - window: context.window || capture.window, - name: box.name, - controlType: box.controlType, - x, - y, - w, - h, - confidence: box.confidence, - screenshotPath: capture.imagePath, - selector: parts.join(";") - }; - } finally { - if (!win.isDestroyed()) win.destroy(); - } -} - export async function listVisibleWindows(): Promise> { const { stdout } = await run( "powershell.exe", @@ -722,3 +553,7 @@ export async function pickScreenRegion(): Promise<{ x: number; y: number; width: await run("powershell.exe", ["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-Command", psScript], { timeout: 30000, windowsHide: true }); return { ...region, imagePath }; } + + + + diff --git a/src/main/index.ts b/src/main/index.ts index 90b9c90..8728b93 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -9,7 +9,9 @@ import { capabilityCount, capabilityIds } from "./engine/registry"; import { closeBrowserSession, startRun, stopRun } from "./run/manager"; import { fixtureUrl, startFixtureServer, stopFixtureServer } from "./fixture/server"; import { loadSettings, updateSettings } from "./settings"; +import { closeLocalVisionWorkers, defaultUiModelPath } from "./local-vision/ui-detect-service"; import { pickDesktopElement, listVisibleWindows, pickScreenRegion } from "./desktop-pick"; +import { detectVisionLabTarget, listVisionLabSources } from "./vision-lab"; import { logMain } from "./logger"; let mainWindow: BrowserWindow | null = null; @@ -113,9 +115,12 @@ function registerIpc(): void { } return session.pickElement(); }); - ipcMain.handle("desktop:pick-element", (_event, options?: { ai?: boolean }) => pickDesktopElement(options)); + ipcMain.handle("desktop:pick-element", () => pickDesktopElement()); ipcMain.handle("desktop:pick-window-list", () => listVisibleWindows()); ipcMain.handle("desktop:pick-region", () => pickScreenRegion()); + ipcMain.handle("vision-lab:sources", () => listVisionLabSources()); + ipcMain.handle("vision-lab:detect", (_event, request: Parameters[0]) => detectVisionLabTarget(request)); + ipcMain.handle("vision-lab:default-model", () => defaultUiModelPath()); ipcMain.handle("flows:list", () => listFlows()); ipcMain.handle("flows:create", (_event, name?: string) => { const flow = createFlow(name); @@ -212,6 +217,7 @@ if (!gotLock) { app.on("before-quit", () => { void closeBrowserSession(); void stopFixtureServer(); + closeLocalVisionWorkers(); }); } diff --git a/src/main/local-vision/ui-detect-service.ts b/src/main/local-vision/ui-detect-service.ts new file mode 100644 index 0000000..518023f --- /dev/null +++ b/src/main/local-vision/ui-detect-service.ts @@ -0,0 +1,145 @@ +import fs from "node:fs"; +import path from "node:path"; +import { Worker } from "node:worker_threads"; +import { app } from "electron"; +import { RuntimeError } from "../engine/types"; +import type { LocalUiBox } from "./ui-detect-worker"; + +export interface UiDetectOptions { + image: Buffer; + width: number; + height: number; + modelPath?: string; + scoreThreshold?: number; + iouThreshold?: number; +} + +interface DetectRequest { + id: number; + image: Buffer; + width: number; + height: number; + scoreThreshold?: number; + iouThreshold?: number; +} + +interface DetectResponse { + id: number; + boxes?: LocalUiBox[]; + error?: string; + elapsedMs?: number; +} + +const workers = new Map(); +const pending = new Map void; reject: (error: unknown) => void; worker: Worker }>(); +let nextRequestId = 1; +const MAX_LOADED_WORKERS = 4; + +export function localUiModelPath(): string { + const candidates = [ + path.join(process.resourcesPath || "", "models", "ui-icon-detect", "model.onnx"), + path.join(app.getAppPath(), "resources", "models", "ui-icon-detect", "model.onnx") + ].filter(Boolean); + const found = candidates.find(candidate => fs.existsSync(candidate)); + if (!found) { + throw new RuntimeError("local_vision_model_missing", "本地 UI 检测模型缺失", { + recommendation: "重新安装应用,或补齐 resources/models/ui-icon-detect/model.onnx" + }); + } + return found; +} + +export function defaultUiModelPath(): string { + return localUiModelPath(); +} + +function resolveModelPath(candidate?: string): string { + if (!candidate) return localUiModelPath(); + const normalized = path.resolve(candidate); + if (!fs.existsSync(normalized) || !fs.statSync(normalized).isFile()) { + throw new RuntimeError("local_vision_model_missing", "模型文件不存在:" + normalized); + } + if (path.extname(normalized).toLowerCase() !== ".onnx") { + throw new RuntimeError("local_vision_model_invalid", "请选择 ONNX 模型文件"); + } + return normalized; +} + +function createWorker(modelPath: string): Worker { + const next = new Worker(path.join(__dirname, "local-vision/ui-detect-worker.js"), { + workerData: { modelPath }, + resourceLimits: { maxOldGenerationSizeMb: 512 } + }); + next.on("message", (message: DetectResponse) => { + const entry = pending.get(message.id); + if (!entry || entry.worker !== next) return; + pending.delete(message.id); + if (message.error) { + entry.reject(new RuntimeError("local_vision_failed", "本地 UI 检测失败:" + message.error)); + return; + } + entry.resolve(message.boxes || []); + }); + next.on("error", error => { + for (const [id, entry] of pending) { + if (entry.worker === next) { + entry.reject(error); + pending.delete(id); + } + } + workers.delete(modelPath); + }); + next.on("exit", code => { + for (const [id, entry] of pending) { + if (entry.worker === next) { + entry.reject(new RuntimeError("local_vision_failed", "本地 UI 检测进程已退出:" + code)); + pending.delete(id); + } + } + if (workers.get(modelPath) === next) workers.delete(modelPath); + }); + return next; +} + +function getWorker(modelPath: string): Worker { + const existing = workers.get(modelPath); + if (existing) return existing; + while (workers.size >= MAX_LOADED_WORKERS) { + const oldest = workers.keys().next().value; + if (!oldest) break; + const worker = workers.get(oldest); + workers.delete(oldest); + void worker?.terminate(); + } + const created = createWorker(modelPath); + workers.set(modelPath, created); + return created; +} + +export async function detectUiElements(input: UiDetectOptions): Promise { + const modelPath = resolveModelPath(input.modelPath); + const current = getWorker(modelPath); + const nextId = nextRequestId++; + + const request: DetectRequest = { + id: nextId, + image: input.image, + width: input.width, + height: input.height, + scoreThreshold: input.scoreThreshold, + iouThreshold: input.iouThreshold + }; + return new Promise((resolve, reject) => { + pending.set(nextId, { resolve, reject, worker: current }); + current.postMessage(request); + }); +} + +export function closeLocalVisionWorkers(): void { + for (const [modelPath, worker] of workers) { + workers.delete(modelPath); + void worker.terminate(); + } + pending.clear(); +} + diff --git a/src/main/local-vision/ui-detect-worker.ts b/src/main/local-vision/ui-detect-worker.ts new file mode 100644 index 0000000..3c6c369 --- /dev/null +++ b/src/main/local-vision/ui-detect-worker.ts @@ -0,0 +1,205 @@ +import { parentPort, workerData } from "node:worker_threads"; +import { InferenceSession, Tensor } from "onnxruntime-node"; +import { createCanvas, loadImage } from "@napi-rs/canvas"; + +export interface LocalUiBox { + x: number; + y: number; + width: number; + height: number; + category: string; + confidence: number; +} + +interface DetectRequest { + id: number; + image: Buffer; + width: number; + height: number; + scoreThreshold?: number; + iouThreshold?: number; +} + +interface DetectResponse { + id: number; + boxes?: LocalUiBox[]; + error?: string; + elapsedMs?: number; +} + +type ModelOutput = { output0: Tensor }; +type TensorLike = { dims: number[]; data: Float32Array }; + +const inputName = "images"; +const outputName = "output0"; +const inputSize = 640; +const defaultScoreThreshold = 0.25; +const defaultIouThreshold = 0.42; +const maxBoxes = 160; + +let sessionPromise: Promise | null = null; + +function loadSession(): Promise { + if (!sessionPromise) { + const modelPath = String((workerData as { modelPath?: string }).modelPath || ""); + sessionPromise = InferenceSession.create(modelPath, { + executionProviders: ["cpu"], + graphOptimizationLevel: "all", + executionMode: "sequential" + }); + } + return sessionPromise; +} + +function iou(a: LocalUiBox, b: LocalUiBox): number { + const x1 = Math.max(a.x, b.x); + const y1 = Math.max(a.y, b.y); + const x2 = Math.min(a.x + a.width, b.x + b.width); + const y2 = Math.min(a.y + a.height, b.y + b.height); + const intersection = Math.max(0, x2 - x1) * Math.max(0, y2 - y1); + if (intersection <= 0) return 0; + return intersection / (a.width * a.height + b.width * b.height - intersection); +} + +function nms(boxes: LocalUiBox[], iouThreshold: number): LocalUiBox[] { + const sorted = [...boxes].sort((a, b) => b.confidence - a.confidence); + const kept: LocalUiBox[] = []; + for (const box of sorted) { + if (kept.every(item => iou(item, box) <= iouThreshold)) kept.push(box); + } + return kept; +} + +function tensorAccessor(output: TensorLike, anchorCount: number, channelCount: number, channelFirst: boolean) { + return (channel: number, anchor: number): number => channelFirst + ? output.data[channel * anchorCount + anchor] + : output.data[anchor * channelCount + channel]; +} + +function extractBoxes( + output: TensorLike, + sourceWidth: number, + sourceHeight: number, + scoreThreshold: number, + iouThreshold: number +): LocalUiBox[] { + // YOLO/OmniParser exports are commonly [1,C,N] or [1,N,C]. C=4+classCount. + if (output.dims.length < 2) throw new Error(`不支持的模型输出维度:${output.dims.join("x")}`); + const [, first, second] = output.dims; + const channelFirst = first <= second; + const channelCount = channelFirst ? first : second; + const anchorCount = channelFirst ? second : first; + if (channelCount < 5) throw new Error(`模型输出通道数异常:${channelCount}`); + const at = tensorAccessor(output, anchorCount, channelCount, channelFirst); + const classCount = channelCount - 4; + + const padded = Math.max(sourceWidth, sourceHeight); + const scale = inputSize / padded; + const drawnWidth = Math.round(sourceWidth * scale); + const drawnHeight = Math.round(sourceHeight * scale); + const offsetX = (inputSize - drawnWidth) / 2; + const offsetY = (inputSize - drawnHeight) / 2; + const minSide = Math.max(8, Math.min(sourceWidth, sourceHeight) * 0.006); + const raw: LocalUiBox[] = []; + + for (let index = 0; index < anchorCount; index++) { + let confidence = 0; + let classIndex = 0; + if (classCount === 1) { + confidence = at(4, index); + } else { + for (let candidate = 0; candidate < classCount; candidate++) { + const score = at(4 + candidate, index); + if (score > confidence) { + confidence = score; + classIndex = candidate; + } + } + } + if (!Number.isFinite(confidence) || confidence < scoreThreshold) continue; + const cx = at(0, index); + const cy = at(1, index); + const width = at(2, index); + const height = at(3, index); + let x = (cx - width / 2 - offsetX) / scale; + let y = (cy - height / 2 - offsetY) / scale; + let boxWidth = width / scale; + let boxHeight = height / scale; + x = Math.max(0, Math.min(sourceWidth - 1, x)); + y = Math.max(0, Math.min(sourceHeight - 1, y)); + boxWidth = Math.max(2, Math.min(sourceWidth - x, boxWidth)); + boxHeight = Math.max(2, Math.min(sourceHeight - y, boxHeight)); + if (boxWidth < minSide || boxHeight < minSide) continue; + const areaRatio = (boxWidth * boxHeight) / (sourceWidth * sourceHeight); + if (areaRatio > 0.92) continue; + raw.push({ + x: Math.round(x), + y: Math.round(y), + width: Math.round(boxWidth), + height: Math.round(boxHeight), + category: classCount <= 1 ? "控件" : `类别${classIndex + 1}`, + confidence: Math.round(Math.max(0, Math.min(1, confidence)) * 1000) / 1000 + }); + } + + return nms(raw, iouThreshold) + .sort((a, b) => b.confidence - a.confidence || a.y - b.y || a.x - b.x) + .slice(0, maxBoxes) + .sort((a, b) => a.y - b.y || a.x - b.x); +} + +async function detect(request: DetectRequest): Promise { + const model = await loadSession(); + const source = await loadImage(request.image); + const width = request.width || source.width; + const height = request.height || source.height; + + const canvas = createCanvas(inputSize, inputSize); + const context = canvas.getContext("2d"); + context.fillStyle = "#727272"; + context.fillRect(0, 0, inputSize, inputSize); + context.imageSmoothingEnabled = true; + context.imageSmoothingQuality = "high"; + const scale = Math.min(inputSize / width, inputSize / height); + context.drawImage( + source, + (inputSize - width * scale) / 2, + (inputSize - height * scale) / 2, + width * scale, + height * scale + ); + const pixels = context.getImageData(0, 0, inputSize, inputSize).data; + const tensorData = new Float32Array(3 * inputSize * inputSize); + const planeSize = inputSize * inputSize; + for (let index = 0; index < planeSize; index++) { + tensorData[index] = pixels[index * 4] / 255; + tensorData[planeSize + index] = pixels[index * 4 + 1] / 255; + tensorData[planeSize * 2 + index] = pixels[index * 4 + 2] / 255; + } + + const results = await model.run({ [inputName]: new Tensor("float32", tensorData, [1, 3, inputSize, inputSize]) }); + const output = results[outputName] as unknown as TensorLike; + return extractBoxes( + output, + width, + height, + request.scoreThreshold ?? defaultScoreThreshold, + request.iouThreshold ?? defaultIouThreshold + ); +} + +parentPort?.on("message", async (request: DetectRequest) => { + const started = Date.now(); + try { + const boxes = await detect(request); + const response: DetectResponse = { id: request.id, boxes, elapsedMs: Date.now() - started }; + parentPort?.postMessage(response); + } catch (error) { + const response: DetectResponse = { + id: request.id, + error: error instanceof Error ? error.message : String(error), + elapsedMs: Date.now() - started + }; + parentPort?.postMessage(response); + } +}); diff --git a/src/main/vision-lab.ts b/src/main/vision-lab.ts new file mode 100644 index 0000000..819a731 --- /dev/null +++ b/src/main/vision-lab.ts @@ -0,0 +1,70 @@ +import { desktopCapturer } from "electron"; +import path from "node:path"; +import type { VisionLabDetectRequest, VisionLabDetection, VisionLabSource } from "../shared/types"; +import { RuntimeError } from "./engine/types"; +import { defaultUiModelPath, detectUiElements } from "./local-vision/ui-detect-service"; + +function sourceKind(sourceId: string): "window" | "screen" { + return sourceId.startsWith("screen:") ? "screen" : "window"; +} + +function clamp(value: unknown, fallback: number, min: number, max: number): number { + const parsed = Number(value); + if (!Number.isFinite(parsed)) return fallback; + return Math.min(max, Math.max(min, parsed)); +} + +export async function listVisionLabSources(): Promise { + const sources = await desktopCapturer.getSources({ + types: ["window", "screen"], + thumbnailSize: { width: 360, height: 210 }, + fetchWindowIcons: true + }); + return sources + .filter(source => source.thumbnail.isEmpty() === false) + .filter(source => sourceKind(source.id) === "screen" || source.name !== "RPA Studio") + .map(source => ({ + id: source.id, + kind: sourceKind(source.id), + name: source.name || (sourceKind(source.id) === "screen" ? "屏幕" : "未命名窗口"), + thumbnail: source.thumbnail.toDataURL(), + icon: source.appIcon?.toDataURL() + })); +} + +export async function detectVisionLabTarget(request: VisionLabDetectRequest): Promise { + if (!request?.sourceId) throw new RuntimeError("vision_lab_source_required", "请先选择识别目标"); + const sourceType = sourceKind(request.sourceId); + const captureStarted = Date.now(); + const sources = await desktopCapturer.getSources({ + types: [sourceType], + thumbnailSize: { width: 1280, height: 800 } + }); + const source = sources.find(item => item.id === request.sourceId); + if (!source) throw new RuntimeError("vision_lab_source_missing", "识别目标已失效,请重新选择窗口"); + if (source.thumbnail.isEmpty()) throw new RuntimeError("vision_lab_capture_failed", "目标窗口没有可捕获的画面"); + + const image = source.thumbnail.toPNG(); + const width = source.thumbnail.getSize().width; + const height = source.thumbnail.getSize().height; + const detectStarted = Date.now(); + const boxes = await detectUiElements({ + image, + width, + height, + modelPath: request.modelPath, + scoreThreshold: clamp(request.scoreThreshold, 0.25, 0.01, 0.95), + iouThreshold: clamp(request.iouThreshold, 0.42, 0.05, 0.9) + }); + return { + image: source.thumbnail.toDataURL(), + width, + height, + boxes, + captureMs: detectStarted - captureStarted, + detectMs: Date.now() - detectStarted, + modelPath: path.resolve(request.modelPath || defaultUiModelPath()), + sourceName: source.name || (sourceType === "screen" ? "屏幕" : "未命名窗口") + }; +} + diff --git a/src/preload/index.ts b/src/preload/index.ts index 77d0cce..6ba6b75 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -1,5 +1,5 @@ import { contextBridge, ipcRenderer } from "electron"; -import type { AppInfo, AppSettings, Catalog, FlowDefinition, FlowSaveOptions, FlowSaveResult, FlowSummary, RunEvent } from "../shared/types"; +import type { AppInfo, AppSettings, Catalog, FlowDefinition, FlowSaveOptions, FlowSaveResult, FlowSummary, RunEvent, VisionLabDetectRequest, VisionLabDetection, VisionLabSource } from "../shared/types"; const api = { app: { @@ -9,7 +9,7 @@ const api = { get: (): Promise => ipcRenderer.invoke("catalog:get") }, desktop: { - pickElement: (options?: { ai?: boolean }) => ipcRenderer.invoke("desktop:pick-element", options || {}), + pickElement: () => ipcRenderer.invoke("desktop:pick-element"), pickWindowList: () => ipcRenderer.invoke('desktop:pick-window-list'), pickRegion: () => ipcRenderer.invoke('desktop:pick-region') }, @@ -32,6 +32,11 @@ const api = { pickElement: (options?: { url?: string }): Promise => ipcRenderer.invoke("browser:pick-element", options || {}) }, + visionLab: { + sources: (): Promise => ipcRenderer.invoke("vision-lab:sources"), + detect: (request: VisionLabDetectRequest): Promise => ipcRenderer.invoke("vision-lab:detect", request), + defaultModel: (): Promise => ipcRenderer.invoke("vision-lab:default-model") + }, settings: { get: (): Promise => ipcRenderer.invoke("settings:get"), update: (patch: Partial): Promise => ipcRenderer.invoke("settings:update", patch || {}) diff --git a/src/renderer/src/App.tsx b/src/renderer/src/App.tsx index f3e954e..2c68d8e 100644 --- a/src/renderer/src/App.tsx +++ b/src/renderer/src/App.tsx @@ -1,16 +1,17 @@ import { useCallback, useEffect, useState } from "react"; -import { Bot, CircleAlert, Loader2, ListChecks } from "lucide-react"; +import { Bot, CircleAlert, Loader2, ListChecks, Microscope } from "lucide-react"; import type { Catalog, FlowDefinition, FlowSummary } from "@shared/types"; import { useEditorStore } from "./store/editorStore"; import { useRunStore } from "./store/runStore"; import { EditorPage } from "./pages/EditorPage"; import { RunsPage } from "./pages/RunsPage"; import { SettingsPage } from "./pages/SettingsPage"; +import { VisionLabPage } from "./pages/VisionLabPage"; import { TasksPage } from "./pages/TasksPage"; import { railIcons } from "./components/icons"; import { validateFlow } from "./validation/flowValidation"; -type Page = "tasks" | "editor" | "runs" | "settings"; +type Page = "tasks" | "editor" | "runs" | "settings" | "visionLab"; interface ToastState { id: number; @@ -235,6 +236,14 @@ export function App() { 设置 +
RA @@ -259,6 +268,7 @@ export function App() { {page === "editor" && { setPage("tasks"); void refreshFlows(); }} />} {page === "runs" && { setPage("tasks"); void refreshFlows(); }} />} {page === "settings" && } + {page === "visionLab" && } {toast && ( @@ -272,3 +282,4 @@ export function App() { } + diff --git a/src/renderer/src/pages/VisionLabPage.tsx b/src/renderer/src/pages/VisionLabPage.tsx new file mode 100644 index 0000000..27b7148 --- /dev/null +++ b/src/renderer/src/pages/VisionLabPage.tsx @@ -0,0 +1,232 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { FolderOpen, Play, Pause, RefreshCw, Sparkles } from "lucide-react"; +import type { VisionLabDetection, VisionLabSource } from "@shared/types"; + +function fileName(value: string): string { + const normalized = value.replace(/[\\/]+$/, ""); + const index = Math.max(normalized.lastIndexOf("\\"), normalized.lastIndexOf("/")); + return index >= 0 ? normalized.slice(index + 1) : normalized; +} + +export function VisionLabPage() { + const [sources, setSources] = useState([]); + const [sourceId, setSourceId] = useState(""); + const [defaultModel, setDefaultModel] = useState(""); + const [customModel, setCustomModel] = useState(null); + const [score, setScore] = useState(0.25); + const [overlap, setOverlap] = useState(0.42); + const [intervalMs, setIntervalMs] = useState(600); + const [continuous, setContinuous] = useState(false); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + const [detection, setDetection] = useState(null); + const canvasRef = useRef(null); + + const runTokenRef = useRef(0); + const selectedSource = sources.find(source => source.id === sourceId); + const activeModel = customModel || defaultModel; + + const refreshSources = useCallback(async () => { + try { + const next = await window.rpa.visionLab.sources(); + setSources(next); + setSourceId(current => next.some(source => source.id === current) ? current : next[0]?.id || ""); + setError(null); + } catch (refreshError) { + setError(refreshError instanceof Error ? refreshError.message : String(refreshError)); + } + }, []); + + useEffect(() => { + void refreshSources(); + void window.rpa.visionLab.defaultModel().then(setDefaultModel).catch(error => { + setError(error instanceof Error ? error.message : String(error)); + }); + }, [refreshSources]); + + const detectOnce = useCallback(async () => { + if (!sourceId) return; + setBusy(true); + try { + const result = await window.rpa.visionLab.detect({ + sourceId, + modelPath: customModel || undefined, + scoreThreshold: score, + iouThreshold: overlap + }); + setDetection(result); + setError(null); + } catch (detectError) { + setError(detectError instanceof Error ? detectError.message : String(detectError)); + } finally { + setBusy(false); + } + }, [customModel, overlap, score, sourceId]); + + useEffect(() => { + if (!continuous || !sourceId) return; + const token = ++runTokenRef.current; + let alive = true; + const tick = async () => { + while (alive) { + const started = Date.now(); + try { + const result = await window.rpa.visionLab.detect({ + sourceId, + modelPath: customModel || undefined, + scoreThreshold: score, + iouThreshold: overlap + }); + if (!alive || token !== runTokenRef.current) return; + setDetection(result); + setError(null); + } catch (detectError) { + if (!alive || token !== runTokenRef.current) return; + setError(detectError instanceof Error ? detectError.message : String(detectError)); + } + const wait = Math.max(80, intervalMs - (Date.now() - started)); + await new Promise(resolve => setTimeout(resolve, wait)); + } + }; + void tick(); + return () => { alive = false; }; + }, [continuous, customModel, intervalMs, overlap, score, sourceId]); + + useEffect(() => { + const canvas = canvasRef.current; + if (!canvas || !detection) return; + const image = new Image(); + image.onload = () => { + canvas.width = detection.width; + canvas.height = detection.height; + const context = canvas.getContext("2d"); + if (!context) return; + context.clearRect(0, 0, canvas.width, canvas.height); + context.drawImage(image, 0, 0, canvas.width, canvas.height); + const scale = Math.max(1, detection.width / 1000); + context.lineWidth = 2 * scale; + context.strokeStyle = "#22d3ee"; + context.font = `${Math.round(14 * scale)}px Microsoft YaHei, Arial, sans-serif`; + context.textBaseline = "top"; + for (const box of detection.boxes) { + context.strokeRect(box.x, box.y, box.width, box.height); + const label = `${box.category} ${Math.round(box.confidence * 100)}%`; + const metrics = context.measureText(label); + const labelWidth = metrics.width + 8 * scale; + const labelHeight = 19 * scale; + const labelY = box.y >= labelHeight + 2 * scale ? box.y - labelHeight - 2 * scale : box.y + 2 * scale; + context.fillStyle = "#0e7490"; + context.fillRect(box.x, labelY, labelWidth, labelHeight); + context.fillStyle = "#ffffff"; + context.fillText(label, box.x + 4 * scale, labelY + 3 * scale); + } + }; + image.src = detection.image; + }, [detection]); + + const chooseModel = async () => { + const selected = await window.rpa.dialog.open({ + title: "选择 ONNX UI 检测模型", + filters: [{ name: "ONNX 模型", extensions: ["onnx"] }] + }); + if (selected) setCustomModel(selected); + }; + + const modelLabel = useMemo(() => customModel ? fileName(customModel) : "内置 OmniParser v2", [customModel]); + const boxes = detection?.boxes || []; + + return ( +
+ + +
+
+ {detection?.sourceName || selectedSource?.name || "未选择目标"} + {boxes.length} 个候选 · 采集 {detection?.captureMs ?? "-"}ms · 推理 {detection?.detectMs ?? "-"}ms +
+
+ {detection ? ( + + ) : ( +
选择窗口和模型后开始识别
+ )} +
+
+
+ ); +} + diff --git a/src/renderer/src/panels/Properties.tsx b/src/renderer/src/panels/Properties.tsx index 2f3a9ec..3403e20 100644 --- a/src/renderer/src/panels/Properties.tsx +++ b/src/renderer/src/panels/Properties.tsx @@ -464,10 +464,10 @@ function FieldControl({ onChange(field.key, current === "" || known ? expression : `${textOf(raw)}${expression}`); }; - const pickDesktop = async (ai = false) => { + const pickDesktop = async () => { setPicking(true); try { - const picked = await window.rpa.desktop.pickElement({ ai }); + const picked = await window.rpa.desktop.pickElement(); if (picked.cancelled) { onToast("已取消拾取"); return; @@ -647,12 +647,9 @@ function FieldControl({ /> {isGroundTarget ? ( <> - - ) : (