feat: 新增本地UI模型AI拾取与识别实验台
This commit is contained in:
yu
parent
ee60435e2b
commit
e956ec9055
2
.gitignore
vendored
2
.gitignore
vendored
@ -12,3 +12,5 @@ test-output/
|
||||
web-capability-audit.txt
|
||||
|
||||
*.traineddata
|
||||
|
||||
temp/
|
||||
|
||||
@ -23,6 +23,8 @@ export default {
|
||||
extraResources: [
|
||||
{ from: path.join(root, "resources", "catalog.json"), to: "catalog.json" },
|
||||
{ from: path.join(root, "resources", "excluded.json"), to: "excluded.json" },
|
||||
{ from: path.join(root, "resources", "desktop-picker.ps1"), to: "desktop-picker.ps1" },
|
||||
{ from: path.join(root, "resources", "models"), to: "models" },
|
||||
{ from: path.join(root, "resources", "fixture"), to: "fixture" }
|
||||
],
|
||||
asar: true,
|
||||
|
||||
@ -12,7 +12,7 @@ export default defineConfig({
|
||||
},
|
||||
build: {
|
||||
rollupOptions: {
|
||||
input: { index: resolve("src/main/index.ts") }
|
||||
input: { index: resolve("src/main/index.ts"), "local-vision/ui-detect-worker": resolve("src/main/local-vision/ui-detect-worker.ts") }
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
5
package-lock.json
generated
5
package-lock.json
generated
@ -22,6 +22,7 @@
|
||||
"mssql": "^12.7.13",
|
||||
"mysql2": "^3.24.6",
|
||||
"nodemailer": "^6.9.16",
|
||||
"onnxruntime-node": "^1.31.1",
|
||||
"pdf-lib": "^1.17.1",
|
||||
"pdfjs-dist": "^4.10.38",
|
||||
"pg": "^8.23.1",
|
||||
@ -2362,7 +2363,7 @@
|
||||
},
|
||||
"node_modules/@napi-rs/canvas/node_modules/@napi-rs/canvas-linux-riscv64-gnu": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmmirror.com/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-1.1.0.tgz",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-1.1.0.tgz",
|
||||
"integrity": "sha512-kJ7m5n2VffxWXM5QGa5lfq8ZN4LdOJEoz3aAJAmIqYvKiZw1gRdXqpucnRXsF6D9nLYiiIlevaAtAclR7c5ilg==",
|
||||
"cpu": [
|
||||
"riscv64"
|
||||
@ -8452,7 +8453,7 @@
|
||||
},
|
||||
"node_modules/onnxruntime-node": {
|
||||
"version": "1.31.1",
|
||||
"resolved": "https://registry.npmmirror.com/onnxruntime-node/-/onnxruntime-node-1.31.1.tgz",
|
||||
"resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.31.1.tgz",
|
||||
"integrity": "sha512-ijnSghbDHVWaQ2nDPmVccPlCHI8kH1Mx1aJ4a/qzRP16bBcVs4E3EqZQKZCkhWnbqN1tuQWxagimFtDpbPYYQw==",
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
|
||||
@ -31,6 +31,7 @@
|
||||
"mssql": "^12.7.13",
|
||||
"mysql2": "^3.24.6",
|
||||
"nodemailer": "^6.9.16",
|
||||
"onnxruntime-node": "^1.31.1",
|
||||
"pdf-lib": "^1.17.1",
|
||||
"pdfjs-dist": "^4.10.38",
|
||||
"pg": "^8.23.1",
|
||||
|
||||
@ -150,10 +150,15 @@ function Set-ClickThrough($form) {
|
||||
}
|
||||
|
||||
function Set-HighlightBox([int]$x, [int]$y, [int]$w, [int]$h) {
|
||||
$hl.Location = New-Object System.Drawing.Point(($x - 2), ($y - 2))
|
||||
$hl.Size = New-Object System.Drawing.Size(($w + 4), ($h + 4))
|
||||
$hl.Show()
|
||||
Set-ClickThrough $hl
|
||||
$bounds = New-Object System.Drawing.Rectangle(($x - 2), ($y - 2), ($w + 4), ($h + 4))
|
||||
if (-not $hl.Visible) {
|
||||
$hl.Bounds = $bounds
|
||||
$hl.Show()
|
||||
Set-ClickThrough $hl
|
||||
} elseif ($hl.Bounds.X -ne $bounds.X -or $hl.Bounds.Y -ne $bounds.Y -or $hl.Bounds.Width -ne $bounds.Width -or $hl.Bounds.Height -ne $bounds.Height) {
|
||||
# 不重设隐藏；同一元素内移动时不重复改窗口，避免 WinForms 分层窗体重绘闪烁
|
||||
$hl.Bounds = $bounds
|
||||
}
|
||||
}
|
||||
function Hide-Highlight { $hl.Hide() }
|
||||
function Get-DeepestAtPoint([double]$px, [double]$py) {
|
||||
@ -187,6 +192,19 @@ $prevSpace = $false
|
||||
$prevPgUp = $false
|
||||
$prevPgDn = $false
|
||||
$prevF2 = $false
|
||||
$cacheX = [int]::MinValue
|
||||
$cacheY = [int]::MinValue
|
||||
$cacheEl = $null
|
||||
$cacheCtrlName = ""
|
||||
$cacheName = ""
|
||||
$cachePaneX = 0
|
||||
$cachePaneY = 0
|
||||
$cachePaneW = 0
|
||||
$cachePaneH = 0
|
||||
$cacheAt = [DateTime]::MinValue
|
||||
$lastQueryAt = [DateTime]::MinValue
|
||||
$queryGapMs = 45
|
||||
$cacheTtlMs = 1800
|
||||
Write-PickLog "loop start"
|
||||
while ($true) {
|
||||
try {
|
||||
@ -199,17 +217,49 @@ while ($true) {
|
||||
if ($stage -eq "hover") {
|
||||
$ptX = [System.Windows.Forms.Cursor]::Position.X
|
||||
$ptY = [System.Windows.Forms.Cursor]::Position.Y
|
||||
$el = Get-DeepestAtPoint $ptX $ptY
|
||||
$now = [DateTime]::UtcNow
|
||||
$samePoint = ($ptX -eq $cacheX -and $ptY -eq $cacheY)
|
||||
$insideCache = $cacheEl -ne $null -and $ptX -ge $cachePaneX -and $ptX -lt ($cachePaneX + $cachePaneW) -and $ptY -ge $cachePaneY -and $ptY -lt ($cachePaneY + $cachePaneH)
|
||||
$cacheAlive = ($null -ne $cacheEl -and ($now - $cacheAt).TotalMilliseconds -le $cacheTtlMs)
|
||||
# 鼠标还在当前元素矩形内部时不丢弃缓存；UIA 查询完成后才切换，避免移动过程先隐藏再显示
|
||||
if (-not $samePoint -and -not $insideCache) { $cacheEl = $null }
|
||||
if (-not $cacheAlive -and ($now - $lastQueryAt).TotalMilliseconds -ge $queryGapMs) {
|
||||
$queryX = $ptX
|
||||
$queryY = $ptY
|
||||
$lastQueryAt = [DateTime]::UtcNow
|
||||
try { $queried = Get-DeepestAtPoint $queryX $queryY } catch { $queried = $null; Write-PickLog ("QUERY ERR: " + $_.Exception.Message) }
|
||||
$afterX = [System.Windows.Forms.Cursor]::Position.X
|
||||
$afterY = [System.Windows.Forms.Cursor]::Position.Y
|
||||
if ($afterX -ne $queryX -or $afterY -ne $queryY) {
|
||||
$queried = $null
|
||||
Write-PickLog "query stale: cursor moved"
|
||||
}
|
||||
if ($null -ne $queried) {
|
||||
$qc = $queried.Current
|
||||
$qr = $qc.BoundingRectangle
|
||||
$cacheEl = $queried
|
||||
$cacheX = $queryX
|
||||
$cacheY = $queryY
|
||||
$cacheAt = [DateTime]::UtcNow
|
||||
$cacheCtrlName = $qc.ControlType.ProgrammaticName.Replace("ControlType.", "")
|
||||
$cacheName = [string]$qc.Name
|
||||
$cachePaneX = [int]$qr.X
|
||||
$cachePaneY = [int]$qr.Y
|
||||
$cachePaneW = [int]$qr.Width
|
||||
$cachePaneH = [int]$qr.Height
|
||||
} else {
|
||||
$cacheEl = $null
|
||||
$cacheCtrlName = ""
|
||||
$cacheName = ""
|
||||
}
|
||||
}
|
||||
$el = $cacheEl
|
||||
if ($el) {
|
||||
$c = $el.Current
|
||||
$ctrlName = $c.ControlType.ProgrammaticName.Replace("ControlType.", "")
|
||||
$r = $c.BoundingRectangle
|
||||
$paneX = [int]$r.X; $paneY = [int]$r.Y; $paneW = [int]$r.Width; $paneH = [int]$r.Height
|
||||
Set-HighlightBox $paneX $paneY $paneW $paneH
|
||||
$aiHint = if ($ctrlName -eq "Pane") { " [F2]AI识别" } else { "" }
|
||||
$infoLabel.Text = "名称: " + $c.Name + " 类型: " + $ctrlName + " [空格]锁定" + $aiHint + " [Esc]取消"
|
||||
Set-HighlightBox $cachePaneX $cachePaneY $cachePaneW $cachePaneH
|
||||
$aiHint = if ($cacheCtrlName -eq "Pane") { " [F2]AI识别" } else { "" }
|
||||
$infoLabel.Text = "名称: " + $cacheName + " 类型: " + $cacheCtrlName + " [空格]锁定" + $aiHint + " [Esc]取消"
|
||||
} else { Hide-Highlight }
|
||||
if ($f2 -and -not $prevF2 -and $el -and $ctrlName -eq "Pane") {
|
||||
if ($f2 -and -not $prevF2 -and $el -and $cacheCtrlName -eq "Pane") {
|
||||
Write-PickLog "f2 ai"
|
||||
$walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker
|
||||
$cur = $el
|
||||
@ -218,7 +268,7 @@ while ($true) {
|
||||
$topTitle = [string]$cur.Current.Name
|
||||
$cur = $walker.GetParent($cur)
|
||||
}
|
||||
$result = @{ aiRequested = $true; cursorX = $ptX; cursorY = $ptY; window = $topTitle; name = [string]$c.Name; controlType = $ctrlName; paneX = $paneX; paneY = $paneY; paneW = $paneW; paneH = $paneH }
|
||||
$result = @{ aiRequested = $true; cursorX = $ptX; cursorY = $ptY; window = $topTitle; name = $cacheName; controlType = $cacheCtrlName; paneX = $cachePaneX; paneY = $cachePaneY; paneW = $cachePaneW; paneH = $cachePaneH }
|
||||
break
|
||||
}
|
||||
if ($space -and -not $prevSpace -and $el) {
|
||||
@ -226,7 +276,7 @@ while ($true) {
|
||||
$stage = "locked"
|
||||
$chain = New-Object System.Collections.ArrayList
|
||||
$walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker
|
||||
$cur = $el
|
||||
$cur = $cacheEl
|
||||
while ($cur -and -not $cur.Equals([System.Windows.Automation.AutomationElement]::RootElement)) {
|
||||
$c = $cur.Current
|
||||
$r = $c.BoundingRectangle
|
||||
|
||||
BIN
resources/models/ui-icon-detect/model.onnx
Normal file
BIN
resources/models/ui-icon-detect/model.onnx
Normal file
Binary file not shown.
12
resources/models/ui-icon-detect/preprocessor_config.json
Normal file
12
resources/models/ui-icon-detect/preprocessor_config.json
Normal file
@ -0,0 +1,12 @@
|
||||
{
|
||||
"do_normalize": false,
|
||||
"do_rescale": true,
|
||||
"do_resize": true,
|
||||
"feature_extractor_type": "ImageFeatureExtractor",
|
||||
"resample": 2,
|
||||
"rescale_factor": 0.00392156862745098,
|
||||
"size_divisor": 16,
|
||||
"size": {
|
||||
"longest_edge": 640
|
||||
}
|
||||
}
|
||||
@ -115,9 +115,13 @@ if (!fs.existsSync(path.join(builderUnpacked, "RPA Studio.exe"))) {
|
||||
}
|
||||
const packagedCatalogPath = path.join(builderUnpacked, "resources", "catalog.json");
|
||||
const packagedFixturePath = path.join(builderUnpacked, "resources", "fixture", "index.html");
|
||||
const packagedPickerPath = path.join(builderUnpacked, "resources", "desktop-picker.ps1");
|
||||
if (!fs.existsSync(packagedCatalogPath) || !fs.existsSync(packagedFixturePath)) {
|
||||
throw new Error("打包产物缺少运行资源：resources/catalog.json 或 resources/fixture/index.html");
|
||||
}
|
||||
if (!fs.existsSync(packagedPickerPath)) {
|
||||
throw new Error("打包产物缺少桌面拾取脚本：resources/desktop-picker.ps1 未随包发布");
|
||||
}
|
||||
const packagedCatalog = JSON.parse(fs.readFileSync(packagedCatalogPath, "utf8"));
|
||||
if (!Array.isArray(packagedCatalog.components) || packagedCatalog.components.length === 0) {
|
||||
throw new Error("打包产物 resources/catalog.json 无效：组件目录为空");
|
||||
@ -216,6 +220,9 @@ if (!fs.existsSync(path.join(installSmokeTarget, "RPA Studio.exe"))) {
|
||||
if (!fs.existsSync(path.join(installSmokeTarget, "resources", "catalog.json"))) {
|
||||
throw new Error("NSIS 安装包冒烟失败：catalog.json 未安装");
|
||||
}
|
||||
if (!fs.existsSync(path.join(installSmokeTarget, "resources", "desktop-picker.ps1"))) {
|
||||
throw new Error("NSIS 安装包冒烟失败：desktop-picker.ps1 未安装");
|
||||
}
|
||||
await verifyLaunch(path.join(installSmokeTarget, "RPA Studio.exe"), "installed");
|
||||
removeTree(installSmokeRoot);
|
||||
|
||||
|
||||
@ -233,6 +233,18 @@ export class BrowserSession implements BrowserLike {
|
||||
return !name || accessibleName.includes(name);
|
||||
}) || null;
|
||||
}
|
||||
if (spec.kind === 'css') {
|
||||
const segments = String(spec.value).split('>>>').map((item) => item.trim()).filter(Boolean);
|
||||
let root = document;
|
||||
for (let index = 0; index < segments.length; index += 1) {
|
||||
const found = root.querySelector(segments[index]);
|
||||
if (!found) return null;
|
||||
if (index === segments.length - 1) return found;
|
||||
root = found.shadowRoot;
|
||||
if (!root) return null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
return document.querySelector(spec.value);
|
||||
};
|
||||
const el = find();
|
||||
@ -269,33 +281,46 @@ export class BrowserSession implements BrowserLike {
|
||||
try { return document.querySelectorAll(candidate).length === 1; } catch { return false; }
|
||||
};
|
||||
const cssPath = (node) => {
|
||||
if (node.id) {
|
||||
const candidate = '#' + cssEscape(node.id);
|
||||
if (isUnique(candidate)) return candidate;
|
||||
}
|
||||
const testId = node.getAttribute('data-testid');
|
||||
if (testId) {
|
||||
const candidate = '[data-testid="' + cssEscape(testId) + '"]';
|
||||
if (isUnique(candidate)) return candidate;
|
||||
}
|
||||
const parts = [];
|
||||
const localPath = (start, root) => {
|
||||
const stable = (item) => {
|
||||
if (item.id) {
|
||||
const candidate = '#' + cssEscape(item.id);
|
||||
try { if (root.querySelectorAll(candidate).length === 1) return candidate; } catch {}
|
||||
}
|
||||
const testId = item.getAttribute('data-testid');
|
||||
if (testId) {
|
||||
const candidate = '[data-testid="' + cssEscape(testId) + '"]';
|
||||
try { if (root.querySelectorAll(candidate).length === 1) return candidate; } catch {}
|
||||
}
|
||||
return null;
|
||||
};
|
||||
const parts = [];
|
||||
let current = start;
|
||||
while (current && current.nodeType === 1) {
|
||||
const stableCandidate = stable(current);
|
||||
if (stableCandidate) {
|
||||
parts.unshift(stableCandidate);
|
||||
return parts.join(' > ');
|
||||
}
|
||||
let part = current.tagName.toLowerCase();
|
||||
const parent = current.parentElement;
|
||||
if (parent) {
|
||||
const siblings = [...parent.children].filter((item) => item.tagName === current.tagName);
|
||||
if (siblings.length > 1) part += ':nth-of-type(' + (siblings.indexOf(current) + 1) + ')';
|
||||
}
|
||||
parts.unshift(part);
|
||||
current = parent;
|
||||
}
|
||||
return parts.join(' > ');
|
||||
};
|
||||
const trees = [];
|
||||
let current = node;
|
||||
while (current && current.nodeType === 1) {
|
||||
let part = current.tagName.toLowerCase();
|
||||
if (current.id) {
|
||||
part += '#' + cssEscape(current.id);
|
||||
parts.unshift(part);
|
||||
break;
|
||||
}
|
||||
const parent = current.parentElement;
|
||||
if (parent) {
|
||||
const siblings = [...parent.children].filter((item) => item.tagName === current.tagName);
|
||||
if (siblings.length > 1) part += ':nth-of-type(' + (siblings.indexOf(current) + 1) + ')';
|
||||
}
|
||||
parts.unshift(part);
|
||||
current = parent;
|
||||
const root = current.getRootNode();
|
||||
trees.unshift(localPath(current, root));
|
||||
current = root instanceof ShadowRoot ? root.host : null;
|
||||
}
|
||||
return parts.join(' > ');
|
||||
return trees.join(' >>> ');
|
||||
};
|
||||
const describe = (node) => ({
|
||||
kind: 'css',
|
||||
@ -1143,33 +1168,46 @@ export class BrowserSession implements BrowserLike {
|
||||
try { return document.querySelectorAll(candidate).length === 1; } catch { return false; }
|
||||
};
|
||||
const cssPath = (node) => {
|
||||
if (node.id) {
|
||||
const candidate = '#' + cssEscape(node.id);
|
||||
if (isUnique(candidate)) return candidate;
|
||||
}
|
||||
const testId = node.getAttribute('data-testid');
|
||||
if (testId) {
|
||||
const candidate = '[data-testid="' + cssEscape(testId) + '"]';
|
||||
if (isUnique(candidate)) return candidate;
|
||||
}
|
||||
const parts = [];
|
||||
const localPath = (start, root) => {
|
||||
const stable = (item) => {
|
||||
if (item.id) {
|
||||
const candidate = '#' + cssEscape(item.id);
|
||||
try { if (root.querySelectorAll(candidate).length === 1) return candidate; } catch {}
|
||||
}
|
||||
const testId = item.getAttribute('data-testid');
|
||||
if (testId) {
|
||||
const candidate = '[data-testid="' + cssEscape(testId) + '"]';
|
||||
try { if (root.querySelectorAll(candidate).length === 1) return candidate; } catch {}
|
||||
}
|
||||
return null;
|
||||
};
|
||||
const parts = [];
|
||||
let current = start;
|
||||
while (current && current.nodeType === 1) {
|
||||
const stableCandidate = stable(current);
|
||||
if (stableCandidate) {
|
||||
parts.unshift(stableCandidate);
|
||||
return parts.join(' > ');
|
||||
}
|
||||
let part = current.tagName.toLowerCase();
|
||||
const parent = current.parentElement;
|
||||
if (parent) {
|
||||
const siblings = [...parent.children].filter((item) => item.tagName === current.tagName);
|
||||
if (siblings.length > 1) part += ':nth-of-type(' + (siblings.indexOf(current) + 1) + ')';
|
||||
}
|
||||
parts.unshift(part);
|
||||
current = parent;
|
||||
}
|
||||
return parts.join(' > ');
|
||||
};
|
||||
const trees = [];
|
||||
let current = node;
|
||||
while (current && current.nodeType === 1) {
|
||||
let part = current.tagName.toLowerCase();
|
||||
if (current.id) {
|
||||
part += '#' + cssEscape(current.id);
|
||||
parts.unshift(part);
|
||||
break;
|
||||
}
|
||||
const parent = current.parentElement;
|
||||
if (parent) {
|
||||
const siblings = [...parent.children].filter((item) => item.tagName === current.tagName);
|
||||
if (siblings.length > 1) part += ':nth-of-type(' + (siblings.indexOf(current) + 1) + ')';
|
||||
}
|
||||
parts.unshift(part);
|
||||
current = parent;
|
||||
const root = current.getRootNode();
|
||||
trees.unshift(localPath(current, root));
|
||||
current = root instanceof ShadowRoot ? root.host : null;
|
||||
}
|
||||
return parts.join(' > ');
|
||||
return trees.join(' >>> ');
|
||||
};
|
||||
const describe = (node) => {
|
||||
const rect = node.getBoundingClientRect();
|
||||
@ -1200,9 +1238,33 @@ export class BrowserSession implements BrowserLike {
|
||||
badge.style.left = Math.max(0, rect.left) + 'px';
|
||||
badge.style.top = Math.max(0, rect.top - 30) + 'px';
|
||||
};
|
||||
const interactiveSelector = [
|
||||
'button', 'a', 'input', 'select', 'textarea', 'summary', 'label', 'option',
|
||||
'[role]', '[onclick]', '[contenteditable]', '[tabindex]'
|
||||
].join(',');
|
||||
const closestInteractive = (start) => {
|
||||
let node = start;
|
||||
while (node && node.nodeType === 1) {
|
||||
try { if (node.matches(interactiveSelector)) return node; } catch {}
|
||||
if (node.parentElement) {
|
||||
node = node.parentElement;
|
||||
continue;
|
||||
}
|
||||
const root = node.getRootNode();
|
||||
node = root instanceof ShadowRoot ? root.host : null;
|
||||
}
|
||||
return start;
|
||||
};
|
||||
const firstElement = (event) => {
|
||||
for (const item of event.composedPath()) if (item instanceof Element) return item;
|
||||
return event.target instanceof Element ? event.target : null;
|
||||
let deepest = null;
|
||||
for (const item of event.composedPath()) {
|
||||
if (item instanceof Element) {
|
||||
deepest = item;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!deepest && event.target instanceof Element) deepest = event.target;
|
||||
return closestInteractive(deepest);
|
||||
};
|
||||
const move = (event) => {
|
||||
if (finished) return;
|
||||
|
||||
@ -34,16 +34,29 @@ export function requireAiConfig(options: { vision?: boolean } = {}): ResolvedAiC
|
||||
return config;
|
||||
}
|
||||
|
||||
export async function chatCompletion(messages: ChatMessage[], options: { temperature?: number; vision?: boolean } = {}): Promise<string> {
|
||||
export interface ChatCompletionOptions {
|
||||
temperature?: number;
|
||||
vision?: boolean;
|
||||
/** 交互式请求（用户正在等待，如 AI 拾取）：默认短超时 45s、最多 2 次，超时不重试 */
|
||||
interactive?: boolean;
|
||||
/** 单次请求超时毫秒数；显式指定后覆盖交互式/后台默认值 */
|
||||
timeoutMs?: number;
|
||||
/** 最大尝试次数；显式指定后覆盖交互式/后台默认值 */
|
||||
maxAttempts?: number;
|
||||
}
|
||||
|
||||
export async function chatCompletion(messages: ChatMessage[], options: ChatCompletionOptions = {}): Promise<string> {
|
||||
const config = requireAiConfig({ vision: options.vision });
|
||||
const base = config.baseUrl.replace(/\/+$/, "");
|
||||
const maxAttempts = 3;
|
||||
const interactive = options.interactive === true;
|
||||
const timeoutMs = Math.max(1000, Math.round(options.timeoutMs ?? (interactive ? 45000 : 170000)));
|
||||
const maxAttempts = Math.max(1, Math.round(options.maxAttempts ?? (interactive ? 2 : 3)));
|
||||
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch(base + "/chat/completions", {
|
||||
signal: AbortSignal.timeout(170000),
|
||||
signal: AbortSignal.timeout(timeoutMs),
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", Authorization: "Bearer " + config.apiKey },
|
||||
body: JSON.stringify({
|
||||
@ -53,6 +66,10 @@ export async function chatCompletion(messages: ChatMessage[], options: { tempera
|
||||
})
|
||||
});
|
||||
} catch (error) {
|
||||
// 超时（AbortSignal.timeout）不重试：重发只会成倍拉长等待
|
||||
if ((error as { name?: string }).name === "TimeoutError") {
|
||||
throw new RuntimeError("ai_request_timeout", `AI 请求超时（${timeoutMs}ms），请检查网络或稍后重试`);
|
||||
}
|
||||
if (attempt === maxAttempts) throw error;
|
||||
await new Promise(resolve => setTimeout(resolve, 1200 * attempt));
|
||||
continue;
|
||||
|
||||
50
src/main/capabilities/cv-vision.test.ts
Normal file
50
src/main/capabilities/cv-vision.test.ts
Normal file
@ -0,0 +1,50 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const powershellMock = vi.hoisted(() => vi.fn(async () => '{"ok":true}'));
|
||||
const detectMock = vi.hoisted(() => vi.fn(async () => ({
|
||||
texts: [{ text: "确认保存", box: [[0, 0], [20, 0], [20, 10], [0, 10]] }]
|
||||
})));
|
||||
|
||||
vi.mock("./desktop", () => ({ powershell: powershellMock }));
|
||||
vi.mock("@gutenye/ocr-node", () => ({
|
||||
default: { create: vi.fn(async () => ({ detect: detectMock })) }
|
||||
}));
|
||||
|
||||
import { getHandler } from "../engine/registry";
|
||||
import "./cv-vision";
|
||||
|
||||
describe("CV vision capabilities", () => {
|
||||
beforeEach(() => {
|
||||
powershellMock.mockClear();
|
||||
detectMock.mockClear();
|
||||
});
|
||||
|
||||
it.each(["CV_GetText", "CV_SetText", "CV_MouseHover", "CV_MouseClick"])("registers %s", (id) => {
|
||||
expect(getHandler(id), id).toBeTruthy();
|
||||
});
|
||||
|
||||
it("rejects an empty CV selector", async () => {
|
||||
await expect(getHandler("CV_GetText")!({ log: vi.fn() } as any, { id: "n1", componentId: "CV_GetText" } as any, {}, {} as any))
|
||||
.rejects.toThrow("CV 目标为空");
|
||||
});
|
||||
|
||||
it("locates OCR text and clicks its center with offset", async () => {
|
||||
const handler = getHandler("CV_MouseClick")!;
|
||||
await expect(handler({ log: vi.fn() } as any, { id: "n1", componentId: "CV_MouseClick" } as any, {
|
||||
cvSelector: "确认",
|
||||
offsetX: 5,
|
||||
offsetY: 2
|
||||
}, {} as any)).resolves.toBe(true);
|
||||
expect(detectMock).toHaveBeenCalledTimes(1);
|
||||
const script = String((powershellMock.mock.calls as unknown as Array<[string]>)[1]?.[0] ?? "");
|
||||
expect(script).toContain("SetCursorPos(15, 7)");
|
||||
expect(script).toContain("mouse_event(0x0002");
|
||||
});
|
||||
|
||||
it("returns matched OCR text", async () => {
|
||||
await expect(getHandler("CV_GetText")!({ log: vi.fn() } as any, { id: "n2", componentId: "CV_GetText" } as any, {
|
||||
cvSelector: "保存"
|
||||
}, {} as any)).resolves.toBe("确认保存");
|
||||
expect(powershellMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@ -1,5 +1,5 @@
|
||||
import { register } from "../engine/registry";
|
||||
import { RuntimeError } from "../engine/types";
|
||||
import { RuntimeError, type CapabilityHandler } from "../engine/types";
|
||||
import { powershell } from "./desktop";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
@ -57,4 +57,95 @@ export async function locateTextOnScreen(target: string, nodeId: string): Promis
|
||||
const cx = (Math.min(...xs) + Math.max(...xs)) / 2;
|
||||
const cy = (Math.min(...ys) + Math.max(...ys)) / 2;
|
||||
return { target: keyword, text: hit.text, cx, cy };
|
||||
}
|
||||
}
|
||||
function screenUser32(): string {
|
||||
return [
|
||||
"Add-Type -MemberDefinition '[DllImport(\"user32.dll\")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value); [DllImport(\"user32.dll\")] public static extern bool SetCursorPos(int X, int Y); [DllImport(\"user32.dll\")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra);' -Name RpaCvInput -Namespace RpaCv | Out-Null;",
|
||||
"try { [RpaCv.RpaCvInput]::SetProcessDpiAwarenessContext([IntPtr](-4)) | Out-Null } catch {}"
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function coordinateScript(x: number, y: number, action: "hover" | "click" | "doubleClick" | "paste", value = ""): string {
|
||||
const lines = [
|
||||
screenUser32(),
|
||||
"if (-not [RpaCv.RpaCvInput]::SetCursorPos(" + x + ", " + y + ")) { throw 'cursor_move_failed' }",
|
||||
"Start-Sleep -Milliseconds 100"
|
||||
];
|
||||
if (action !== "hover") {
|
||||
lines.push(
|
||||
"[RpaCv.RpaCvInput]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)",
|
||||
"Start-Sleep -Milliseconds 40",
|
||||
"[RpaCv.RpaCvInput]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)"
|
||||
);
|
||||
}
|
||||
if (action === "doubleClick") {
|
||||
lines.push(
|
||||
"Start-Sleep -Milliseconds 70",
|
||||
"[RpaCv.RpaCvInput]::mouse_event(0x0002,0,0,0,[UIntPtr]::Zero)",
|
||||
"Start-Sleep -Milliseconds 40",
|
||||
"[RpaCv.RpaCvInput]::mouse_event(0x0004,0,0,0,[UIntPtr]::Zero)"
|
||||
);
|
||||
}
|
||||
if (action === "paste") {
|
||||
lines.push(
|
||||
"Add-Type -AssemblyName System.Windows.Forms",
|
||||
"Start-Sleep -Milliseconds 220",
|
||||
"$saved = $null",
|
||||
"try { $saved = [System.Windows.Forms.Clipboard]::GetText() } catch {}",
|
||||
"try { [System.Windows.Forms.Clipboard]::SetText(" + JSON.stringify(value) + ") } catch { throw 'clipboard_failed' }",
|
||||
"Start-Sleep -Milliseconds 80",
|
||||
"[System.Windows.Forms.SendKeys]::SendWait('^a')",
|
||||
"Start-Sleep -Milliseconds 60",
|
||||
"[System.Windows.Forms.SendKeys]::SendWait('^v')",
|
||||
"Start-Sleep -Milliseconds 280",
|
||||
"try { if ($saved) { [System.Windows.Forms.Clipboard]::SetText($saved) } else { [System.Windows.Forms.Clipboard]::Clear() } } catch {}"
|
||||
);
|
||||
}
|
||||
lines.push("Write-Output (@{ ok = $true; x = " + x + "; y = " + y + " } | ConvertTo-Json -Compress)");
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
async function locateFor(args: Record<string, unknown>, node: { id: string; componentId: string }) {
|
||||
const target = cvTarget(args, node.id);
|
||||
const located = await locateTextOnScreen(target, node.id);
|
||||
const offsetX = Number(args.offsetX ?? args.offset_x ?? 0) || 0;
|
||||
const offsetY = Number(args.offsetY ?? args.offset_y ?? 0) || 0;
|
||||
return { located, x: Math.round(located.cx + offsetX), y: Math.round(located.cy + offsetY) };
|
||||
}
|
||||
|
||||
const getTextGround: CapabilityHandler = async (ctx, node, args) => {
|
||||
const target = cvTarget(args, node.id);
|
||||
const result = await locateTextOnScreen(target, node.id);
|
||||
ctx.log("CV_GetText found: " + result.text, "info", node.id);
|
||||
return result.text;
|
||||
};
|
||||
|
||||
const setTextGround: CapabilityHandler = async (ctx, node, args) => {
|
||||
const value = String(args.value ?? args.text ?? "");
|
||||
if (!value) throw new RuntimeError("validation_error", "CV 文本输入内容为空", { nodeId: node.id, componentId: node.componentId });
|
||||
const { x, y } = await locateFor(args, node);
|
||||
await powershell(coordinateScript(x, y, "paste", value), 30000);
|
||||
ctx.log("CV_SetText input at " + x + "," + y, "info", node.id);
|
||||
return true;
|
||||
};
|
||||
|
||||
const mouseHoverGround: CapabilityHandler = async (ctx, node, args) => {
|
||||
const { x, y } = await locateFor(args, node);
|
||||
await powershell(coordinateScript(x, y, "hover"), 15000);
|
||||
ctx.log("CV_MouseHover hover at " + x + "," + y, "info", node.id);
|
||||
return true;
|
||||
};
|
||||
|
||||
const mouseClickGround: CapabilityHandler = async (ctx, node, args) => {
|
||||
const { x, y } = await locateFor(args, node);
|
||||
const clickway = String(args.clickway ?? "0");
|
||||
const action = clickway === "1" || /double/i.test(clickway) ? "doubleClick" as const : "click" as const;
|
||||
await powershell(coordinateScript(x, y, action), 15000);
|
||||
ctx.log("CV_MouseClick " + action + " at " + x + "," + y, "info", node.id);
|
||||
return true;
|
||||
};
|
||||
|
||||
register("CV_GetText", getTextGround);
|
||||
register("CV_SetText", setTextGround);
|
||||
register("CV_MouseHover", mouseHoverGround);
|
||||
register("CV_MouseClick", mouseClickGround);
|
||||
|
||||
@ -1,7 +1,16 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const powershellMock = vi.hoisted(() => vi.fn(async () => '{"clicked":true,"value":"cached"}'));
|
||||
|
||||
vi.mock("./desktop", () => ({
|
||||
powershell: powershellMock,
|
||||
psString: (value: unknown) => JSON.stringify(String(value ?? ""))
|
||||
}));
|
||||
|
||||
import { getHandler } from "../engine/registry";
|
||||
import { parseUiaSelector } from "./desktop-uia";
|
||||
|
||||
|
||||
describe("desktop UIA ground capabilities", () => {
|
||||
it.each([
|
||||
"Element_ClickGround",
|
||||
@ -21,7 +30,10 @@ describe("desktop UIA ground capabilities", () => {
|
||||
"Element_MouseToGround",
|
||||
"Element_ScrollIntoViewGround",
|
||||
"Element_ChildrenGround",
|
||||
"Element_DesktopChildren"
|
||||
"Element_DesktopChildren",
|
||||
"Element_RestoreWindowGround",
|
||||
"Element_ActivateWindowGround",
|
||||
"Element_MoveWindowGround"
|
||||
])("registers %s", (id) => {
|
||||
expect(getHandler(id), id).toBeTruthy();
|
||||
});
|
||||
@ -57,3 +69,61 @@ describe("desktop UIA ground capabilities", () => {
|
||||
expect(() => parseUiaSelector("screen=1,2,3")).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("desktop UIA selector compatibility", () => {
|
||||
beforeEach(() => {
|
||||
powershellMock.mockClear();
|
||||
powershellMock.mockImplementation(async () => '{"clicked":true,"value":"cached"}');
|
||||
});
|
||||
|
||||
it("maps compound .NET ControlType names explicitly and preserves raw input", () => {
|
||||
expect(parseUiaSelector("type=combobox")).toMatchObject({ type: "ComboBox", typeRaw: "combobox" });
|
||||
expect(parseUiaSelector("type=listitem")).toMatchObject({ type: "ListItem", typeRaw: "listitem" });
|
||||
expect(parseUiaSelector("type=datagrid")).toMatchObject({ type: "DataGrid", typeRaw: "datagrid" });
|
||||
});
|
||||
|
||||
it("parses a stable ancestor path", () => {
|
||||
const selector = parseUiaSelector("window=App;path=Pane>Edit[name=用户名;index=2]>Button[name=确定]");
|
||||
expect(selector.path).toEqual([
|
||||
{ type: "Pane" },
|
||||
{ type: "Edit", name: "用户名", index: 2 },
|
||||
{ type: "Button", name: "确定" }
|
||||
]);
|
||||
});
|
||||
|
||||
const context = { log: vi.fn() } as any;
|
||||
const node = { id: "n1", componentId: "Element_ClickGround" } as any;
|
||||
|
||||
it("dispatches screen selectors to coordinate execution instead of UIA lookup", async () => {
|
||||
const args = {
|
||||
selector: "screen=100,200,320,80;ai=1",
|
||||
clickway: "1"
|
||||
};
|
||||
await expect(getHandler("Element_ClickGround")!(context, node, args, {} as any)).resolves.toEqual({ clicked: true, value: "cached" });
|
||||
const script = String((powershellMock.mock.calls as unknown as Array<[string]>)[0]?.[0] ?? "");
|
||||
expect(script).toContain("RpaScreenInput");
|
||||
expect(script).toContain("SetCursorPos(260, 240)");
|
||||
expect(script).not.toContain("Find-RpaWindow");
|
||||
});
|
||||
|
||||
it("dispatches screen setValue to coordinate paste", async () => {
|
||||
await getHandler("Element_SetValueGround")!(context, node, {
|
||||
selector: "screen=10,20,30,40",
|
||||
value: "hello"
|
||||
}, {} as any);
|
||||
const script = String((powershellMock.mock.calls as unknown as Array<[string]>)[0]?.[0] ?? "");
|
||||
expect(script).toContain("screen_coordinate_paste");
|
||||
expect(script).toContain("hello");
|
||||
expect(script).not.toContain("Find-RpaWindow");
|
||||
});
|
||||
|
||||
it("treats a picked screen rectangle as present for exists/wait", async () => {
|
||||
await expect(getHandler("Element_ExistsGround")!(context, node, {
|
||||
selector: "screen=1,2,3,4"
|
||||
}, {} as any)).resolves.toBe(true);
|
||||
await expect(getHandler("Element_WaitElementGround")!(context, node, {
|
||||
selector: "screen=1,2,3,4"
|
||||
}, {} as any)).resolves.toBe(true);
|
||||
expect(powershellMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@ -8,12 +8,124 @@ interface UiaSelector {
|
||||
window?: string;
|
||||
name?: string;
|
||||
id?: string;
|
||||
/** .NET ControlType 字段名（已映射，如 ComboBox/ListItem/DataGrid） */
|
||||
type?: string;
|
||||
/** 用户输入的原始类型字符串 */
|
||||
typeRaw?: string;
|
||||
className?: string;
|
||||
screen?: { x: number; y: number; w: number; h: number };
|
||||
/** 从窗口（或桌面根）到目标元素的祖先路径，逐段用稳定属性定位 */
|
||||
path?: Array<{ type?: string; name?: string; id?: string; className?: string; index?: number }>;
|
||||
index: number;
|
||||
}
|
||||
|
||||
/** ControlType 显式映射：type= 值 → .NET ControlType 字段名；复合名称不得用首字母大写推导 */
|
||||
const UIA_CONTROL_TYPE_MAP: Record<string, string> = {
|
||||
button: "Button",
|
||||
calendar: "Calendar",
|
||||
checkbox: "CheckBox",
|
||||
combo: "ComboBox",
|
||||
combobox: "ComboBox",
|
||||
custom: "Custom",
|
||||
datagrid: "DataGrid",
|
||||
dataitem: "DataItem",
|
||||
document: "Document",
|
||||
edit: "Edit",
|
||||
group: "Group",
|
||||
header: "Header",
|
||||
headeritem: "HeaderItem",
|
||||
hyperlink: "Hyperlink",
|
||||
image: "Image",
|
||||
list: "List",
|
||||
listbox: "List",
|
||||
listitem: "ListItem",
|
||||
menu: "Menu",
|
||||
menubar: "MenuBar",
|
||||
menuitem: "MenuItem",
|
||||
pane: "Pane",
|
||||
progressbar: "ProgressBar",
|
||||
radio: "RadioButton",
|
||||
radiobutton: "RadioButton",
|
||||
scrollbar: "ScrollBar",
|
||||
separator: "Separator",
|
||||
slider: "Slider",
|
||||
spinner: "Spinner",
|
||||
splitbutton: "SplitButton",
|
||||
statusbar: "StatusBar",
|
||||
tab: "Tab",
|
||||
tabitem: "TabItem",
|
||||
table: "Table",
|
||||
text: "Text",
|
||||
thumb: "Thumb",
|
||||
titlebar: "TitleBar",
|
||||
toolbar: "ToolBar",
|
||||
tooltip: "ToolTip",
|
||||
tree: "Tree",
|
||||
treeitem: "TreeItem",
|
||||
window: "Window"
|
||||
};
|
||||
|
||||
function normalizeUiaControlType(raw: string): string {
|
||||
const key = raw.trim().toLowerCase().replace(/[\s_-]+/g, "");
|
||||
return UIA_CONTROL_TYPE_MAP[key] ?? raw.charAt(0).toUpperCase() + raw.slice(1).toLowerCase();
|
||||
}
|
||||
|
||||
/** 解析 path= 段：Pane>Edit[name=用户名;index=2]，段间用 > 分隔，段内 [prop=value;...] */
|
||||
function parseUiaSelectorPath(value: string): UiaSelector["path"] {
|
||||
// 按括号外的 > 分段，避免 path=Edit[name=a>b] 这类属性值被错误拆开
|
||||
const rawSegments: string[] = [];
|
||||
let buffer = "";
|
||||
let bracketDepth = 0;
|
||||
for (const char of value) {
|
||||
if (char === "[") bracketDepth += 1;
|
||||
else if (char === "]") bracketDepth = Math.max(0, bracketDepth - 1);
|
||||
if (char === ">" && bracketDepth === 0) {
|
||||
rawSegments.push(buffer);
|
||||
buffer = "";
|
||||
} else {
|
||||
buffer += char;
|
||||
}
|
||||
}
|
||||
rawSegments.push(buffer);
|
||||
const segments = rawSegments.map(item => item.trim()).filter(Boolean);
|
||||
if (!segments.length) return undefined;
|
||||
const path: NonNullable<UiaSelector["path"]> = [];
|
||||
for (const segment of segments) {
|
||||
const match = segment.match(/^([A-Za-z][A-Za-z0-9_]*)?\s*(?:\[(.*)\])?$/);
|
||||
if (!match) {
|
||||
throw new RuntimeError("validation_error", `UIA 路径段无效：${segment}`, { recommendation: "使用 Type[name=xx;id=yy;class=zz;index=1] > 下一级" });
|
||||
}
|
||||
const entry: NonNullable<UiaSelector["path"]>[number] = {};
|
||||
if (match[1]) entry.type = normalizeUiaControlType(match[1]);
|
||||
const props = match[2];
|
||||
if (props) {
|
||||
for (const pair of props.split(/[;，,]/).map(item => item.trim()).filter(Boolean)) {
|
||||
const eq = pair.indexOf("=");
|
||||
if (eq <= 0) {
|
||||
throw new RuntimeError("validation_error", `UIA 路径属性无效：${pair}`, { recommendation: "使用 name= / id= / class= / index=" });
|
||||
}
|
||||
const key = pair.slice(0, eq).trim().toLowerCase();
|
||||
const val = pair.slice(eq + 1).trim();
|
||||
if (key === "name" || key === "n") entry.name = val;
|
||||
else if (key === "id" || key === "automationid") entry.id = val;
|
||||
else if (key === "class" || key === "classname") entry.className = val;
|
||||
else if (key === "type" || key === "controltype") entry.type = normalizeUiaControlType(val);
|
||||
else if (key === "index") {
|
||||
const n = Number(val);
|
||||
if (!Number.isFinite(n) || n < 1) throw new RuntimeError("validation_error", `UIA 路径 index 无效：${val}`);
|
||||
entry.index = Math.floor(n);
|
||||
}
|
||||
else throw new RuntimeError("validation_error", `UIA 路径属性不支持：${key}`);
|
||||
}
|
||||
}
|
||||
if (!entry.type && !entry.name && !entry.id && !entry.className) {
|
||||
throw new RuntimeError("validation_error", `UIA 路径段缺少定位条件：${segment}`, { recommendation: "每段至少提供 type/name/id/class 之一" });
|
||||
}
|
||||
path.push(entry);
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
export function parseUiaSelector(raw: unknown): UiaSelector {
|
||||
let text = "";
|
||||
if (raw && typeof raw === "object") {
|
||||
@ -33,7 +145,22 @@ export function parseUiaSelector(raw: unknown): UiaSelector {
|
||||
}
|
||||
if (text.startsWith("uia:")) text = text.slice(4);
|
||||
const selector: UiaSelector = { index: 1 };
|
||||
for (const part of text.split(";")) {
|
||||
// path 段内的 [name=a;b] 不应被顶层 ; 拆开
|
||||
const selectorParts: string[] = [];
|
||||
let partBuffer = "";
|
||||
let selectorBracketDepth = 0;
|
||||
for (const char of text) {
|
||||
if (char === "[") selectorBracketDepth += 1;
|
||||
else if (char === "]") selectorBracketDepth = Math.max(0, selectorBracketDepth - 1);
|
||||
if (char === ";" && selectorBracketDepth === 0) {
|
||||
selectorParts.push(partBuffer);
|
||||
partBuffer = "";
|
||||
} else {
|
||||
partBuffer += char;
|
||||
}
|
||||
}
|
||||
selectorParts.push(partBuffer);
|
||||
for (const part of selectorParts) {
|
||||
const eq = part.indexOf("=");
|
||||
if (eq <= 0) continue;
|
||||
const key = part.slice(0, eq).trim().toLowerCase();
|
||||
@ -42,7 +169,10 @@ export function parseUiaSelector(raw: unknown): UiaSelector {
|
||||
if (key === "window" || key === "win") selector.window = value;
|
||||
else if (key === "name" || key === "n") selector.name = value;
|
||||
else if (key === "id" || key === "automationid") selector.id = value;
|
||||
else if (key === "type" || key === "controltype") selector.type = value.charAt(0).toUpperCase() + value.slice(1).toLowerCase();
|
||||
else if (key === "type" || key === "controltype") {
|
||||
selector.typeRaw = value;
|
||||
selector.type = normalizeUiaControlType(value);
|
||||
}
|
||||
else if (key === "class" || key === "classname") selector.className = value;
|
||||
else if (key === "screen") {
|
||||
const nums = value.split(/[,，]/).map(item => Number(item.trim()));
|
||||
@ -55,8 +185,11 @@ export function parseUiaSelector(raw: unknown): UiaSelector {
|
||||
const n = Number(value);
|
||||
if (Number.isFinite(n) && n >= 1) selector.index = Math.floor(n);
|
||||
}
|
||||
else if (key === "path" || key === "ancestors" || key === "ancestorpath") {
|
||||
selector.path = parseUiaSelectorPath(value);
|
||||
}
|
||||
}
|
||||
if (!selector.name && !selector.id && !selector.className && !selector.type && !selector.screen) {
|
||||
if (!selector.name && !selector.id && !selector.className && !selector.type && !selector.screen && !selector.path) {
|
||||
throw new RuntimeError("validation_error", `桌面元素选择器缺少定位条件：${text}`, { recommendation: "至少提供 name= / id= / type= 之一" });
|
||||
}
|
||||
return selector;
|
||||
@ -69,6 +202,7 @@ const UIA_USING = [
|
||||
|
||||
const FIND_HELPER = `
|
||||
function Find-RpaWindow([string]$winName) {
|
||||
# 只读查找：绝不移动/还原/缩放窗口；还原、激活、移动是独立显式动作（Restore/Activate/Move-RpaWindow）
|
||||
$root = [System.Windows.Automation.AutomationElement]::RootElement
|
||||
if (-not $winName) { return $root }
|
||||
$cond = New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::NameProperty, $winName)
|
||||
@ -78,41 +212,88 @@ function Find-RpaWindow([string]$winName) {
|
||||
$script:hitHwnd = [IntPtr]::Zero
|
||||
$cb = [WinRpa.RpaUser32Click+EnumProc]{ param($h, $l)
|
||||
$sb = New-Object System.Text.StringBuilder 256
|
||||
[WinRpa.RpaUser32Click]::GetWindowTextW($h, $sb, 256) | Out-Null
|
||||
[void][WinRpa.RpaUser32Click]::GetWindowTextW($h, $sb, 256)
|
||||
if ($sb.ToString() -eq $winName) { $script:hitHwnd = $h; return $false }
|
||||
return $true
|
||||
}
|
||||
[WinRpa.RpaUser32Click]::EnumWindows($cb, [IntPtr]::Zero) | Out-Null
|
||||
[void][WinRpa.RpaUser32Click]::EnumWindows($cb, [IntPtr]::Zero)
|
||||
if ($script:hitHwnd -ne [IntPtr]::Zero) {
|
||||
[WinRpa.RpaUser32Click]::ShowWindow($script:hitHwnd, 9) | Out-Null
|
||||
Start-Sleep -Milliseconds 500
|
||||
$sw2 = [WinRpa.RpaUser32Click]::GetSystemMetrics(0)
|
||||
$sh2 = [WinRpa.RpaUser32Click]::GetSystemMetrics(1)
|
||||
$w2 = [Math]::Min(1400, $sw2 - 120)
|
||||
$h2 = [Math]::Min(820, $sh2 - 120)
|
||||
[WinRpa.RpaUser32Click]::SetWindowPos($script:hitHwnd, [IntPtr]::Zero, 60, 60, $w2, $h2, 0x0040) | Out-Null
|
||||
Start-Sleep -Milliseconds 600
|
||||
$win = $root.FindFirst([System.Windows.Automation.TreeScope]::Children, $cond)
|
||||
if ($win) { return $win }
|
||||
try { $win = [System.Windows.Automation.AutomationElement]::FromHandle($script:hitHwnd); if ($win) { return $win } } catch {}
|
||||
}
|
||||
throw "window_not_found:$winName"
|
||||
}
|
||||
function Find-RpaElement($parent, [string]$name, [string]$id, [string]$type, [string]$className, [int]$index) {
|
||||
function Restore-RpaWindow($win) {
|
||||
# 显式动作：还原窗口（最小化/最大化状态恢复），不用于查询
|
||||
Add-RpaUser32
|
||||
$hwnd = [IntPtr]$win.Current.NativeWindowHandle
|
||||
if ($hwnd -eq [IntPtr]::Zero) { throw "window_no_hwnd" }
|
||||
[void][WinRpa.RpaUser32Click]::ShowWindow($hwnd, 9)
|
||||
Start-Sleep -Milliseconds 300
|
||||
return $hwnd
|
||||
}
|
||||
function Activate-RpaWindow($win) {
|
||||
# 显式动作：把窗口带到前台
|
||||
Add-RpaUser32
|
||||
$hwnd = [IntPtr]$win.Current.NativeWindowHandle
|
||||
if ($hwnd -eq [IntPtr]::Zero) { throw "window_no_hwnd" }
|
||||
$fgHwnd = [WinRpa.RpaUser32Click]::GetForegroundWindow()
|
||||
if ($fgHwnd -ne $hwnd) {
|
||||
$fgPid = [uint32]0
|
||||
$fgThread = [WinRpa.RpaUser32Click]::GetWindowThreadProcessId($fgHwnd, [ref]$fgPid)
|
||||
$thisThread = [WinRpa.RpaUser32Click]::GetCurrentThreadId()
|
||||
[void][WinRpa.RpaUser32Click]::AttachThreadInput($thisThread, $fgThread, $true)
|
||||
[void][WinRpa.RpaUser32Click]::SetForegroundWindow($hwnd)
|
||||
[void][WinRpa.RpaUser32Click]::AttachThreadInput($thisThread, $fgThread, $false)
|
||||
}
|
||||
Start-Sleep -Milliseconds 200
|
||||
return $hwnd
|
||||
}
|
||||
function Move-RpaWindow($win, [int]$x, [int]$y, [int]$w, [int]$h) {
|
||||
# 显式动作：移动/缩放窗口；w/h<=0 时使用安全默认尺寸
|
||||
Add-RpaUser32
|
||||
$hwnd = [IntPtr]$win.Current.NativeWindowHandle
|
||||
if ($hwnd -eq [IntPtr]::Zero) { throw "window_no_hwnd" }
|
||||
$sw = [WinRpa.RpaUser32Click]::GetSystemMetrics(0)
|
||||
$sh = [WinRpa.RpaUser32Click]::GetSystemMetrics(1)
|
||||
if ($w -le 0) { $w = [Math]::Min(1400, $sw - 120) }
|
||||
if ($h -le 0) { $h = [Math]::Min(820, $sh - 120) }
|
||||
[void][WinRpa.RpaUser32Click]::SetWindowPos($hwnd, [IntPtr]::Zero, $x, $y, $w, $h, 0x0040)
|
||||
Start-Sleep -Milliseconds 300
|
||||
return $hwnd
|
||||
}
|
||||
function Find-RpaElement($parent, [string]$name, [string]$id, [string]$type, [string]$className, [int]$index, [switch]$directChild) {
|
||||
$conds = @()
|
||||
if ($name) { $conds += New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::NameProperty, $name) }
|
||||
if ($id) { $conds += New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::AutomationIdProperty, $id) }
|
||||
if ($className) { $conds += New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::ClassNameProperty, $className) }
|
||||
if ($type) {
|
||||
try { $field = [System.Windows.Automation.ControlType].GetField($type); if ($field) { $conds += New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::ControlTypeProperty, $field.GetValue($null)) } } catch {}
|
||||
if ($ct) { $conds += New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::ControlTypeProperty, $ct) }
|
||||
# 不区分大小写匹配 .NET ControlType 字段，兼容 ComboBox/ListItem/DataGrid 等复合名称
|
||||
try {
|
||||
$ctField = [System.Windows.Automation.ControlType].GetFields([System.Reflection.BindingFlags]"Public,Static") | Where-Object { $_.Name -ieq $type } | Select-Object -First 1
|
||||
if ($ctField) { $conds += New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::ControlTypeProperty, $ctField.GetValue($null)) }
|
||||
} catch {}
|
||||
}
|
||||
if ($conds.Count -eq 0) { throw "no_condition" }
|
||||
$final = if ($conds.Count -eq 1) { $conds[0] } else { New-Object System.Windows.Automation.AndCondition($conds) }
|
||||
$all = $parent.FindAll([System.Windows.Automation.TreeScope]::Descendants, $final)
|
||||
$scope = if ($directChild) { [System.Windows.Automation.TreeScope]::Children } else { [System.Windows.Automation.TreeScope]::Descendants }
|
||||
$all = $parent.FindAll($scope, $final)
|
||||
if ($all.Count -eq 0) { return $null }
|
||||
$pick = [Math]::Min([Math]::Max($index, 1), $all.Count)
|
||||
return $all[$pick - 1]
|
||||
}
|
||||
function Find-RpaElementByPath($parent, [string]$pathJsonBase64) {
|
||||
# 逐段走直接子级，用稳定属性（id/type/class 优先）解析窗口→目标祖先路径
|
||||
$segments = [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String($pathJsonBase64)) | ConvertFrom-Json
|
||||
$cur = $parent
|
||||
foreach ($seg in @($segments)) {
|
||||
$ix = 1
|
||||
try { if ($seg.index) { $ix = [int]$seg.index } } catch {}
|
||||
$next = Find-RpaElement $cur $seg.name $seg.id $seg.type $seg.className $ix -directChild
|
||||
if ($null -eq $next) { return $null }
|
||||
$cur = $next
|
||||
}
|
||||
return $cur
|
||||
}
|
||||
function Add-RpaUser32 {
|
||||
Add-Type -MemberDefinition '[DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y); [DllImport("user32.dll")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra); [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd); [DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); [DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr hWnd, int nCmdShow); [DllImport("user32.dll")] public static extern bool SetWindowPos(IntPtr hWnd, IntPtr after, int X, int Y, int cx, int cy, uint flags); [DllImport("user32.dll")] public static extern bool AttachThreadInput(uint idAttach, uint idAttachTo, bool fAttach); [DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint processId); [DllImport("kernel32.dll")] public static extern uint GetCurrentThreadId(); [DllImport("user32.dll")] public static extern int GetSystemMetrics(int nIndex); [DllImport("user32.dll")] public static extern bool EnumWindows(EnumProc cb, IntPtr lp); public delegate bool EnumProc(IntPtr hWnd, IntPtr lp); [DllImport("user32.dll", CharSet = CharSet.Unicode)] public static extern int GetWindowTextW(IntPtr hWnd, System.Text.StringBuilder sb, int max);' -Name RpaUser32Click -Namespace Win32Rpa | Out-Null
|
||||
}
|
||||
@ -129,6 +310,21 @@ function target(args: Record<string, unknown>, node: { id: string; componentId:
|
||||
return { selector: parseUiaSelector(raw), args, node };
|
||||
}
|
||||
|
||||
/** 生成“解析窗口 + 解析元素（路径优先，其次平铺属性）”的 PowerShell 语句，结果写入 $el */
|
||||
function elementResolveLines(s: UiaSelector): string {
|
||||
const parentLine = "$parent = Find-RpaWindow " + psString(s.window ?? "");
|
||||
if (s.path?.length) {
|
||||
const pathB64 = Buffer.from(JSON.stringify(s.path), "utf8").toString("base64");
|
||||
return parentLine + "\n$el = Find-RpaElementByPath $parent '" + pathB64 + "'";
|
||||
}
|
||||
return parentLine
|
||||
+ "\n$el = Find-RpaElement $parent " + psString(s.name ?? "")
|
||||
+ " " + psString(s.id ?? "")
|
||||
+ " " + psString(s.type ?? "")
|
||||
+ " " + psString(s.className ?? "")
|
||||
+ " " + s.index;
|
||||
}
|
||||
|
||||
const SCREEN_USER32 = "Add-Type -MemberDefinition '[DllImport(\"user32.dll\")] public static extern bool SetProcessDpiAwarenessContext(IntPtr value); [DllImport(\"user32.dll\")] public static extern bool SetCursorPos(int X, int Y); [DllImport(\"user32.dll\")] public static extern void mouse_event(uint flags, uint dx, uint dy, uint data, UIntPtr extra);' -Name RpaScreenInput -Namespace RpaScreen | Out-Null; try { [RpaScreen.RpaScreenInput]::SetProcessDpiAwarenessContext([IntPtr](-4)) | Out-Null } catch {}";
|
||||
|
||||
function screenCenter(selector: UiaSelector): { x: number; y: number; w: number; h: number } {
|
||||
@ -204,8 +400,8 @@ $deadline = [DateTime]::UtcNow.AddMilliseconds(${Math.max(0, Math.round(timeoutM
|
||||
$found = $null
|
||||
while ($null -eq $found) {
|
||||
try {
|
||||
$parent = Find-RpaWindow ${psString(s.window ?? "")}
|
||||
$found = Find-RpaElement $parent ${psString(s.name ?? "")} ${psString(s.id ?? "")} ${psString(s.type ?? "")} ${psString(s.className ?? "")} ${s.index}
|
||||
${elementResolveLines(s)}
|
||||
$found = $el
|
||||
} catch {
|
||||
$err = $_.Exception.Message
|
||||
if ($err -like "window_not_found:*") { $found = $null }
|
||||
@ -239,8 +435,7 @@ function clickScript(t: UiaTarget): string {
|
||||
${FIND_HELPER}
|
||||
$el = $null
|
||||
try {
|
||||
$parent = Find-RpaWindow ${psString(s.window ?? "")}
|
||||
$el = Find-RpaElement $parent ${psString(s.name ?? "")} ${psString(s.id ?? "")} ${psString(s.type ?? "")} ${psString(s.className ?? "")} ${s.index}
|
||||
${elementResolveLines(s)}
|
||||
} catch { $el = $null }
|
||||
if ($null -eq $el) { Write-Output "JSON_NOT_FOUND"; exit 0 }
|
||||
$rect = $el.Current.BoundingRectangle
|
||||
@ -335,10 +530,7 @@ export function setValueScript(t: UiaTarget, value: string): string {
|
||||
const s = t.selector;
|
||||
return `${UIA_USING}
|
||||
${FIND_HELPER}
|
||||
try {
|
||||
$parent = Find-RpaWindow ${psString(s.window ?? "")}
|
||||
$el = Find-RpaElement $parent ${psString(s.name ?? "")} ${psString(s.id ?? "")} ${psString(s.type ?? "")} ${psString(s.className ?? "")} ${s.index}
|
||||
} catch { $el = $null }
|
||||
${elementResolveLines(s)}
|
||||
if ($null -eq $el) { Write-Output "JSON_NOT_FOUND"; exit 0 }
|
||||
$value = ${psString(value)}
|
||||
$method = "value_pattern"
|
||||
@ -372,10 +564,7 @@ function getValueScript(t: UiaTarget): string {
|
||||
const s = t.selector;
|
||||
return `${UIA_USING}
|
||||
${FIND_HELPER}
|
||||
try {
|
||||
$parent = Find-RpaWindow ${psString(s.window ?? "")}
|
||||
$el = Find-RpaElement $parent ${psString(s.name ?? "")} ${psString(s.id ?? "")} ${psString(s.type ?? "")} ${psString(s.className ?? "")} ${s.index}
|
||||
} catch { $el = $null }
|
||||
${elementResolveLines(s)}
|
||||
if ($null -eq $el) { Write-Output "JSON_NOT_FOUND"; exit 0 }
|
||||
$value = $el.Current.Name
|
||||
try { $vp = $el.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern); $value = $vp.Value } catch {}
|
||||
@ -426,29 +615,37 @@ function requireFound(result: Record<string, unknown> | null, t: UiaTarget, acti
|
||||
|
||||
const clickGround: CapabilityHandler = async (_ctx, node, args) => {
|
||||
const t = target(args, node);
|
||||
if (t.selector.screen) return runScript(clickScreenScript(t), t);
|
||||
return runWithImplicitWait(t, "click", clickScript(t), runScript, args);
|
||||
};
|
||||
|
||||
const setValueGround: CapabilityHandler = async (_ctx, node, args) => {
|
||||
const t = target(args, node);
|
||||
const value = String(args.value ?? args.text ?? "");
|
||||
if (t.selector.screen) return runScript(setValueScreenScript(t, value), t);
|
||||
return runWithImplicitWait(t, "setValue", setValueScript(t, value), runScript, args);
|
||||
};
|
||||
|
||||
const existsGround: CapabilityHandler = async (_ctx, node, args) => {
|
||||
const t = target(args, node);
|
||||
if (t.selector.screen) return true;
|
||||
const found = await findElement(t, 1000, 150);
|
||||
return found !== null;
|
||||
};
|
||||
|
||||
const getValueGround: CapabilityHandler = async (_ctx, node, args) => {
|
||||
const t = target(args, node);
|
||||
if (t.selector.screen) {
|
||||
const result = await runScript(getValueScreenScript(t), t);
|
||||
return requireFound(result, t, "getValue").value ?? "";
|
||||
}
|
||||
const result = await runWithImplicitWait(t, "getValue", getValueScript(t), runScript, args);
|
||||
return result.value ?? "";
|
||||
};
|
||||
|
||||
const waitElementGround: CapabilityHandler = async (_ctx, node, args) => {
|
||||
const t = target(args, node);
|
||||
if (t.selector.screen) return true;
|
||||
const timeout = Number(args.timeout ?? args.timeoutMs ?? 5000) || 5000;
|
||||
const gap = Number(args.time_gap ?? args.gapMs ?? 200) || 200;
|
||||
const found = await findElement(t, timeout, gap);
|
||||
@ -459,6 +656,39 @@ register("Element_ClickGround", clickGround);
|
||||
register("Element_SetValueGround", setValueGround);
|
||||
register("Element_ExistsGround", existsGround);
|
||||
register("Element_GetValueGround", getValueGround);
|
||||
const restoreWindowGround: CapabilityHandler = async (_ctx, node, args) => {
|
||||
const t = target(args, node);
|
||||
const lines = ["$hwnd = Restore-RpaWindow $el"];
|
||||
if ([args.x, args.y, args.width, args.height].some((value) => value !== undefined && value !== null && String(value).trim() !== "")) {
|
||||
const x = Math.round(Number(args.x ?? 60)) || 0;
|
||||
const y = Math.round(Number(args.y ?? 60)) || 0;
|
||||
const width = Math.round(Number(args.width ?? 0)) || 0;
|
||||
const height = Math.round(Number(args.height ?? 0)) || 0;
|
||||
lines.push("$hwnd = Move-RpaWindow $el " + x + " " + y + " " + width + " " + height);
|
||||
}
|
||||
return runWithImplicitWait(t, "restoreWindow", propertyValue([...lines, "Write-Output (@{ hwnd = [string]$hwnd; name = $el.Current.Name } | ConvertTo-Json -Compress)"], t), runScript, args);
|
||||
};
|
||||
|
||||
const activateWindowGround: CapabilityHandler = async (_ctx, node, args) => {
|
||||
const t = target(args, node);
|
||||
return runWithImplicitWait(t, "activateWindow", propertyValue([
|
||||
"$hwnd = Activate-RpaWindow $el",
|
||||
"Write-Output (@{ hwnd = [string]$hwnd; name = $el.Current.Name } | ConvertTo-Json -Compress)"
|
||||
], t), runScript, args);
|
||||
};
|
||||
|
||||
const moveWindowGround: CapabilityHandler = async (_ctx, node, args) => {
|
||||
const t = target(args, node);
|
||||
const x = Math.round(Number(args.x ?? 60)) || 0;
|
||||
const y = Math.round(Number(args.y ?? 60)) || 0;
|
||||
const width = Math.round(Number(args.width ?? 0)) || 0;
|
||||
const height = Math.round(Number(args.height ?? 0)) || 0;
|
||||
return runWithImplicitWait(t, "moveWindow", propertyValue([
|
||||
"$hwnd = Move-RpaWindow $el " + x + " " + y + " " + width + " " + height,
|
||||
"Write-Output (@{ hwnd = [string]$hwnd; name = $el.Current.Name } | ConvertTo-Json -Compress)"
|
||||
], t), runScript, args);
|
||||
};
|
||||
|
||||
register("Element_WaitElementGround", waitElementGround);
|
||||
|
||||
|
||||
@ -486,8 +716,7 @@ function propertyValue(scriptLines: string[], t: UiaTarget): string {
|
||||
${FIND_HELPER}
|
||||
$el = $null
|
||||
try {
|
||||
$parent = Find-RpaWindow ${psString(s.window ?? "")}
|
||||
$el = Find-RpaElement $parent ${psString(s.name ?? "")} ${psString(s.id ?? "")} ${psString(s.type ?? "")} ${psString(s.className ?? "")} ${s.index}
|
||||
${elementResolveLines(s)}
|
||||
} catch { $el = $null }
|
||||
if ($null -eq $el) { Write-Output "JSON_NOT_FOUND"; exit 0 }
|
||||
${scriptLines.join("\n")}`;
|
||||
@ -709,3 +938,6 @@ register("Element_MouseToGround", mouseToGround);
|
||||
register("Element_ScrollIntoViewGround", scrollIntoViewGround);
|
||||
register("Element_ChildrenGround", childrenGroundHandler(false));
|
||||
register("Element_DesktopChildren", childrenGroundHandler(true));
|
||||
register("Element_RestoreWindowGround", restoreWindowGround);
|
||||
register("Element_ActivateWindowGround", activateWindowGround);
|
||||
register("Element_MoveWindowGround", moveWindowGround);
|
||||
|
||||
@ -24,6 +24,7 @@ import "./word";
|
||||
import "./word-extended";
|
||||
import "./message";
|
||||
import "./vision";
|
||||
import "./cv-vision";
|
||||
import "./extended";
|
||||
import "./system-extended";
|
||||
import "./office-file-extended";
|
||||
|
||||
@ -3,9 +3,9 @@ import { promisify } from "node:util";
|
||||
import fs from "node:fs";
|
||||
|
||||
import path from "node:path";
|
||||
import { app, BrowserWindow, screen } from "electron";
|
||||
import { app, BrowserWindow, desktopCapturer, globalShortcut, screen } from "electron";
|
||||
import { RuntimeError } from "./engine/types";
|
||||
import { chatCompletion, requireAiConfig } from "./capabilities/ai-service";
|
||||
import { detectUiElements, localUiModelPath } from "./local-vision/ui-detect-service";
|
||||
|
||||
const run = promisify(execFile);
|
||||
|
||||
@ -58,6 +58,8 @@ interface UiaPickOutput {
|
||||
}
|
||||
|
||||
interface AiCandidate {
|
||||
/** 候选稳定身份：覆盖层排序后仍按 ID 回填，禁止用排序后的数组下标当身份 */
|
||||
candidateId: string;
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
@ -92,12 +94,10 @@ function safeSelectorText(value: string | undefined): string {
|
||||
return String(value || "").replace(/[;\r\n]+/g, " ").trim();
|
||||
}
|
||||
|
||||
export async function pickDesktopElement(options: { ai?: boolean } = {}): Promise<DesktopPickResult> {
|
||||
if (options.ai) requireAiConfig({ vision: true });
|
||||
export async function pickDesktopElement(): Promise<DesktopPickResult> {
|
||||
const appWindows = BrowserWindow.getAllWindows().filter(w => !w.isDestroyed());
|
||||
for (const w of appWindows) w.minimize();
|
||||
try {
|
||||
if (options.ai) return await runAiPick();
|
||||
const { stdout } = await run(
|
||||
"powershell.exe",
|
||||
["-NoProfile", "-STA", "-ExecutionPolicy", "Bypass", "-File", pickerScriptPath()],
|
||||
@ -109,7 +109,9 @@ export async function pickDesktopElement(options: { ai?: boolean } = {}): Promis
|
||||
.map(value => Number(value))
|
||||
.every(value => Number.isFinite(value)) && Number(parsed.paneW) > 4 && Number(parsed.paneH) > 4;
|
||||
if (parsed.aiRequested && paneAiReady) return await runPaneAiPick(parsed);
|
||||
if (parsed.aiRequested) return await runAiPick(parsed);
|
||||
if (parsed.aiRequested) {
|
||||
throw new RuntimeError("ai_pane_invalid", "AI 兜底识别仅支持包含有效区域的 Pane");
|
||||
}
|
||||
const parts: string[] = [];
|
||||
if (parsed.window) parts.push("window=" + parsed.window);
|
||||
if (parsed.name) parts.push("name=" + parsed.name);
|
||||
@ -125,68 +127,121 @@ export async function pickDesktopElement(options: { ai?: boolean } = {}): Promis
|
||||
|
||||
function paneAiOverlayHtml(): string {
|
||||
return `<!DOCTYPE html><html><head><meta charset='utf-8'><style>
|
||||
html,body{margin:0;width:100%;height:100%;overflow:hidden;background:transparent;cursor:default;font-family:'Microsoft YaHei',Arial,sans-serif}
|
||||
#frame{position:absolute;inset:0;display:none;border:1px solid rgba(56,189,248,.30);animation:framePulse 1.9s ease-in-out infinite;z-index:4}#frame i{position:absolute;width:30px;height:30px;border:3px solid #38bdf8;filter:drop-shadow(0 0 7px rgba(56,189,248,.85))}#frame i:nth-child(1){left:2px;top:2px;border-right:0;border-bottom:0;border-radius:10px 0 0 0}#frame i:nth-child(2){right:2px;top:2px;border-left:0;border-bottom:0;border-radius:0 10px 0 0}#frame i:nth-child(3){left:2px;bottom:2px;border-right:0;border-top:0;border-radius:0 0 0 10px}#frame i:nth-child(4){right:2px;bottom:2px;border-left:0;border-top:0;border-radius:0 0 10px 0}
|
||||
#scan{position:absolute;left:0;right:0;top:-20%;height:19%;display:none;background:linear-gradient(to bottom,rgba(56,189,248,0),rgba(56,189,248,.06) 48%,rgba(125,211,252,.20) 82%,rgba(224,242,254,.16));z-index:5;animation:scan 1.65s linear infinite;will-change:top}#scan b{position:absolute;left:3%;right:3%;bottom:-2px;height:3px;border-radius:999px;background:linear-gradient(90deg,rgba(56,189,248,0),#7dd3fc 14%,#fff 50%,#7dd3fc 86%,rgba(56,189,248,0));box-shadow:0 0 10px #38bdf8,0 0 28px rgba(56,189,248,.78),0 0 54px rgba(56,189,248,.38)}#scan b:after{content:'';position:absolute;left:0;right:0;top:2px;height:9px;background:linear-gradient(to bottom,rgba(56,189,248,.18),rgba(56,189,248,0))}
|
||||
#tip{position:absolute;left:50%;top:18px;transform:translateX(-50%);display:none;padding:8px 15px;border-radius:999px;color:#f8fafc;background:rgba(2,20,40,.78);border:1px solid rgba(125,211,252,.42);box-shadow:0 8px 28px rgba(0,0,0,.28),inset 0 1px 0 rgba(255,255,255,.09);font-size:13px;line-height:1;white-space:nowrap;z-index:2147483647}#tip:before{content:'';display:inline-block;width:7px;height:7px;margin-right:8px;border-radius:50%;background:#38bdf8;box-shadow:0 0 8px #38bdf8;vertical-align:1px}body.scanning #tip:before{animation:tipPulse 1.1s ease-in-out infinite}
|
||||
.candidate{position:absolute;border:1px solid rgba(255,255,255,.28);background:transparent;cursor:pointer;box-sizing:border-box}
|
||||
.candidate:hover,.candidate.active{border:3px solid #ff2d2d;box-shadow:0 0 0 1px rgba(255,255,255,.45);z-index:2147483646}
|
||||
.badge{position:absolute;display:none;padding:3px 7px;border-radius:5px;color:#fff;background:rgba(190,18,18,.95);border:1px solid rgba(255,220,220,.82);font-size:12px;line-height:16px;white-space:nowrap;pointer-events:none;box-shadow:0 4px 12px rgba(0,0,0,.28);z-index:2147483647}.badge.active{display:block}
|
||||
@keyframes scan{from{top:-20%}to{top:100%}}@keyframes framePulse{0%,100%{border-color:rgba(56,189,248,.24)}50%{border-color:rgba(56,189,248,.58)}}@keyframes tipPulse{0%,100%{opacity:.45}50%{opacity:1}}
|
||||
html,body{margin:0;width:100%;height:100%;overflow:hidden;background:transparent;font-family:'Microsoft YaHei',Arial,sans-serif}
|
||||
#veil{position:absolute;inset:0;background:rgba(2,10,28,.035);opacity:0;transition:opacity .18s ease;z-index:3;pointer-events:none}
|
||||
body.scanning #veil{opacity:1}
|
||||
#paneFrame{position:absolute;inset:0;border:2px solid #ff3b30;box-shadow:0 0 0 1px rgba(255,255,255,.34),0 0 22px rgba(255,59,48,.24);z-index:4;pointer-events:none}
|
||||
#aiFrame{position:absolute;inset:6px;border:1px solid rgba(56,189,248,.26);border-radius:12px;animation:aiPulse 2s ease-in-out infinite;z-index:5;pointer-events:none}
|
||||
#aiFrame i{position:absolute;width:22px;height:22px;border:2px solid #38bdf8;filter:drop-shadow(0 0 6px rgba(56,189,248,.72))}
|
||||
#aiFrame i:nth-child(1){left:-1px;top:-1px;border-right:0;border-bottom:0;border-radius:10px 0 0 0}
|
||||
#aiFrame i:nth-child(2){right:-1px;top:-1px;border-left:0;border-bottom:0;border-radius:0 10px 0 0}
|
||||
#aiFrame i:nth-child(3){left:-1px;bottom:-1px;border-right:0;border-top:0;border-radius:0 0 0 10px}
|
||||
#aiFrame i:nth-child(4){right:-1px;bottom:-1px;border-left:0;border-top:0;border-radius:0 0 10px 0}
|
||||
#scan{position:absolute;left:0;right:0;top:0;height:1px;display:none;background:linear-gradient(90deg,rgba(56,189,248,0),rgba(125,211,252,.72) 16%,rgba(255,255,255,.94) 50%,rgba(125,211,252,.72) 84%,rgba(56,189,248,0));box-shadow:0 0 10px rgba(56,189,248,.78);animation:scan 1.35s cubic-bezier(.4,0,.6,1) infinite;z-index:6;pointer-events:none}
|
||||
#scan:before{content:'';position:absolute;left:0;right:0;top:1px;height:58px;background:linear-gradient(to bottom,rgba(56,189,248,.13),rgba(56,189,248,0))}
|
||||
#tip{position:absolute;left:50%;top:14px;transform:translateX(-50%);display:none;min-height:30px;padding:0 13px;border-radius:16px;color:#f8fafc;background:rgba(4,18,34,.84);border:1px solid rgba(125,211,252,.34);box-shadow:0 8px 24px rgba(0,0,0,.24),inset 0 1px 0 rgba(255,255,255,.06);font-size:12.5px;line-height:28px;white-space:nowrap;z-index:2147483647;pointer-events:none}
|
||||
#tip:before{content:'';display:inline-block;width:6px;height:6px;margin-right:8px;border-radius:50%;background:#38bdf8;box-shadow:0 0 8px #38bdf8;vertical-align:2px}
|
||||
body.scanning #tip:before{animation:tipPulse 1.15s ease-in-out infinite}
|
||||
body.debug .candidate{border:1px solid rgba(255,59,48,.74)}
|
||||
.candidate{position:absolute;background:transparent;cursor:pointer;box-sizing:border-box;z-index:100}
|
||||
.candidate{border:3px solid transparent}.candidate.active{border-color:#ff3b30;box-shadow:0 0 0 1px rgba(255,255,255,.38);z-index:101}
|
||||
.badge{position:absolute;display:none;padding:3px 7px;border-radius:5px;color:#fff;background:rgba(190,18,18,.94);border:1px solid rgba(255,220,220,.78);font-size:12px;line-height:16px;white-space:nowrap;pointer-events:none;box-shadow:0 4px 12px rgba(0,0,0,.24);z-index:2147483647}
|
||||
.badge.active{display:block;z-index:102}
|
||||
@keyframes scan{from{top:0}to{top:100%}}
|
||||
@keyframes aiPulse{0%,100%{border-color:rgba(56,189,248,.18)}50%{border-color:rgba(56,189,248,.46)}}
|
||||
@keyframes tipPulse{0%,100%{opacity:.4}50%{opacity:1}}
|
||||
</style></head><body>
|
||||
<div id=frame><i></i><i></i><i></i><i></i></div><div id=scan><b></b></div><div id=tip>AI 识别中…</div>
|
||||
<div id=veil></div><div id=paneFrame></div><div id=aiFrame><i></i><i></i><i></i><i></i></div><div id=scan></div><div id=tip>AI 识别中…</div>
|
||||
<script>
|
||||
var paneW=1,paneH=1;
|
||||
function startScan(label){paneW=window.__paneW||document.documentElement.clientWidth;paneH=window.__paneH||document.documentElement.clientHeight;document.getElementById('frame').style.display='block';document.getElementById('scan').style.display='block';document.getElementById('tip').style.display='block';document.getElementById('tip').textContent=label||'AI 识别中…';document.body.classList.add('scanning')}
|
||||
var paneW=1,paneH=1,lastMouse=null,lastCid=null;
|
||||
function showStatus(label){document.getElementById('tip').style.display='block';document.getElementById('tip').textContent=label||'AI 识别中…';}
|
||||
function startScan(label){
|
||||
paneW=window.__paneW||document.documentElement.clientWidth;paneH=window.__paneH||document.documentElement.clientHeight;
|
||||
document.body.classList.add('scanning');
|
||||
document.getElementById('scan').style.display='block';
|
||||
document.getElementById('aiFrame').style.display='block';
|
||||
showStatus(label||'AI 识别中…');
|
||||
}
|
||||
function clearActive(){
|
||||
if(!lastCid)return;
|
||||
Array.prototype.forEach.call(document.querySelectorAll('.candidate.active,.badge.active'),function(n){n.classList.remove('active')});
|
||||
lastCid=null;window.__hoverCid=null;window.__hoverCandidate=null;
|
||||
if(window.__candidatesReady)logHoverState('clear');
|
||||
}
|
||||
function activateBox(box){
|
||||
var cid=box.dataset.cid;
|
||||
if(lastCid===cid)return;
|
||||
clearActive();
|
||||
var index=Number(box.dataset.index),label=document.getElementById('badge-'+index);
|
||||
box.classList.add('active');if(label)label.classList.add('active');
|
||||
lastCid=cid;window.__hoverCid=cid;window.__hoverCandidate=window.__candidateMap[cid];
|
||||
logHoverState('active');
|
||||
}
|
||||
function syncHover(){
|
||||
if(!window.__candidatesReady)return;
|
||||
var event=lastMouse;if(!event){clearActive();return}
|
||||
// 当前框未离开前保持选中，避免重叠候选因 1px 抖动来回抢高亮。
|
||||
var current=lastCid?document.querySelector('.candidate[data-cid="'+lastCid+'"]'):null;
|
||||
if(current){
|
||||
var r=current.getBoundingClientRect();
|
||||
if(event.clientX>=r.left-2&&event.clientX<=r.right+2&&event.clientY>=r.top-2&&event.clientY<=r.bottom+2)return;
|
||||
}
|
||||
var hit=null,hitArea=Infinity;
|
||||
Array.prototype.forEach.call(document.querySelectorAll('.candidate'),function(box){
|
||||
var rect=box.getBoundingClientRect();
|
||||
if(event.clientX<rect.left||event.clientX>rect.right||event.clientY<rect.top||event.clientY>rect.bottom)return;
|
||||
var area=rect.width*rect.height;
|
||||
if(area<hitArea){hitArea=area;hit=box}
|
||||
});
|
||||
if(hit)activateBox(hit);else clearActive();
|
||||
}
|
||||
function logHoverState(state){
|
||||
console.log('pane-ai hover ' + state + ' mouse=' + (lastMouse?lastMouse.clientX+','+lastMouse.clientY:'none') +
|
||||
' cid=' + (lastCid||'none') + ' candidates=' + document.querySelectorAll('.candidate').length);
|
||||
}
|
||||
window.__toggleDebug=function(){
|
||||
document.body.classList.toggle('debug');
|
||||
var debug=document.body.classList.contains('debug');
|
||||
showStatus(debug?'开发模式：F8 关闭候选框':(window.__candidatesReady?'AI 完成：移动高亮；单击选中，再次单击或空格回填；Esc 取消':'开发模式已开启，识别完成后显示全部候选框'));
|
||||
};
|
||||
function showCandidates(items){
|
||||
paneW=window.__paneW||document.documentElement.clientWidth;paneH=window.__paneH||document.documentElement.clientHeight;
|
||||
document.getElementById('scan').style.display='none';document.getElementById('frame').style.display='none';document.getElementById('tip').style.display='block';
|
||||
document.getElementById('tip').textContent='AI 识别完成：移动鼠标高亮；单击选中，再次单击或空格回填；Esc 取消';
|
||||
document.getElementById('scan').style.display='none';
|
||||
document.getElementById('aiFrame').style.display='none';
|
||||
document.body.classList.remove('scanning');document.body.style.cursor='pointer';
|
||||
window.__hover=null;window.__selected=null;window.__confirmed=null;window.__readyAt=Date.now()+600;
|
||||
items.sort(function(a,b){return a.width*a.height-b.width*b.height}).forEach(function(c,index){
|
||||
var box=document.createElement('div');box.className='candidate';box.__index=index;
|
||||
window.__candidateMap={};window.__candidatesReady=false;clearActive();
|
||||
items.sort(function(a,b){return a.y-b.y||a.x-b.x}).forEach(function(c,index){
|
||||
window.__candidateMap[c.candidateId]=c;
|
||||
var box=document.createElement('div');box.className='candidate';box.dataset.cid=c.candidateId;box.dataset.index=index;
|
||||
box.style.left=(c.x/paneW*100)+'%';box.style.top=(c.y/paneH*100)+'%';
|
||||
box.style.width=(c.width/paneW*100)+'%';box.style.height=(c.height/paneH*100)+'%';
|
||||
function activate(){
|
||||
Array.prototype.forEach.call(document.querySelectorAll('.candidate,.badge'),function(n){n.classList.remove('active')});
|
||||
box.classList.add('active');var label=document.getElementById('badge-'+index);if(label)label.classList.add('active');
|
||||
window.__hover=index;window.__hoverCandidate=c;
|
||||
}
|
||||
window.confirmCandidate=function(index,c){
|
||||
var tip=document.getElementById('tip');
|
||||
tip.textContent='已选中：'+(c.name||'控件')+'；再次单击或空格回填，Esc 取消';
|
||||
window.__confirmed=index;window.__selected=index;
|
||||
};
|
||||
function deactivate(){
|
||||
box.classList.remove('active');var label=document.getElementById('badge-'+index);if(label)label.classList.remove('active');
|
||||
if(window.__hover===index){window.__hover=null;window.__hoverCandidate=null}
|
||||
}
|
||||
box.addEventListener('mouseenter',activate);box.addEventListener('mousemove',activate);box.addEventListener('mouseleave',deactivate);
|
||||
box.addEventListener('mousedown',function(e){
|
||||
e.preventDefault();e.stopPropagation();activate();
|
||||
if(Date.now()<(window.__readyAt||0)){console.log('pane-ai ignore early click',index);return}
|
||||
if(window.__confirmed!==index){console.log('pane-ai first select',index);window.confirmCandidate(index,c);return}
|
||||
console.log('pane-ai confirm click',index);window.__selected=index;window.__done=true;
|
||||
var label=document.createElement('div');label.className='badge';label.id='badge-'+index;label.textContent='AI识别 · '+(c.controlType||'控件');
|
||||
label.style.left=(c.x/paneW*100)+'%';label.style.top=(Math.max(0,c.y-24)/paneH*100)+'%';
|
||||
box.addEventListener('click',function(event){
|
||||
event.stopPropagation();if(Date.now()<window.__readyAt)return;
|
||||
activateBox(box);
|
||||
if(window.__selectedCid!==c.candidateId){window.__selectedCid=c.candidateId;showStatus('已选中：'+(c.controlType||'控件')+'；再次单击或空格回填');return}
|
||||
window.__confirmedCid=c.candidateId;window.__done=true;
|
||||
});
|
||||
document.body.appendChild(box);
|
||||
var yTop=c.y/paneH*100,yBottom=(c.y+c.height)/paneH*100,leftPct=Math.max(0,Math.min(72,c.x/paneW*100));
|
||||
var badge=document.createElement('div');badge.id='badge-'+index;badge.className='badge';
|
||||
badge.textContent='AI识别 · '+(c.name||'控件');
|
||||
badge.style.left=leftPct+'%';
|
||||
badge.style.top=(yTop<9?'calc('+yBottom+'% + 8px)':'calc('+yTop+'% - 32px)');
|
||||
document.body.appendChild(badge);
|
||||
document.body.appendChild(box);document.body.appendChild(label);
|
||||
});
|
||||
window.__candidatesReady=true;window.__readyAt=Date.now()+450;
|
||||
showStatus(document.body.classList.contains('debug')?'开发模式：F8 关闭候选框':'AI 完成：移动高亮；单击选中，再次单击或空格回填；Esc 取消');
|
||||
syncHover();
|
||||
}
|
||||
document.addEventListener('keydown',function(e){
|
||||
if(e.key==='Escape'){window.__cancelled=true}
|
||||
if(e.key===' '&&window.__hoverCandidate&&!window.__done){
|
||||
e.preventDefault();
|
||||
if(Date.now()<(window.__readyAt||0)){console.log('pane-ai ignore early space');return}
|
||||
if(window.__confirmed!==window.__hover){window.confirmCandidate(window.__hover,window.__hoverCandidate);return}
|
||||
window.__selected=window.__hover;window.__done=true;
|
||||
document.addEventListener('pointermove',function(event){lastMouse=event;syncHover()});
|
||||
document.addEventListener('keydown',function(event){
|
||||
if(event.key==='F8'||event.code==='F8'){event.preventDefault();window.__toggleDebug();return}
|
||||
if(event.key==='Escape'){window.__cancelled=true;return}
|
||||
if(event.key===' '||event.code==='Space'){event.preventDefault();
|
||||
if(window.__confirmedCid!==window.__hoverCid){window.__confirmedCid=window.__hoverCid;window.__done=true}
|
||||
}
|
||||
});
|
||||
document.addEventListener('mousedown',function(event){
|
||||
var candidate=event.target&&event.target.classList&&event.target.classList.contains('candidate');
|
||||
if(candidate||Date.now()<(window.__readyAt||0))return;
|
||||
if(window.__selectedCid&&window.__hoverCid===window.__selectedCid){window.__confirmedCid=window.__hoverCid;window.__done=true}
|
||||
});
|
||||
</script></body></html>`;
|
||||
}
|
||||
|
||||
@ -236,160 +291,168 @@ async function createPaneAiOverlay(rect: { x: number; y: number; w: number; h: n
|
||||
return win;
|
||||
}
|
||||
|
||||
async function capturePaneForAi(rect: { x: number; y: number; w: number; h: number }): Promise<AiCaptureOutput> {
|
||||
const outDir = path.join(app.getPath("userData"), "ai-picks");
|
||||
fs.mkdirSync(outDir, { recursive: true });
|
||||
const imagePath = path.join(outDir, "ai-pane-" + Date.now() + ".png");
|
||||
const { stdout } = await run(
|
||||
"powershell.exe",
|
||||
[
|
||||
"-NoProfile", "-STA", "-ExecutionPolicy", "Bypass", "-File", pickerScriptPath(),
|
||||
"-Mode", "capture", "-CaptureExact",
|
||||
"-CaptureX", String(rect.x), "-CaptureY", String(rect.y),
|
||||
"-CaptureWidth", String(rect.w), "-CaptureHeight", String(rect.h),
|
||||
"-CapturePath", imagePath
|
||||
],
|
||||
{ timeout: 30000, windowsHide: true, maxBuffer: 2 * 1024 * 1024 }
|
||||
);
|
||||
const parsed = jsonLine<AiCaptureOutput>(stdout, "{");
|
||||
if (!parsed.imagePath || !fs.existsSync(parsed.imagePath)) {
|
||||
throw new RuntimeError("ai_capture_failed", "AI 拾取截图失败");
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
async function capturePaneForAi(rect: { x: number; y: number; w: number; h: number }, overlay: BrowserWindow): Promise<AiCaptureOutput> {
|
||||
const centerDip = screen.screenToDipPoint({ x: rect.x + rect.w / 2, y: rect.y + rect.h / 2 });
|
||||
const display = displayForPoint(centerDip.x, centerDip.y);
|
||||
const scale = display.scaleFactor || 1;
|
||||
const displayPhysicalX = Math.round(display.bounds.x * scale);
|
||||
const displayPhysicalY = Math.round(display.bounds.y * scale);
|
||||
// UIA/Desktop Picker 返回的是物理像素坐标；截图裁剪也必须直接使用物理像素。
|
||||
// 这里不能把 cropX/cropY 再乘 actualScale，否则会重复缩放并导致候选框整体偏移。
|
||||
const cropX = rect.x - displayPhysicalX;
|
||||
const cropY = rect.y - displayPhysicalY;
|
||||
const thumbnailWidth = Math.round(display.bounds.width * scale);
|
||||
const thumbnailHeight = Math.round(display.bounds.height * scale);
|
||||
|
||||
function parseAiCandidates(text: string, cropW: number, cropH: number): AiCandidate[] {
|
||||
const fence = String.fromCharCode(96);
|
||||
const fencePattern = new RegExp(fence + "{3}(?:json)?" + fence + "{3}", "gi");
|
||||
const cleaned = text.replace(fencePattern, "").trim();
|
||||
const start = cleaned.indexOf("{");
|
||||
const arrayStart = cleaned.indexOf("[");
|
||||
const objectStart = start >= 0 && (start < arrayStart || arrayStart < 0) ? start : arrayStart;
|
||||
const end = Math.max(cleaned.lastIndexOf("}"), cleaned.lastIndexOf("]"));
|
||||
if (objectStart < 0 || end <= objectStart) throw new RuntimeError("ai_response_invalid", "AI 视觉模型没有返回 JSON");
|
||||
const raw = JSON.parse(cleaned.slice(objectStart, end + 1)) as Record<string, unknown>;
|
||||
const list = Array.isArray(raw) ? raw : Array.isArray(raw.candidates) ? raw.candidates : [];
|
||||
const result: AiCandidate[] = [];
|
||||
for (const item of list.slice(0, 40)) {
|
||||
const record = (item || {}) as Record<string, unknown>;
|
||||
const num = (...keys: string[]): number => {
|
||||
for (const key of keys) {
|
||||
const value = Number(record[key]);
|
||||
if (Number.isFinite(value)) return value;
|
||||
}
|
||||
return NaN;
|
||||
};
|
||||
let x = num("x", "left", "x1");
|
||||
let y = num("y", "top", "y1");
|
||||
let width = num("width", "w");
|
||||
let height = num("height", "h");
|
||||
const x2 = num("x2", "right");
|
||||
const y2 = num("y2", "bottom");
|
||||
if (!Number.isFinite(width) && Number.isFinite(x2)) width = x2 - x;
|
||||
if (!Number.isFinite(height) && Number.isFinite(y2)) height = y2 - y;
|
||||
if (![x, y, width, height].every(Number.isFinite) || width < 6 || height < 6) continue;
|
||||
x = Math.max(0, Math.min(cropW - width, x));
|
||||
y = Math.max(0, Math.min(cropH - height, y));
|
||||
width = Math.min(width, cropW - x);
|
||||
height = Math.min(height, cropH - y);
|
||||
if (width * height >= cropW * cropH * 0.97) continue;
|
||||
const confidence = Number(record.confidence);
|
||||
result.push({
|
||||
x: Math.round(x), y: Math.round(y), width: Math.round(width), height: Math.round(height),
|
||||
name: String(record.name ?? "AI识别控件").slice(0, 80) || "AI识别控件",
|
||||
controlType: String(record.controlType ?? "Control").slice(0, 40) || "Control",
|
||||
confidence: Number.isFinite(confidence) ? Math.max(0, Math.min(1, confidence)) : 0.72
|
||||
// 只为避开拾取窗口本身；推理和动画继续在后台进行。
|
||||
overlay.hide();
|
||||
await new Promise(resolve => setTimeout(resolve, 60));
|
||||
try {
|
||||
const sources = await desktopCapturer.getSources({
|
||||
types: ["screen"],
|
||||
thumbnailSize: { width: thumbnailWidth, height: thumbnailHeight }
|
||||
});
|
||||
overlay.show();
|
||||
const source = sources.find(item => item.display_id === String(display.id)) || sources[0];
|
||||
if (!source || source.thumbnail.isEmpty()) {
|
||||
throw new RuntimeError("ai_capture_failed", "AI 拾取截图失败");
|
||||
}
|
||||
|
||||
const thumbnail = source.thumbnail;
|
||||
const actualScale = thumbnail.getSize().width / Math.max(1, display.bounds.width);
|
||||
const x = Math.max(0, Math.min(thumbnail.getSize().width - 1, cropX));
|
||||
const y = Math.max(0, Math.min(thumbnail.getSize().height - 1, cropY));
|
||||
const width = Math.max(1, Math.min(thumbnail.getSize().width - x, Math.round(rect.w * actualScale)));
|
||||
const height = Math.max(1, Math.min(thumbnail.getSize().height - y, Math.round(rect.h * actualScale)));
|
||||
const cropped = thumbnail.crop({ x, y, width, height });
|
||||
const image = cropped.toPNG();
|
||||
const outDir = path.join(app.getPath("userData"), "ai-picks");
|
||||
fs.mkdirSync(outDir, { recursive: true });
|
||||
const imagePath = path.join(outDir, `ai-pane-${Date.now()}.png`);
|
||||
await fs.promises.writeFile(imagePath, image);
|
||||
return {
|
||||
cursorX: rect.x,
|
||||
cursorY: rect.y,
|
||||
cropX: rect.x,
|
||||
cropY: rect.y,
|
||||
cropW: width,
|
||||
cropH: height,
|
||||
markerX: 0,
|
||||
markerY: 0,
|
||||
imagePath
|
||||
};
|
||||
} catch (error) {
|
||||
overlay.show();
|
||||
throw error;
|
||||
}
|
||||
if (!result.length) throw new RuntimeError("ai_response_invalid", "AI 没有识别到可用候选区域");
|
||||
return result;
|
||||
}
|
||||
|
||||
async function runPaneAiPick(context: UiaPickOutput): Promise<DesktopPickResult> {
|
||||
requireAiConfig({ vision: true });
|
||||
// 本地小模型是 Pane AI 的主链路；这里不要求用户配置远程 VLM。
|
||||
const localModelPath = localUiModelPath();
|
||||
console.log("[pane-ai] local model", localModelPath);
|
||||
const pane = paneAiRect(context);
|
||||
const win = await createPaneAiOverlay(pane);
|
||||
const startedAt = Date.now();
|
||||
// F8 用全局快捷键，避免键盘焦点停在目标应用时 overlay 收不到按键。
|
||||
const debugHotkeyRegistered = globalShortcut.register("F8", () => {
|
||||
void win.webContents.executeJavaScript("window.__toggleDebug && window.__toggleDebug()", true).catch(() => {});
|
||||
});
|
||||
|
||||
try {
|
||||
// Let the visible scan animation complete over the actual Pane before taking the screenshot.
|
||||
await new Promise(resolve => setTimeout(resolve, 950));
|
||||
win.hide();
|
||||
await new Promise(resolve => setTimeout(resolve, 240));
|
||||
const capture = await capturePaneForAi(pane);
|
||||
win.show();
|
||||
win.focus();
|
||||
await win.webContents.executeJavaScript("startScan('AI 理解中…');");
|
||||
// 稍等 overlay 完成首轮绘制，但不再等一整套“表演动画”。
|
||||
await new Promise(resolve => setTimeout(resolve, 350));
|
||||
const detection = capturePaneForAi(pane, win)
|
||||
.then(async capture => {
|
||||
const image = await fs.promises.readFile(capture.imagePath);
|
||||
const inferenceStarted = Date.now();
|
||||
const boxes = await detectUiElements({ image, width: capture.cropW, height: capture.cropH });
|
||||
console.log("[pane-ai] local inference", boxes.length, Date.now() - inferenceStarted + "ms");
|
||||
return { capture, boxes };
|
||||
});
|
||||
|
||||
const image = await fs.promises.readFile(capture.imagePath);
|
||||
let aiSettled = false;
|
||||
const aiRequest = chatCompletion([
|
||||
{
|
||||
role: "system",
|
||||
content: "你是桌面 RPA 视觉控件分析器。只输出 JSON，不输出解释或 Markdown。"
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: [
|
||||
"这是目标 Pane 区域的截图，尺寸 " + capture.cropW + "x" + capture.cropH + " 像素。",
|
||||
"请把这个 Pane 里的可用控件/内容块拆分成多个候选矩形，最多返回 30 个，按从上到下、从左到右排序。",
|
||||
"优先识别按钮、输入框、列表行、卡片、菜单项、链接和明显可操作区域；不要把整个 Pane 当作候选。",
|
||||
"只返回 JSON：{\"candidates\":[{\"x\":number,\"y\":number,\"width\":number,\"height\":number,\"name\":string,\"controlType\":string,\"confidence\":number}]}",
|
||||
"坐标全部是相对这张截图的整数像素坐标；x/y 是左上角，width/height 也是像素。"
|
||||
].join("\n")
|
||||
},
|
||||
{ type: "image_url", image_url: { url: "data:image/png;base64," + image.toString("base64") } }
|
||||
]
|
||||
let settled = false;
|
||||
const tickStatus = async () => {
|
||||
const elapsed = Date.now() - startedAt;
|
||||
const label = elapsed < 2600
|
||||
? "AI 识别中…"
|
||||
: elapsed < 6000
|
||||
? "AI 理解中…"
|
||||
: "正在整理候选区域…";
|
||||
try {
|
||||
await win.webContents.executeJavaScript("showStatus(" + JSON.stringify(label) + ");", true);
|
||||
} catch {
|
||||
// Overlay 销毁时状态更新可以安全忽略。
|
||||
}
|
||||
], { vision: true, temperature: 0 }).finally(() => { aiSettled = true; });
|
||||
};
|
||||
const statusTimer = setInterval(() => void tickStatus(), 1500);
|
||||
|
||||
const cancelWatcher = (async () => {
|
||||
while (!aiSettled) {
|
||||
while (!settled) {
|
||||
try {
|
||||
const state = await win.webContents.executeJavaScript("JSON.stringify({cancelled:window.__cancelled===true})", true);
|
||||
if (JSON.parse(state).cancelled) throw new RuntimeError("pick_cancelled", "已取消 AI 拾取");
|
||||
} catch (error) {
|
||||
if (error instanceof RuntimeError && error.code === "pick_cancelled") throw error;
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
await new Promise(resolve => setTimeout(resolve, 80));
|
||||
}
|
||||
})();
|
||||
|
||||
let aiText: string;
|
||||
let detected: { capture: AiCaptureOutput; boxes: Awaited<ReturnType<typeof detectUiElements>> };
|
||||
try {
|
||||
aiText = await Promise.race([aiRequest, cancelWatcher]) as string;
|
||||
detected = await Promise.race([detection, cancelWatcher]) as { capture: AiCaptureOutput; boxes: Awaited<ReturnType<typeof detectUiElements>> };
|
||||
} finally {
|
||||
aiSettled = true;
|
||||
settled = true;
|
||||
clearInterval(statusTimer);
|
||||
}
|
||||
|
||||
const candidates: AiCandidate[] = detected.boxes.map((box, index) => ({
|
||||
candidateId: "local-" + index,
|
||||
x: box.x,
|
||||
y: box.y,
|
||||
width: box.width,
|
||||
height: box.height,
|
||||
name: box.category,
|
||||
controlType: box.category,
|
||||
confidence: box.confidence
|
||||
}));
|
||||
if (!candidates.length) {
|
||||
throw new RuntimeError("ai_response_invalid", "本地 UI 模型没有识别到可用候选区域");
|
||||
}
|
||||
|
||||
const candidates = parseAiCandidates(aiText, capture.cropW, capture.cropH);
|
||||
const payload = JSON.stringify(candidates);
|
||||
console.log("[pane-ai] candidates", candidates.length, candidates.map(item => ({ name: item.name, x: item.x, y: item.y, width: item.width, height: item.height })));
|
||||
await win.webContents.executeJavaScript("window.__done=false;window.__cancelled=false;window.__selected=null;window.__hover=null;window.__hoverCandidate=null;window.__confirmed=null;window.__readyAt=Date.now()+600;showCandidates(" + payload + ");");
|
||||
await win.show();
|
||||
win.focus();
|
||||
await win.webContents.executeJavaScript(
|
||||
"window.__done=false;window.__cancelled=false;window.__selectedCid=null;window.__hoverCid=null;window.__hoverCandidate=null;window.__confirmedCid=null;window.__readyAt=Date.now()+500;showCandidates(" + payload + ");",
|
||||
true
|
||||
);
|
||||
|
||||
const deadline = Date.now() + 180000;
|
||||
let selectedIndex: number | null = null;
|
||||
let selectedId: string | null = null;
|
||||
while (Date.now() < deadline) {
|
||||
await new Promise(resolve => setTimeout(resolve, 80));
|
||||
const state = await win.webContents.executeJavaScript(
|
||||
"JSON.stringify({done:window.__done===true,cancelled:window.__cancelled===true,selected:window.__selected})", true
|
||||
"JSON.stringify({done:window.__done===true,cancelled:window.__cancelled===true,selectedCid:window.__selectedCid||null})",
|
||||
true
|
||||
);
|
||||
const parsed = JSON.parse(state) as { done: boolean; cancelled: boolean; selected: number | null };
|
||||
const parsed = JSON.parse(state) as { done: boolean; cancelled: boolean; selectedCid: string | null };
|
||||
if (parsed.cancelled) throw new RuntimeError("pick_cancelled", "已取消 AI 拾取");
|
||||
if (parsed.done && Number.isInteger(parsed.selected)) {
|
||||
const confirmedIndex = Number(parsed.selected);
|
||||
console.log("[pane-ai] confirmed", confirmedIndex, candidates[confirmedIndex]);
|
||||
selectedIndex = confirmedIndex;
|
||||
break;
|
||||
if (parsed.done && parsed.selectedCid) {
|
||||
const box = candidates.find(item => item.candidateId === parsed.selectedCid);
|
||||
console.log("[pane-ai] confirmed", parsed.selectedCid, box);
|
||||
if (box) {
|
||||
selectedId = box.candidateId;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (selectedIndex === null || !candidates[selectedIndex]) throw new RuntimeError("pick_cancelled", "AI 拾取超时");
|
||||
if (!selectedId) throw new RuntimeError("pick_cancelled", "AI 拾取超时");
|
||||
|
||||
const box = candidates[selectedIndex];
|
||||
const x = capture.cropX + box.x;
|
||||
const y = capture.cropY + box.y;
|
||||
const box = candidates.find(item => item.candidateId === selectedId)!;
|
||||
const x = detected.capture.cropX + box.x;
|
||||
const y = detected.capture.cropY + box.y;
|
||||
const selectorParts = [
|
||||
"screen=" + [x, y, box.width, box.height].map(Math.round).join(","),
|
||||
context.window ? "window=" + safeSelectorText(context.window) : "",
|
||||
@ -404,37 +467,15 @@ async function runPaneAiPick(context: UiaPickOutput): Promise<DesktopPickResult>
|
||||
controlType: box.controlType,
|
||||
x, y, w: box.width, h: box.height,
|
||||
confidence: box.confidence,
|
||||
screenshotPath: capture.imagePath,
|
||||
screenshotPath: detected.capture.imagePath,
|
||||
selector: selectorParts.join(";")
|
||||
};
|
||||
} finally {
|
||||
if (debugHotkeyRegistered) globalShortcut.unregister("F8");
|
||||
if (!win.isDestroyed()) win.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
function aiOverlayHtml(): string {
|
||||
return `<!DOCTYPE html><html><head><meta charset='utf-8'><style>
|
||||
html,body{margin:0;width:100%;height:100%;overflow:hidden;background:rgba(3,20,45,.12);cursor:crosshair;font-family:'Microsoft YaHei',Arial,sans-serif}
|
||||
#tip{position:fixed;top:28px;left:50%;transform:translateX(-50%);padding:10px 20px;border-radius:8px;color:#f8fafc;background:rgba(2,14,34,.82);border:1px solid rgba(96,165,250,.55);font-size:15px;box-shadow:0 8px 30px rgba(0,0,0,.25)}
|
||||
#scan{position:absolute;left:0;right:0;top:-240px;height:240px;display:none;background:linear-gradient(to bottom,rgba(59,130,246,0),rgba(59,130,246,.12),rgba(147,197,253,.46),rgba(59,130,246,.10),rgba(59,130,246,0));box-shadow:0 0 38px rgba(96,165,250,.88);animation:scan 1.7s linear infinite}
|
||||
#point{position:absolute;width:46px;height:46px;border:2px solid #38bdf8;border-radius:8px;display:none;box-shadow:0 0 20px rgba(56,189,248,.9)}
|
||||
body.scanning{background:rgba(3,20,45,.03)}
|
||||
body.scanning #scan{display:block}
|
||||
body.scanning #tip{font-size:13px}
|
||||
@keyframes scan{from{top:-240px}to{top:100%}}
|
||||
</style></head><body><div id=scan></div><div id=point></div><div id=tip>AI 拾取：单击要识别的控件；Esc 取消</div><script>
|
||||
document.addEventListener('mousedown', function(e) {
|
||||
if (document.body.classList.contains('scanning')) return;
|
||||
var p=document.getElementById('point');
|
||||
p.style.left=(e.clientX-23)+'px';p.style.top=(e.clientY-23)+'px';p.style.display='block';
|
||||
document.getElementById('tip').textContent='AI 视觉识别中…自上而下扫描画面';
|
||||
document.body.classList.add('scanning');
|
||||
window.__point={x:e.clientX,y:e.clientY};window.__done=true;
|
||||
});
|
||||
document.addEventListener('keydown',function(e){if(e.key==='Escape'){window.__cancelled=true;}});
|
||||
</script></body></html>`;
|
||||
}
|
||||
|
||||
function displayForPoint(x: number, y: number) {
|
||||
return screen.getAllDisplays().find(display => {
|
||||
const b = display.bounds;
|
||||
@ -442,216 +483,6 @@ function displayForPoint(x: number, y: number) {
|
||||
}) || screen.getPrimaryDisplay();
|
||||
}
|
||||
|
||||
async function createAiOverlay(): Promise<{ win: BrowserWindow; display: ReturnType<typeof screen.getPrimaryDisplay> }> {
|
||||
const cursor = screen.getCursorScreenPoint();
|
||||
const display = displayForPoint(cursor.x, cursor.y);
|
||||
const win = new BrowserWindow({
|
||||
x: display.bounds.x,
|
||||
y: display.bounds.y,
|
||||
width: display.bounds.width,
|
||||
height: display.bounds.height,
|
||||
frame: false,
|
||||
transparent: true,
|
||||
resizable: false,
|
||||
movable: false,
|
||||
minimizable: false,
|
||||
maximizable: false,
|
||||
alwaysOnTop: true,
|
||||
skipTaskbar: true,
|
||||
show: false,
|
||||
webPreferences: { contextIsolation: true, nodeIntegration: false, sandbox: true }
|
||||
});
|
||||
win.setAlwaysOnTop(true, "screen-saver");
|
||||
await win.loadURL("data:text/html;charset=utf-8," + encodeURIComponent(aiOverlayHtml()));
|
||||
win.show();
|
||||
win.focus();
|
||||
return { win, display };
|
||||
}
|
||||
|
||||
async function captureAiAtPoint(physicalX: number, physicalY: number): Promise<AiCaptureOutput> {
|
||||
const outDir = path.join(app.getPath("userData"), "ai-picks");
|
||||
fs.mkdirSync(outDir, { recursive: true });
|
||||
const imagePath = path.join(outDir, "ai-pick-" + Date.now() + ".png");
|
||||
const { stdout } = await run(
|
||||
"powershell.exe",
|
||||
[
|
||||
"-NoProfile", "-STA", "-ExecutionPolicy", "Bypass", "-File", pickerScriptPath(),
|
||||
"-Mode", "capture", "-CaptureX", String(physicalX), "-CaptureY", String(physicalY),
|
||||
"-CaptureWidth", "1000", "-CaptureHeight", "700", "-CapturePath", imagePath
|
||||
],
|
||||
{ timeout: 30000, windowsHide: true, maxBuffer: 2 * 1024 * 1024 }
|
||||
);
|
||||
const parsed = jsonLine<AiCaptureOutput>(stdout, "{");
|
||||
if (!parsed.imagePath || !fs.existsSync(parsed.imagePath)) {
|
||||
throw new RuntimeError("ai_capture_failed", "AI 拾取截图失败");
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function parseAiBox(text: string, cropW: number, cropH: number) {
|
||||
const fence = String.fromCharCode(96);
|
||||
const fencePattern = new RegExp(fence + "{3}(?:json)?" + fence + "{3}", "gi");
|
||||
const cleaned = text.replace(fencePattern, "").trim();
|
||||
const start = cleaned.indexOf("{");
|
||||
const end = cleaned.lastIndexOf("}");
|
||||
if (start < 0 || end <= start) throw new RuntimeError("ai_response_invalid", "AI 视觉模型没有返回 JSON");
|
||||
const raw = JSON.parse(cleaned.slice(start, end + 1)) as Record<string, unknown>;
|
||||
const num = (...keys: string[]): number => {
|
||||
for (const key of keys) {
|
||||
const value = Number(raw[key]);
|
||||
if (Number.isFinite(value)) return value;
|
||||
}
|
||||
return NaN;
|
||||
};
|
||||
const x1 = num("x", "left", "x1");
|
||||
const y1 = num("y", "top", "y1");
|
||||
let width = num("width", "w");
|
||||
let height = num("height", "h");
|
||||
const x2 = num("x2", "right");
|
||||
const y2 = num("y2", "bottom");
|
||||
if (!Number.isFinite(width) && Number.isFinite(x2)) width = x2 - x1;
|
||||
if (!Number.isFinite(height) && Number.isFinite(y2)) height = y2 - y1;
|
||||
if (![x1, y1].every(Number.isFinite)) throw new RuntimeError("ai_response_invalid", "AI 视觉模型缺少目标坐标");
|
||||
if (!Number.isFinite(width) || width <= 0) width = 24;
|
||||
if (!Number.isFinite(height) || height <= 0) height = 18;
|
||||
width = Math.max(4, Math.min(cropW, width));
|
||||
height = Math.max(4, Math.min(cropH, height));
|
||||
const localX = Math.max(0, Math.min(cropW - width, x1));
|
||||
const localY = Math.max(0, Math.min(cropH - height, y1));
|
||||
const confidence = Number(raw.confidence);
|
||||
return {
|
||||
x: localX,
|
||||
y: localY,
|
||||
width,
|
||||
height,
|
||||
name: String(raw.name ?? "AI识别控件"),
|
||||
controlType: String(raw.controlType ?? "Control"),
|
||||
confidence: Number.isFinite(confidence) ? Math.max(0, Math.min(1, confidence)) : 0.7
|
||||
};
|
||||
}
|
||||
|
||||
async function runAiPick(context: UiaPickOutput = {}): Promise<DesktopPickResult> {
|
||||
requireAiConfig({ vision: true });
|
||||
const { win, display } = await createAiOverlay();
|
||||
try {
|
||||
let point: { x: number; y: number } | null = null;
|
||||
let physicalX = 0;
|
||||
let physicalY = 0;
|
||||
const autoAi = context.aiRequested && Number.isFinite(Number(context.cursorX)) && Number.isFinite(Number(context.cursorY));
|
||||
if (autoAi) {
|
||||
const cursor = screen.getCursorScreenPoint();
|
||||
point = { x: cursor.x - display.bounds.x, y: cursor.y - display.bounds.y };
|
||||
physicalX = Math.round(Number(context.cursorX));
|
||||
physicalY = Math.round(Number(context.cursorY));
|
||||
await win.webContents.executeJavaScript(
|
||||
"var p=document.getElementById('point');p.style.left=(window.__autoX=" + point.x + ")-23+'px';p.style.top=(window.__autoY=" + point.y + ")-23+'px';p.style.display='block';document.getElementById('tip').textContent='AI 视觉识别中…自上而下扫描画面';document.body.classList.add('scanning');window.__done=true;"
|
||||
);
|
||||
await new Promise(resolve => setTimeout(resolve, 120));
|
||||
}
|
||||
const deadline = Date.now() + 120000;
|
||||
while (!autoAi && Date.now() < deadline) {
|
||||
await new Promise(resolve => setTimeout(resolve, 80));
|
||||
try {
|
||||
const state = await win.webContents.executeJavaScript(
|
||||
"JSON.stringify({done:window.__done===true,cancelled:window.__cancelled===true,point:window.__point||null})", true
|
||||
);
|
||||
const parsed = JSON.parse(state) as { done: boolean; cancelled: boolean; point: { x: number; y: number } | null };
|
||||
if (parsed.cancelled) throw new RuntimeError("pick_cancelled", "已取消 AI 拾取");
|
||||
if (parsed.done && parsed.point) {
|
||||
point = parsed.point;
|
||||
break;
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof RuntimeError && error.code === "pick_cancelled") throw error;
|
||||
}
|
||||
}
|
||||
if (!point) throw new RuntimeError("pick_cancelled", "AI 拾取超时");
|
||||
|
||||
// Hide the effect layer so the screen capture contains the real application only.
|
||||
win.hide();
|
||||
await new Promise(resolve => setTimeout(resolve, 180));
|
||||
if (!autoAi) {
|
||||
const scale = display.scaleFactor || 1;
|
||||
physicalX = Math.round((display.bounds.x + point.x) * scale);
|
||||
physicalY = Math.round((display.bounds.y + point.y) * scale);
|
||||
}
|
||||
const capture = await captureAiAtPoint(physicalX, physicalY);
|
||||
win.showInactive();
|
||||
await win.webContents.executeJavaScript(
|
||||
"document.getElementById('tip').textContent='AI 视觉识别中…请稍候';document.body.classList.add('scanning');"
|
||||
);
|
||||
|
||||
const image = await fs.promises.readFile(capture.imagePath);
|
||||
let aiSettled = false;
|
||||
const aiRequest = chatCompletion([
|
||||
{
|
||||
role: "system",
|
||||
content: "你是桌面 RPA 视觉定位器。只输出 JSON，不输出解释或 Markdown。"
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: [
|
||||
"截图尺寸 " + capture.cropW + "x" + capture.cropH + " 像素。",
|
||||
"红黑色十字中心位于 (" + capture.markerX + ", " + capture.markerY + ")。",
|
||||
"请定位十字中心处或半径 35 像素内最小的可交互 UI 控件；不要返回整个窗口或大容器。",
|
||||
"只返回 JSON：{\"x\":number,\"y\":number,\"width\":number,\"height\":number,\"name\":string,\"controlType\":string,\"confidence\":number}",
|
||||
"x/y 是控件左上角在截图内的像素坐标，width/height 单位也是像素。"
|
||||
].join("\n")
|
||||
},
|
||||
{ type: "image_url", image_url: { url: "data:image/png;base64," + image.toString("base64") } }
|
||||
]
|
||||
}
|
||||
], { vision: true, temperature: 0 }).finally(() => { aiSettled = true; });
|
||||
const cancelWatcher = (async () => {
|
||||
while (!aiSettled) {
|
||||
try {
|
||||
const state = await win.webContents.executeJavaScript("JSON.stringify({cancelled:window.__cancelled===true})", true);
|
||||
if (JSON.parse(state).cancelled) throw new RuntimeError("pick_cancelled", "已取消 AI 拾取");
|
||||
} catch (error) {
|
||||
if (error instanceof RuntimeError && error.code === "pick_cancelled") throw error;
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
}
|
||||
})();
|
||||
try {
|
||||
var result = (await Promise.race([aiRequest, cancelWatcher])) as string;
|
||||
} finally {
|
||||
aiSettled = true;
|
||||
}
|
||||
|
||||
const box = parseAiBox(result, capture.cropW, capture.cropH);
|
||||
const x = capture.cropX + box.x;
|
||||
const y = capture.cropY + box.y;
|
||||
const w = box.width;
|
||||
const h = box.height;
|
||||
const parts = [
|
||||
"screen=" + [x, y, w, h].map(Math.round).join(","),
|
||||
context.window || capture.window ? "window=" + safeSelectorText(context.window || capture.window) : "",
|
||||
"name=" + safeSelectorText(box.name),
|
||||
"type=" + safeSelectorText(box.controlType || "Control"),
|
||||
"ai=1"
|
||||
].filter(Boolean);
|
||||
return {
|
||||
source: "ai",
|
||||
window: context.window || capture.window,
|
||||
name: box.name,
|
||||
controlType: box.controlType,
|
||||
x,
|
||||
y,
|
||||
w,
|
||||
h,
|
||||
confidence: box.confidence,
|
||||
screenshotPath: capture.imagePath,
|
||||
selector: parts.join(";")
|
||||
};
|
||||
} finally {
|
||||
if (!win.isDestroyed()) win.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
export async function listVisibleWindows(): Promise<Array<{ title: string; process: string }>> {
|
||||
const { stdout } = await run(
|
||||
"powershell.exe",
|
||||
@ -722,3 +553,7 @@ export async function pickScreenRegion(): Promise<{ x: number; y: number; width:
|
||||
await run("powershell.exe", ["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-Command", psScript], { timeout: 30000, windowsHide: true });
|
||||
return { ...region, imagePath };
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@ -9,7 +9,9 @@ import { capabilityCount, capabilityIds } from "./engine/registry";
|
||||
import { closeBrowserSession, startRun, stopRun } from "./run/manager";
|
||||
import { fixtureUrl, startFixtureServer, stopFixtureServer } from "./fixture/server";
|
||||
import { loadSettings, updateSettings } from "./settings";
|
||||
import { closeLocalVisionWorkers, defaultUiModelPath } from "./local-vision/ui-detect-service";
|
||||
import { pickDesktopElement, listVisibleWindows, pickScreenRegion } from "./desktop-pick";
|
||||
import { detectVisionLabTarget, listVisionLabSources } from "./vision-lab";
|
||||
import { logMain } from "./logger";
|
||||
|
||||
let mainWindow: BrowserWindow | null = null;
|
||||
@ -113,9 +115,12 @@ function registerIpc(): void {
|
||||
}
|
||||
return session.pickElement();
|
||||
});
|
||||
ipcMain.handle("desktop:pick-element", (_event, options?: { ai?: boolean }) => pickDesktopElement(options));
|
||||
ipcMain.handle("desktop:pick-element", () => pickDesktopElement());
|
||||
ipcMain.handle("desktop:pick-window-list", () => listVisibleWindows());
|
||||
ipcMain.handle("desktop:pick-region", () => pickScreenRegion());
|
||||
ipcMain.handle("vision-lab:sources", () => listVisionLabSources());
|
||||
ipcMain.handle("vision-lab:detect", (_event, request: Parameters<typeof detectVisionLabTarget>[0]) => detectVisionLabTarget(request));
|
||||
ipcMain.handle("vision-lab:default-model", () => defaultUiModelPath());
|
||||
ipcMain.handle("flows:list", () => listFlows());
|
||||
ipcMain.handle("flows:create", (_event, name?: string) => {
|
||||
const flow = createFlow(name);
|
||||
@ -212,6 +217,7 @@ if (!gotLock) {
|
||||
app.on("before-quit", () => {
|
||||
void closeBrowserSession();
|
||||
void stopFixtureServer();
|
||||
closeLocalVisionWorkers();
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
145
src/main/local-vision/ui-detect-service.ts
Normal file
145
src/main/local-vision/ui-detect-service.ts
Normal file
@ -0,0 +1,145 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { Worker } from "node:worker_threads";
|
||||
import { app } from "electron";
|
||||
import { RuntimeError } from "../engine/types";
|
||||
import type { LocalUiBox } from "./ui-detect-worker";
|
||||
|
||||
export interface UiDetectOptions {
|
||||
image: Buffer;
|
||||
width: number;
|
||||
height: number;
|
||||
modelPath?: string;
|
||||
scoreThreshold?: number;
|
||||
iouThreshold?: number;
|
||||
}
|
||||
|
||||
interface DetectRequest {
|
||||
id: number;
|
||||
image: Buffer;
|
||||
width: number;
|
||||
height: number;
|
||||
scoreThreshold?: number;
|
||||
iouThreshold?: number;
|
||||
}
|
||||
|
||||
interface DetectResponse {
|
||||
id: number;
|
||||
boxes?: LocalUiBox[];
|
||||
error?: string;
|
||||
elapsedMs?: number;
|
||||
}
|
||||
|
||||
const workers = new Map<string, Worker>();
|
||||
const pending = new Map<number, { resolve: (boxes: LocalUiBox[]) => void; reject: (error: unknown) => void; worker: Worker }>();
|
||||
let nextRequestId = 1;
|
||||
const MAX_LOADED_WORKERS = 4;
|
||||
|
||||
export function localUiModelPath(): string {
|
||||
const candidates = [
|
||||
path.join(process.resourcesPath || "", "models", "ui-icon-detect", "model.onnx"),
|
||||
path.join(app.getAppPath(), "resources", "models", "ui-icon-detect", "model.onnx")
|
||||
].filter(Boolean);
|
||||
const found = candidates.find(candidate => fs.existsSync(candidate));
|
||||
if (!found) {
|
||||
throw new RuntimeError("local_vision_model_missing", "本地 UI 检测模型缺失", {
|
||||
recommendation: "重新安装应用，或补齐 resources/models/ui-icon-detect/model.onnx"
|
||||
});
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
export function defaultUiModelPath(): string {
|
||||
return localUiModelPath();
|
||||
}
|
||||
|
||||
function resolveModelPath(candidate?: string): string {
|
||||
if (!candidate) return localUiModelPath();
|
||||
const normalized = path.resolve(candidate);
|
||||
if (!fs.existsSync(normalized) || !fs.statSync(normalized).isFile()) {
|
||||
throw new RuntimeError("local_vision_model_missing", "模型文件不存在：" + normalized);
|
||||
}
|
||||
if (path.extname(normalized).toLowerCase() !== ".onnx") {
|
||||
throw new RuntimeError("local_vision_model_invalid", "请选择 ONNX 模型文件");
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function createWorker(modelPath: string): Worker {
|
||||
const next = new Worker(path.join(__dirname, "local-vision/ui-detect-worker.js"), {
|
||||
workerData: { modelPath },
|
||||
resourceLimits: { maxOldGenerationSizeMb: 512 }
|
||||
});
|
||||
next.on("message", (message: DetectResponse) => {
|
||||
const entry = pending.get(message.id);
|
||||
if (!entry || entry.worker !== next) return;
|
||||
pending.delete(message.id);
|
||||
if (message.error) {
|
||||
entry.reject(new RuntimeError("local_vision_failed", "本地 UI 检测失败：" + message.error));
|
||||
return;
|
||||
}
|
||||
entry.resolve(message.boxes || []);
|
||||
});
|
||||
next.on("error", error => {
|
||||
for (const [id, entry] of pending) {
|
||||
if (entry.worker === next) {
|
||||
entry.reject(error);
|
||||
pending.delete(id);
|
||||
}
|
||||
}
|
||||
workers.delete(modelPath);
|
||||
});
|
||||
next.on("exit", code => {
|
||||
for (const [id, entry] of pending) {
|
||||
if (entry.worker === next) {
|
||||
entry.reject(new RuntimeError("local_vision_failed", "本地 UI 检测进程已退出：" + code));
|
||||
pending.delete(id);
|
||||
}
|
||||
}
|
||||
if (workers.get(modelPath) === next) workers.delete(modelPath);
|
||||
});
|
||||
return next;
|
||||
}
|
||||
|
||||
function getWorker(modelPath: string): Worker {
|
||||
const existing = workers.get(modelPath);
|
||||
if (existing) return existing;
|
||||
while (workers.size >= MAX_LOADED_WORKERS) {
|
||||
const oldest = workers.keys().next().value;
|
||||
if (!oldest) break;
|
||||
const worker = workers.get(oldest);
|
||||
workers.delete(oldest);
|
||||
void worker?.terminate();
|
||||
}
|
||||
const created = createWorker(modelPath);
|
||||
workers.set(modelPath, created);
|
||||
return created;
|
||||
}
|
||||
|
||||
export async function detectUiElements(input: UiDetectOptions): Promise<LocalUiBox[]> {
|
||||
const modelPath = resolveModelPath(input.modelPath);
|
||||
const current = getWorker(modelPath);
|
||||
const nextId = nextRequestId++;
|
||||
|
||||
const request: DetectRequest = {
|
||||
id: nextId,
|
||||
image: input.image,
|
||||
width: input.width,
|
||||
height: input.height,
|
||||
scoreThreshold: input.scoreThreshold,
|
||||
iouThreshold: input.iouThreshold
|
||||
};
|
||||
return new Promise<LocalUiBox[]>((resolve, reject) => {
|
||||
pending.set(nextId, { resolve, reject, worker: current });
|
||||
current.postMessage(request);
|
||||
});
|
||||
}
|
||||
|
||||
export function closeLocalVisionWorkers(): void {
|
||||
for (const [modelPath, worker] of workers) {
|
||||
workers.delete(modelPath);
|
||||
void worker.terminate();
|
||||
}
|
||||
pending.clear();
|
||||
}
|
||||
|
||||
205
src/main/local-vision/ui-detect-worker.ts
Normal file
205
src/main/local-vision/ui-detect-worker.ts
Normal file
@ -0,0 +1,205 @@
|
||||
import { parentPort, workerData } from "node:worker_threads";
|
||||
import { InferenceSession, Tensor } from "onnxruntime-node";
|
||||
import { createCanvas, loadImage } from "@napi-rs/canvas";
|
||||
|
||||
export interface LocalUiBox {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
category: string;
|
||||
confidence: number;
|
||||
}
|
||||
|
||||
interface DetectRequest {
|
||||
id: number;
|
||||
image: Buffer;
|
||||
width: number;
|
||||
height: number;
|
||||
scoreThreshold?: number;
|
||||
iouThreshold?: number;
|
||||
}
|
||||
|
||||
interface DetectResponse {
|
||||
id: number;
|
||||
boxes?: LocalUiBox[];
|
||||
error?: string;
|
||||
elapsedMs?: number;
|
||||
}
|
||||
|
||||
type ModelOutput = { output0: Tensor };
|
||||
type TensorLike = { dims: number[]; data: Float32Array };
|
||||
|
||||
const inputName = "images";
|
||||
const outputName = "output0";
|
||||
const inputSize = 640;
|
||||
const defaultScoreThreshold = 0.25;
|
||||
const defaultIouThreshold = 0.42;
|
||||
const maxBoxes = 160;
|
||||
|
||||
let sessionPromise: Promise<InferenceSession> | null = null;
|
||||
|
||||
function loadSession(): Promise<InferenceSession> {
|
||||
if (!sessionPromise) {
|
||||
const modelPath = String((workerData as { modelPath?: string }).modelPath || "");
|
||||
sessionPromise = InferenceSession.create(modelPath, {
|
||||
executionProviders: ["cpu"],
|
||||
graphOptimizationLevel: "all",
|
||||
executionMode: "sequential"
|
||||
});
|
||||
}
|
||||
return sessionPromise;
|
||||
}
|
||||
|
||||
function iou(a: LocalUiBox, b: LocalUiBox): number {
|
||||
const x1 = Math.max(a.x, b.x);
|
||||
const y1 = Math.max(a.y, b.y);
|
||||
const x2 = Math.min(a.x + a.width, b.x + b.width);
|
||||
const y2 = Math.min(a.y + a.height, b.y + b.height);
|
||||
const intersection = Math.max(0, x2 - x1) * Math.max(0, y2 - y1);
|
||||
if (intersection <= 0) return 0;
|
||||
return intersection / (a.width * a.height + b.width * b.height - intersection);
|
||||
}
|
||||
|
||||
function nms(boxes: LocalUiBox[], iouThreshold: number): LocalUiBox[] {
|
||||
const sorted = [...boxes].sort((a, b) => b.confidence - a.confidence);
|
||||
const kept: LocalUiBox[] = [];
|
||||
for (const box of sorted) {
|
||||
if (kept.every(item => iou(item, box) <= iouThreshold)) kept.push(box);
|
||||
}
|
||||
return kept;
|
||||
}
|
||||
|
||||
function tensorAccessor(output: TensorLike, anchorCount: number, channelCount: number, channelFirst: boolean) {
|
||||
return (channel: number, anchor: number): number => channelFirst
|
||||
? output.data[channel * anchorCount + anchor]
|
||||
: output.data[anchor * channelCount + channel];
|
||||
}
|
||||
|
||||
function extractBoxes(
|
||||
output: TensorLike,
|
||||
sourceWidth: number,
|
||||
sourceHeight: number,
|
||||
scoreThreshold: number,
|
||||
iouThreshold: number
|
||||
): LocalUiBox[] {
|
||||
// YOLO/OmniParser exports are commonly [1,C,N] or [1,N,C]. C=4+classCount.
|
||||
if (output.dims.length < 2) throw new Error(`不支持的模型输出维度：${output.dims.join("x")}`);
|
||||
const [, first, second] = output.dims;
|
||||
const channelFirst = first <= second;
|
||||
const channelCount = channelFirst ? first : second;
|
||||
const anchorCount = channelFirst ? second : first;
|
||||
if (channelCount < 5) throw new Error(`模型输出通道数异常：${channelCount}`);
|
||||
const at = tensorAccessor(output, anchorCount, channelCount, channelFirst);
|
||||
const classCount = channelCount - 4;
|
||||
|
||||
const padded = Math.max(sourceWidth, sourceHeight);
|
||||
const scale = inputSize / padded;
|
||||
const drawnWidth = Math.round(sourceWidth * scale);
|
||||
const drawnHeight = Math.round(sourceHeight * scale);
|
||||
const offsetX = (inputSize - drawnWidth) / 2;
|
||||
const offsetY = (inputSize - drawnHeight) / 2;
|
||||
const minSide = Math.max(8, Math.min(sourceWidth, sourceHeight) * 0.006);
|
||||
const raw: LocalUiBox[] = [];
|
||||
|
||||
for (let index = 0; index < anchorCount; index++) {
|
||||
let confidence = 0;
|
||||
let classIndex = 0;
|
||||
if (classCount === 1) {
|
||||
confidence = at(4, index);
|
||||
} else {
|
||||
for (let candidate = 0; candidate < classCount; candidate++) {
|
||||
const score = at(4 + candidate, index);
|
||||
if (score > confidence) {
|
||||
confidence = score;
|
||||
classIndex = candidate;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!Number.isFinite(confidence) || confidence < scoreThreshold) continue;
|
||||
const cx = at(0, index);
|
||||
const cy = at(1, index);
|
||||
const width = at(2, index);
|
||||
const height = at(3, index);
|
||||
let x = (cx - width / 2 - offsetX) / scale;
|
||||
let y = (cy - height / 2 - offsetY) / scale;
|
||||
let boxWidth = width / scale;
|
||||
let boxHeight = height / scale;
|
||||
x = Math.max(0, Math.min(sourceWidth - 1, x));
|
||||
y = Math.max(0, Math.min(sourceHeight - 1, y));
|
||||
boxWidth = Math.max(2, Math.min(sourceWidth - x, boxWidth));
|
||||
boxHeight = Math.max(2, Math.min(sourceHeight - y, boxHeight));
|
||||
if (boxWidth < minSide || boxHeight < minSide) continue;
|
||||
const areaRatio = (boxWidth * boxHeight) / (sourceWidth * sourceHeight);
|
||||
if (areaRatio > 0.92) continue;
|
||||
raw.push({
|
||||
x: Math.round(x),
|
||||
y: Math.round(y),
|
||||
width: Math.round(boxWidth),
|
||||
height: Math.round(boxHeight),
|
||||
category: classCount <= 1 ? "控件" : `类别${classIndex + 1}`,
|
||||
confidence: Math.round(Math.max(0, Math.min(1, confidence)) * 1000) / 1000
|
||||
});
|
||||
}
|
||||
|
||||
return nms(raw, iouThreshold)
|
||||
.sort((a, b) => b.confidence - a.confidence || a.y - b.y || a.x - b.x)
|
||||
.slice(0, maxBoxes)
|
||||
.sort((a, b) => a.y - b.y || a.x - b.x);
|
||||
}
|
||||
|
||||
async function detect(request: DetectRequest): Promise<LocalUiBox[]> {
|
||||
const model = await loadSession();
|
||||
const source = await loadImage(request.image);
|
||||
const width = request.width || source.width;
|
||||
const height = request.height || source.height;
|
||||
|
||||
const canvas = createCanvas(inputSize, inputSize);
|
||||
const context = canvas.getContext("2d");
|
||||
context.fillStyle = "#727272";
|
||||
context.fillRect(0, 0, inputSize, inputSize);
|
||||
context.imageSmoothingEnabled = true;
|
||||
context.imageSmoothingQuality = "high";
|
||||
const scale = Math.min(inputSize / width, inputSize / height);
|
||||
context.drawImage(
|
||||
source,
|
||||
(inputSize - width * scale) / 2,
|
||||
(inputSize - height * scale) / 2,
|
||||
width * scale,
|
||||
height * scale
|
||||
);
|
||||
const pixels = context.getImageData(0, 0, inputSize, inputSize).data;
|
||||
const tensorData = new Float32Array(3 * inputSize * inputSize);
|
||||
const planeSize = inputSize * inputSize;
|
||||
for (let index = 0; index < planeSize; index++) {
|
||||
tensorData[index] = pixels[index * 4] / 255;
|
||||
tensorData[planeSize + index] = pixels[index * 4 + 1] / 255;
|
||||
tensorData[planeSize * 2 + index] = pixels[index * 4 + 2] / 255;
|
||||
}
|
||||
|
||||
const results = await model.run({ [inputName]: new Tensor("float32", tensorData, [1, 3, inputSize, inputSize]) });
|
||||
const output = results[outputName] as unknown as TensorLike;
|
||||
return extractBoxes(
|
||||
output,
|
||||
width,
|
||||
height,
|
||||
request.scoreThreshold ?? defaultScoreThreshold,
|
||||
request.iouThreshold ?? defaultIouThreshold
|
||||
);
|
||||
}
|
||||
|
||||
parentPort?.on("message", async (request: DetectRequest) => {
|
||||
const started = Date.now();
|
||||
try {
|
||||
const boxes = await detect(request);
|
||||
const response: DetectResponse = { id: request.id, boxes, elapsedMs: Date.now() - started };
|
||||
parentPort?.postMessage(response);
|
||||
} catch (error) {
|
||||
const response: DetectResponse = {
|
||||
id: request.id,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
elapsedMs: Date.now() - started
|
||||
};
|
||||
parentPort?.postMessage(response);
|
||||
}
|
||||
});
|
||||
70
src/main/vision-lab.ts
Normal file
70
src/main/vision-lab.ts
Normal file
@ -0,0 +1,70 @@
|
||||
import { desktopCapturer } from "electron";
|
||||
import path from "node:path";
|
||||
import type { VisionLabDetectRequest, VisionLabDetection, VisionLabSource } from "../shared/types";
|
||||
import { RuntimeError } from "./engine/types";
|
||||
import { defaultUiModelPath, detectUiElements } from "./local-vision/ui-detect-service";
|
||||
|
||||
function sourceKind(sourceId: string): "window" | "screen" {
|
||||
return sourceId.startsWith("screen:") ? "screen" : "window";
|
||||
}
|
||||
|
||||
function clamp(value: unknown, fallback: number, min: number, max: number): number {
|
||||
const parsed = Number(value);
|
||||
if (!Number.isFinite(parsed)) return fallback;
|
||||
return Math.min(max, Math.max(min, parsed));
|
||||
}
|
||||
|
||||
export async function listVisionLabSources(): Promise<VisionLabSource[]> {
|
||||
const sources = await desktopCapturer.getSources({
|
||||
types: ["window", "screen"],
|
||||
thumbnailSize: { width: 360, height: 210 },
|
||||
fetchWindowIcons: true
|
||||
});
|
||||
return sources
|
||||
.filter(source => source.thumbnail.isEmpty() === false)
|
||||
.filter(source => sourceKind(source.id) === "screen" || source.name !== "RPA Studio")
|
||||
.map(source => ({
|
||||
id: source.id,
|
||||
kind: sourceKind(source.id),
|
||||
name: source.name || (sourceKind(source.id) === "screen" ? "屏幕" : "未命名窗口"),
|
||||
thumbnail: source.thumbnail.toDataURL(),
|
||||
icon: source.appIcon?.toDataURL()
|
||||
}));
|
||||
}
|
||||
|
||||
export async function detectVisionLabTarget(request: VisionLabDetectRequest): Promise<VisionLabDetection> {
|
||||
if (!request?.sourceId) throw new RuntimeError("vision_lab_source_required", "请先选择识别目标");
|
||||
const sourceType = sourceKind(request.sourceId);
|
||||
const captureStarted = Date.now();
|
||||
const sources = await desktopCapturer.getSources({
|
||||
types: [sourceType],
|
||||
thumbnailSize: { width: 1280, height: 800 }
|
||||
});
|
||||
const source = sources.find(item => item.id === request.sourceId);
|
||||
if (!source) throw new RuntimeError("vision_lab_source_missing", "识别目标已失效，请重新选择窗口");
|
||||
if (source.thumbnail.isEmpty()) throw new RuntimeError("vision_lab_capture_failed", "目标窗口没有可捕获的画面");
|
||||
|
||||
const image = source.thumbnail.toPNG();
|
||||
const width = source.thumbnail.getSize().width;
|
||||
const height = source.thumbnail.getSize().height;
|
||||
const detectStarted = Date.now();
|
||||
const boxes = await detectUiElements({
|
||||
image,
|
||||
width,
|
||||
height,
|
||||
modelPath: request.modelPath,
|
||||
scoreThreshold: clamp(request.scoreThreshold, 0.25, 0.01, 0.95),
|
||||
iouThreshold: clamp(request.iouThreshold, 0.42, 0.05, 0.9)
|
||||
});
|
||||
return {
|
||||
image: source.thumbnail.toDataURL(),
|
||||
width,
|
||||
height,
|
||||
boxes,
|
||||
captureMs: detectStarted - captureStarted,
|
||||
detectMs: Date.now() - detectStarted,
|
||||
modelPath: path.resolve(request.modelPath || defaultUiModelPath()),
|
||||
sourceName: source.name || (sourceType === "screen" ? "屏幕" : "未命名窗口")
|
||||
};
|
||||
}
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
import { contextBridge, ipcRenderer } from "electron";
|
||||
import type { AppInfo, AppSettings, Catalog, FlowDefinition, FlowSaveOptions, FlowSaveResult, FlowSummary, RunEvent } from "../shared/types";
|
||||
import type { AppInfo, AppSettings, Catalog, FlowDefinition, FlowSaveOptions, FlowSaveResult, FlowSummary, RunEvent, VisionLabDetectRequest, VisionLabDetection, VisionLabSource } from "../shared/types";
|
||||
|
||||
const api = {
|
||||
app: {
|
||||
@ -9,7 +9,7 @@ const api = {
|
||||
get: (): Promise<Catalog> => ipcRenderer.invoke("catalog:get")
|
||||
},
|
||||
desktop: {
|
||||
pickElement: (options?: { ai?: boolean }) => ipcRenderer.invoke("desktop:pick-element", options || {}),
|
||||
pickElement: () => ipcRenderer.invoke("desktop:pick-element"),
|
||||
pickWindowList: () => ipcRenderer.invoke('desktop:pick-window-list'),
|
||||
pickRegion: () => ipcRenderer.invoke('desktop:pick-region')
|
||||
},
|
||||
@ -32,6 +32,11 @@ const api = {
|
||||
pickElement: (options?: { url?: string }): Promise<import("../shared/types").ElementPickResult> =>
|
||||
ipcRenderer.invoke("browser:pick-element", options || {})
|
||||
},
|
||||
visionLab: {
|
||||
sources: (): Promise<VisionLabSource[]> => ipcRenderer.invoke("vision-lab:sources"),
|
||||
detect: (request: VisionLabDetectRequest): Promise<VisionLabDetection> => ipcRenderer.invoke("vision-lab:detect", request),
|
||||
defaultModel: (): Promise<string> => ipcRenderer.invoke("vision-lab:default-model")
|
||||
},
|
||||
settings: {
|
||||
get: (): Promise<AppSettings> => ipcRenderer.invoke("settings:get"),
|
||||
update: (patch: Partial<AppSettings>): Promise<AppSettings> => ipcRenderer.invoke("settings:update", patch || {})
|
||||
|
||||
@ -1,16 +1,17 @@
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
import { Bot, CircleAlert, Loader2, ListChecks } from "lucide-react";
|
||||
import { Bot, CircleAlert, Loader2, ListChecks, Microscope } from "lucide-react";
|
||||
import type { Catalog, FlowDefinition, FlowSummary } from "@shared/types";
|
||||
import { useEditorStore } from "./store/editorStore";
|
||||
import { useRunStore } from "./store/runStore";
|
||||
import { EditorPage } from "./pages/EditorPage";
|
||||
import { RunsPage } from "./pages/RunsPage";
|
||||
import { SettingsPage } from "./pages/SettingsPage";
|
||||
import { VisionLabPage } from "./pages/VisionLabPage";
|
||||
import { TasksPage } from "./pages/TasksPage";
|
||||
import { railIcons } from "./components/icons";
|
||||
import { validateFlow } from "./validation/flowValidation";
|
||||
|
||||
type Page = "tasks" | "editor" | "runs" | "settings";
|
||||
type Page = "tasks" | "editor" | "runs" | "settings" | "visionLab";
|
||||
|
||||
interface ToastState {
|
||||
id: number;
|
||||
@ -235,6 +236,14 @@ export function App() {
|
||||
<railIcons.settings className="icon" />
|
||||
设置
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={`rail-item${page === "visionLab" ? " active" : ""}`}
|
||||
onClick={() => setPage("visionLab")}
|
||||
>
|
||||
<Microscope className="icon" />
|
||||
实验台
|
||||
</button>
|
||||
<span className="rail-spacer" />
|
||||
<div className="rail-user">
|
||||
<span className="rail-avatar">RA</span>
|
||||
@ -259,6 +268,7 @@ export function App() {
|
||||
{page === "editor" && <EditorPage catalog={catalog} onToast={showToast} onBack={() => { setPage("tasks"); void refreshFlows(); }} />}
|
||||
{page === "runs" && <RunsPage onBack={() => { setPage("tasks"); void refreshFlows(); }} />}
|
||||
{page === "settings" && <SettingsPage catalog={catalog} />}
|
||||
{page === "visionLab" && <VisionLabPage />}
|
||||
</main>
|
||||
|
||||
{toast && (
|
||||
@ -272,3 +282,4 @@ export function App() {
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
232
src/renderer/src/pages/VisionLabPage.tsx
Normal file
232
src/renderer/src/pages/VisionLabPage.tsx
Normal file
@ -0,0 +1,232 @@
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { FolderOpen, Play, Pause, RefreshCw, Sparkles } from "lucide-react";
|
||||
import type { VisionLabDetection, VisionLabSource } from "@shared/types";
|
||||
|
||||
function fileName(value: string): string {
|
||||
const normalized = value.replace(/[\\/]+$/, "");
|
||||
const index = Math.max(normalized.lastIndexOf("\\"), normalized.lastIndexOf("/"));
|
||||
return index >= 0 ? normalized.slice(index + 1) : normalized;
|
||||
}
|
||||
|
||||
export function VisionLabPage() {
|
||||
const [sources, setSources] = useState<VisionLabSource[]>([]);
|
||||
const [sourceId, setSourceId] = useState("");
|
||||
const [defaultModel, setDefaultModel] = useState("");
|
||||
const [customModel, setCustomModel] = useState<string | null>(null);
|
||||
const [score, setScore] = useState(0.25);
|
||||
const [overlap, setOverlap] = useState(0.42);
|
||||
const [intervalMs, setIntervalMs] = useState(600);
|
||||
const [continuous, setContinuous] = useState(false);
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [detection, setDetection] = useState<VisionLabDetection | null>(null);
|
||||
const canvasRef = useRef<HTMLCanvasElement | null>(null);
|
||||
|
||||
const runTokenRef = useRef(0);
|
||||
const selectedSource = sources.find(source => source.id === sourceId);
|
||||
const activeModel = customModel || defaultModel;
|
||||
|
||||
const refreshSources = useCallback(async () => {
|
||||
try {
|
||||
const next = await window.rpa.visionLab.sources();
|
||||
setSources(next);
|
||||
setSourceId(current => next.some(source => source.id === current) ? current : next[0]?.id || "");
|
||||
setError(null);
|
||||
} catch (refreshError) {
|
||||
setError(refreshError instanceof Error ? refreshError.message : String(refreshError));
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
void refreshSources();
|
||||
void window.rpa.visionLab.defaultModel().then(setDefaultModel).catch(error => {
|
||||
setError(error instanceof Error ? error.message : String(error));
|
||||
});
|
||||
}, [refreshSources]);
|
||||
|
||||
const detectOnce = useCallback(async () => {
|
||||
if (!sourceId) return;
|
||||
setBusy(true);
|
||||
try {
|
||||
const result = await window.rpa.visionLab.detect({
|
||||
sourceId,
|
||||
modelPath: customModel || undefined,
|
||||
scoreThreshold: score,
|
||||
iouThreshold: overlap
|
||||
});
|
||||
setDetection(result);
|
||||
setError(null);
|
||||
} catch (detectError) {
|
||||
setError(detectError instanceof Error ? detectError.message : String(detectError));
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
}, [customModel, overlap, score, sourceId]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!continuous || !sourceId) return;
|
||||
const token = ++runTokenRef.current;
|
||||
let alive = true;
|
||||
const tick = async () => {
|
||||
while (alive) {
|
||||
const started = Date.now();
|
||||
try {
|
||||
const result = await window.rpa.visionLab.detect({
|
||||
sourceId,
|
||||
modelPath: customModel || undefined,
|
||||
scoreThreshold: score,
|
||||
iouThreshold: overlap
|
||||
});
|
||||
if (!alive || token !== runTokenRef.current) return;
|
||||
setDetection(result);
|
||||
setError(null);
|
||||
} catch (detectError) {
|
||||
if (!alive || token !== runTokenRef.current) return;
|
||||
setError(detectError instanceof Error ? detectError.message : String(detectError));
|
||||
}
|
||||
const wait = Math.max(80, intervalMs - (Date.now() - started));
|
||||
await new Promise(resolve => setTimeout(resolve, wait));
|
||||
}
|
||||
};
|
||||
void tick();
|
||||
return () => { alive = false; };
|
||||
}, [continuous, customModel, intervalMs, overlap, score, sourceId]);
|
||||
|
||||
useEffect(() => {
|
||||
const canvas = canvasRef.current;
|
||||
if (!canvas || !detection) return;
|
||||
const image = new Image();
|
||||
image.onload = () => {
|
||||
canvas.width = detection.width;
|
||||
canvas.height = detection.height;
|
||||
const context = canvas.getContext("2d");
|
||||
if (!context) return;
|
||||
context.clearRect(0, 0, canvas.width, canvas.height);
|
||||
context.drawImage(image, 0, 0, canvas.width, canvas.height);
|
||||
const scale = Math.max(1, detection.width / 1000);
|
||||
context.lineWidth = 2 * scale;
|
||||
context.strokeStyle = "#22d3ee";
|
||||
context.font = `${Math.round(14 * scale)}px Microsoft YaHei, Arial, sans-serif`;
|
||||
context.textBaseline = "top";
|
||||
for (const box of detection.boxes) {
|
||||
context.strokeRect(box.x, box.y, box.width, box.height);
|
||||
const label = `${box.category} ${Math.round(box.confidence * 100)}%`;
|
||||
const metrics = context.measureText(label);
|
||||
const labelWidth = metrics.width + 8 * scale;
|
||||
const labelHeight = 19 * scale;
|
||||
const labelY = box.y >= labelHeight + 2 * scale ? box.y - labelHeight - 2 * scale : box.y + 2 * scale;
|
||||
context.fillStyle = "#0e7490";
|
||||
context.fillRect(box.x, labelY, labelWidth, labelHeight);
|
||||
context.fillStyle = "#ffffff";
|
||||
context.fillText(label, box.x + 4 * scale, labelY + 3 * scale);
|
||||
}
|
||||
};
|
||||
image.src = detection.image;
|
||||
}, [detection]);
|
||||
|
||||
const chooseModel = async () => {
|
||||
const selected = await window.rpa.dialog.open({
|
||||
title: "选择 ONNX UI 检测模型",
|
||||
filters: [{ name: "ONNX 模型", extensions: ["onnx"] }]
|
||||
});
|
||||
if (selected) setCustomModel(selected);
|
||||
};
|
||||
|
||||
const modelLabel = useMemo(() => customModel ? fileName(customModel) : "内置 OmniParser v2", [customModel]);
|
||||
const boxes = detection?.boxes || [];
|
||||
|
||||
return (
|
||||
<div className="vision-lab">
|
||||
<aside className="panel vision-lab-side">
|
||||
<div className="panel-head">
|
||||
<strong><Sparkles className="icon" /> AI 识别实验台</strong>
|
||||
<span className="panel-subtitle">持续识别 · 选窗口 · 选模型</span>
|
||||
</div>
|
||||
|
||||
<div className="vision-block">
|
||||
<label className="field-label" htmlFor="vision-source">识别目标</label>
|
||||
<div className="vision-row">
|
||||
<select
|
||||
id="vision-source"
|
||||
className="text-input"
|
||||
value={sourceId}
|
||||
onChange={event => setSourceId(event.target.value)}
|
||||
>
|
||||
{!sources.length && <option value="">正在读取窗口…</option>}
|
||||
{sources.map(source => (
|
||||
<option key={source.id} value={source.id}>
|
||||
{source.kind === "screen" ? "屏幕 · " : "窗口 · "}{source.name}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
<button className="btn icon-btn" type="button" onClick={() => void refreshSources()} title="刷新窗口列表">
|
||||
<RefreshCw className="icon" />
|
||||
</button>
|
||||
</div>
|
||||
{selectedSource && (
|
||||
<div className="source-preview">
|
||||
<img src={selectedSource.thumbnail} alt={selectedSource.name} />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="vision-block">
|
||||
<span className="field-label">识别模型</span>
|
||||
<div className="vision-row">
|
||||
<button
|
||||
type="button"
|
||||
className={`btn${customModel ? "" : " btn-primary"}`}
|
||||
onClick={() => setCustomModel(null)}
|
||||
>
|
||||
内置 OmniParser v2
|
||||
</button>
|
||||
<button type="button" className={`btn${customModel ? " btn-primary" : ""}`} onClick={() => void chooseModel()}>
|
||||
<FolderOpen className="icon" />
|
||||
选择 ONNX
|
||||
</button>
|
||||
</div>
|
||||
<div className="model-path" title={activeModel || ""}>{activeModel || "模型未找到"}</div>
|
||||
</div>
|
||||
|
||||
<div className="vision-block">
|
||||
<label className="field-label" htmlFor="vision-score">置信度阈值：{score.toFixed(2)}</label>
|
||||
<input id="vision-score" type="range" min={0.05} max={0.9} step={0.01} value={score}
|
||||
onChange={event => setScore(Number(event.target.value))} />
|
||||
<label className="field-label" htmlFor="vision-overlap">重叠抑制：{overlap.toFixed(2)}</label>
|
||||
<input id="vision-overlap" type="range" min={0.1} max={0.85} step={0.01} value={overlap}
|
||||
onChange={event => setOverlap(Number(event.target.value))} />
|
||||
<label className="field-label" htmlFor="vision-interval">识别间隔：{intervalMs} ms</label>
|
||||
<input id="vision-interval" type="range" min={100} max={3000} step={50} value={intervalMs}
|
||||
onChange={event => setIntervalMs(Number(event.target.value))} />
|
||||
</div>
|
||||
|
||||
<div className="vision-actions">
|
||||
<button className="btn" type="button" onClick={() => void detectOnce()} disabled={!sourceId || busy}>
|
||||
识别一次
|
||||
</button>
|
||||
<button className={`btn${continuous ? " btn-danger" : " btn-primary"}`} type="button"
|
||||
onClick={() => setContinuous(value => !value)} disabled={!sourceId}>
|
||||
{continuous ? <Pause className="icon" /> : <Play className="icon" />}
|
||||
{continuous ? "停止连续" : "开始连续"}
|
||||
</button>
|
||||
</div>
|
||||
{error && <div className="vision-error">{error}</div>}
|
||||
</aside>
|
||||
|
||||
<section className="panel vision-lab-stage">
|
||||
<div className="vision-stage-head">
|
||||
<strong>{detection?.sourceName || selectedSource?.name || "未选择目标"}</strong>
|
||||
<span>{boxes.length} 个候选 · 采集 {detection?.captureMs ?? "-"}ms · 推理 {detection?.detectMs ?? "-"}ms</span>
|
||||
</div>
|
||||
<div className="vision-stage">
|
||||
{detection ? (
|
||||
<canvas ref={canvasRef} className="vision-canvas" />
|
||||
) : (
|
||||
<div className="vision-empty">选择窗口和模型后开始识别</div>
|
||||
)}
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@ -464,10 +464,10 @@ function FieldControl({
|
||||
onChange(field.key, current === "" || known ? expression : `${textOf(raw)}${expression}`);
|
||||
};
|
||||
|
||||
const pickDesktop = async (ai = false) => {
|
||||
const pickDesktop = async () => {
|
||||
setPicking(true);
|
||||
try {
|
||||
const picked = await window.rpa.desktop.pickElement({ ai });
|
||||
const picked = await window.rpa.desktop.pickElement();
|
||||
if (picked.cancelled) {
|
||||
onToast("已取消拾取");
|
||||
return;
|
||||
@ -647,12 +647,9 @@ function FieldControl({
|
||||
/>
|
||||
{isGroundTarget ? (
|
||||
<>
|
||||
<button type="button" className="mini-btn" disabled={picking} onClick={() => void pickDesktop(false)} title="从桌面拾取元素（移到目标按空格）">
|
||||
<button type="button" className="mini-btn" disabled={picking} onClick={() => void pickDesktop()} title="从桌面拾取元素（移到目标按空格）">
|
||||
{picking ? "桌面拾取中" : "桌面拾取"}
|
||||
</button>
|
||||
<button type="button" className="mini-btn" disabled={picking} onClick={() => void pickDesktop(true)} title="AI 视觉拾取；UIA 类型为 Pane 时也可在桌面拾取中按 F2">
|
||||
{picking ? "AI识别中" : "AI拾取"}
|
||||
</button>
|
||||
</>
|
||||
) : (
|
||||
<button type="button" className="mini-btn" disabled={picking} onClick={() => void pickElement()} title="从运行浏览器拾取元素">
|
||||
|
||||
@ -3444,3 +3444,187 @@ textarea:focus-visible {
|
||||
.icon-btn.refresh-btn.refreshing .icon {
|
||||
animation: spin 0.8s linear infinite;
|
||||
}
|
||||
|
||||
.vision-lab {
|
||||
display: grid;
|
||||
grid-template-columns: 340px minmax(0, 1fr);
|
||||
gap: 18px;
|
||||
height: 100%;
|
||||
min-height: 0;
|
||||
padding: 18px;
|
||||
}
|
||||
|
||||
.vision-lab-side {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 16px;
|
||||
min-height: 0;
|
||||
overflow: auto;
|
||||
padding: 18px;
|
||||
}
|
||||
|
||||
.vision-lab-side .panel-head {
|
||||
align-items: center;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 4px;
|
||||
}
|
||||
|
||||
.vision-lab-side .panel-head strong {
|
||||
align-items: center;
|
||||
display: flex;
|
||||
font-size: 16px;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.vision-block {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.vision-row {
|
||||
align-items: center;
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.vision-row select {
|
||||
min-width: 0;
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
.vision-row .btn {
|
||||
flex: none;
|
||||
}
|
||||
|
||||
.icon-btn {
|
||||
padding-left: 10px;
|
||||
padding-right: 10px;
|
||||
}
|
||||
|
||||
.source-preview {
|
||||
border: 1px solid #d8e0ea;
|
||||
border-radius: 10px;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.source-preview img {
|
||||
display: block;
|
||||
height: 116px;
|
||||
object-fit: cover;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.model-path {
|
||||
border-radius: 8px;
|
||||
background: #f3f6fa;
|
||||
color: #64748b;
|
||||
font-size: 12px;
|
||||
line-height: 1.4;
|
||||
max-height: 52px;
|
||||
overflow: auto;
|
||||
padding: 7px 9px;
|
||||
word-break: break-all;
|
||||
}
|
||||
|
||||
.vision-actions {
|
||||
display: grid;
|
||||
gap: 8px;
|
||||
grid-template-columns: 1fr 1fr;
|
||||
margin-top: auto;
|
||||
}
|
||||
|
||||
.vision-error {
|
||||
background: #fff1f1;
|
||||
border: 1px solid #fecaca;
|
||||
border-radius: 8px;
|
||||
color: #b91c1c;
|
||||
font-size: 12px;
|
||||
line-height: 1.45;
|
||||
padding: 9px 10px;
|
||||
white-space: pre-wrap;
|
||||
}
|
||||
|
||||
.vision-lab-stage {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
min-height: 0;
|
||||
min-width: 0;
|
||||
overflow: hidden;
|
||||
padding: 18px;
|
||||
}
|
||||
|
||||
.vision-stage-head {
|
||||
align-items: center;
|
||||
display: flex;
|
||||
gap: 12px;
|
||||
justify-content: space-between;
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
|
||||
.vision-stage-head strong {
|
||||
font-size: 15px;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.vision-stage-head span {
|
||||
color: #64748b;
|
||||
flex: none;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.vision-stage {
|
||||
align-items: center;
|
||||
background: #102033;
|
||||
border-radius: 12px;
|
||||
display: flex;
|
||||
flex: 1;
|
||||
justify-content: center;
|
||||
min-height: 0;
|
||||
overflow: hidden;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.vision-image {
|
||||
display: block;
|
||||
height: 100%;
|
||||
object-fit: contain;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.vision-box {
|
||||
border: 2px solid #22d3ee;
|
||||
box-shadow: 0 0 0 1px rgba(0, 0, 0, .35) inset;
|
||||
position: absolute;
|
||||
}
|
||||
|
||||
.vision-box span {
|
||||
background: #0e7490;
|
||||
color: white;
|
||||
display: inline-block;
|
||||
font-size: 11px;
|
||||
line-height: 17px;
|
||||
padding: 0 5px;
|
||||
transform: translateY(-100%);
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.vision-empty {
|
||||
align-items: center;
|
||||
color: #94a3b8;
|
||||
display: flex;
|
||||
height: 100%;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.vision-canvas {
|
||||
display: block;
|
||||
height: auto;
|
||||
max-height: 100%;
|
||||
max-width: 100%;
|
||||
width: auto;
|
||||
}
|
||||
|
||||
|
||||
@ -247,3 +247,38 @@ export interface RunEvent {
|
||||
startedAt?: string;
|
||||
}
|
||||
|
||||
|
||||
export interface VisionLabSource {
|
||||
id: string;
|
||||
kind: "window" | "screen";
|
||||
name: string;
|
||||
thumbnail: string;
|
||||
icon?: string;
|
||||
}
|
||||
|
||||
export interface VisionLabBox {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
category: string;
|
||||
confidence: number;
|
||||
}
|
||||
|
||||
export interface VisionLabDetectRequest {
|
||||
sourceId: string;
|
||||
modelPath?: string;
|
||||
scoreThreshold?: number;
|
||||
iouThreshold?: number;
|
||||
}
|
||||
|
||||
export interface VisionLabDetection {
|
||||
image: string;
|
||||
width: number;
|
||||
height: number;
|
||||
boxes: VisionLabBox[];
|
||||
captureMs: number;
|
||||
detectMs: number;
|
||||
modelPath: string;
|
||||
sourceName: string;
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user