caa8e19332
- engine/(registry/types/runtime/expression)物理迁入包, 解耦shared/types(域类型归包, 宿主type-only re-export) - 31个纯Node能力+27个测试迁入包; 宿主留守Electron/浏览器/AI依赖能力, index.ts经@rpa/core/capabilities拉起 - 新增createEngine()/defaultEngine: excel缓存/Excel句柄/db连接/FTP客户端/OCR实例/拟真设置挂引擎实例 - remaining-core子流程依赖改为host-bridge注入, 包内不再import宿主flows - 宿主main/index.ts启动时注入子流程源; electron-vite/vitest/tsconfig配@rpa/core源码别名 - 验证: typecheck绿, 快照909个id diff为零, 332测试过(2个桌面冒烟因会话无交互SetForegroundWindow被拒, 环境阻塞)
71 lines
2.8 KiB
TypeScript
71 lines
2.8 KiB
TypeScript
import { desktopCapturer } from "electron";
|
|
import path from "node:path";
|
|
import type { VisionLabDetectRequest, VisionLabDetection, VisionLabSource } from "../shared/types";
|
|
import { RuntimeError } from "@rpa/core";
|
|
import { defaultUiModelPath, detectUiElements } from "./local-vision/ui-detect-service";
|
|
|
|
function sourceKind(sourceId: string): "window" | "screen" {
|
|
return sourceId.startsWith("screen:") ? "screen" : "window";
|
|
}
|
|
|
|
function clamp(value: unknown, fallback: number, min: number, max: number): number {
|
|
const parsed = Number(value);
|
|
if (!Number.isFinite(parsed)) return fallback;
|
|
return Math.min(max, Math.max(min, parsed));
|
|
}
|
|
|
|
export async function listVisionLabSources(): Promise<VisionLabSource[]> {
|
|
const sources = await desktopCapturer.getSources({
|
|
types: ["window", "screen"],
|
|
thumbnailSize: { width: 360, height: 210 },
|
|
fetchWindowIcons: true
|
|
});
|
|
return sources
|
|
.filter(source => source.thumbnail.isEmpty() === false)
|
|
.filter(source => sourceKind(source.id) === "screen" || source.name !== "RPA Studio")
|
|
.map(source => ({
|
|
id: source.id,
|
|
kind: sourceKind(source.id),
|
|
name: source.name || (sourceKind(source.id) === "screen" ? "屏幕" : "未命名窗口"),
|
|
thumbnail: source.thumbnail.toDataURL(),
|
|
icon: source.appIcon?.toDataURL()
|
|
}));
|
|
}
|
|
|
|
export async function detectVisionLabTarget(request: VisionLabDetectRequest): Promise<VisionLabDetection> {
|
|
if (!request?.sourceId) throw new RuntimeError("vision_lab_source_required", "请先选择识别目标");
|
|
const sourceType = sourceKind(request.sourceId);
|
|
const captureStarted = Date.now();
|
|
const sources = await desktopCapturer.getSources({
|
|
types: [sourceType],
|
|
thumbnailSize: { width: 1280, height: 800 }
|
|
});
|
|
const source = sources.find(item => item.id === request.sourceId);
|
|
if (!source) throw new RuntimeError("vision_lab_source_missing", "识别目标已失效，请重新选择窗口");
|
|
if (source.thumbnail.isEmpty()) throw new RuntimeError("vision_lab_capture_failed", "目标窗口没有可捕获的画面");
|
|
|
|
const image = source.thumbnail.toPNG();
|
|
const width = source.thumbnail.getSize().width;
|
|
const height = source.thumbnail.getSize().height;
|
|
const detectStarted = Date.now();
|
|
const boxes = await detectUiElements({
|
|
image,
|
|
width,
|
|
height,
|
|
modelPath: request.modelPath,
|
|
scoreThreshold: clamp(request.scoreThreshold, 0.25, 0.01, 0.95),
|
|
iouThreshold: clamp(request.iouThreshold, 0.42, 0.05, 0.9)
|
|
});
|
|
return {
|
|
image: source.thumbnail.toDataURL(),
|
|
width,
|
|
height,
|
|
boxes,
|
|
captureMs: detectStarted - captureStarted,
|
|
detectMs: Date.now() - detectStarted,
|
|
modelPath: path.resolve(request.modelPath || defaultUiModelPath()),
|
|
sourceName: source.name || (sourceType === "screen" ? "屏幕" : "未命名窗口")
|
|
};
|
|
}
|
|
|