import { desktopCapturer } from "electron"; import path from "node:path"; import type { VisionLabDetectRequest, VisionLabDetection, VisionLabSource } from "../shared/types"; import { RuntimeError } from "@rpa/core"; import { defaultUiModelPath, detectUiElements } from "./local-vision/ui-detect-service"; function sourceKind(sourceId: string): "window" | "screen" { return sourceId.startsWith("screen:") ? "screen" : "window"; } function clamp(value: unknown, fallback: number, min: number, max: number): number { const parsed = Number(value); if (!Number.isFinite(parsed)) return fallback; return Math.min(max, Math.max(min, parsed)); } export async function listVisionLabSources(): Promise { const sources = await desktopCapturer.getSources({ types: ["window", "screen"], thumbnailSize: { width: 360, height: 210 }, fetchWindowIcons: true }); return sources .filter(source => source.thumbnail.isEmpty() === false) .filter(source => sourceKind(source.id) === "screen" || source.name !== "RPA Studio") .map(source => ({ id: source.id, kind: sourceKind(source.id), name: source.name || (sourceKind(source.id) === "screen" ? "屏幕" : "未命名窗口"), thumbnail: source.thumbnail.toDataURL(), icon: source.appIcon?.toDataURL() })); } export async function detectVisionLabTarget(request: VisionLabDetectRequest): Promise { if (!request?.sourceId) throw new RuntimeError("vision_lab_source_required", "请先选择识别目标"); const sourceType = sourceKind(request.sourceId); const captureStarted = Date.now(); const sources = await desktopCapturer.getSources({ types: [sourceType], thumbnailSize: { width: 1280, height: 800 } }); const source = sources.find(item => item.id === request.sourceId); if (!source) throw new RuntimeError("vision_lab_source_missing", "识别目标已失效,请重新选择窗口"); if (source.thumbnail.isEmpty()) throw new RuntimeError("vision_lab_capture_failed", "目标窗口没有可捕获的画面"); const image = source.thumbnail.toPNG(); const width = source.thumbnail.getSize().width; const height = source.thumbnail.getSize().height; const detectStarted = Date.now(); const boxes = await detectUiElements({ image, width, height, modelPath: request.modelPath, scoreThreshold: clamp(request.scoreThreshold, 0.25, 0.01, 0.95), iouThreshold: clamp(request.iouThreshold, 0.42, 0.05, 0.9) }); return { image: source.thumbnail.toDataURL(), width, height, boxes, captureMs: detectStarted - captureStarted, detectMs: Date.now() - detectStarted, modelPath: path.resolve(request.modelPath || defaultUiModelPath()), sourceName: source.name || (sourceType === "screen" ? "屏幕" : "未命名窗口") }; }