import { browserSession, externalBrowserSession } from "../browser/session"; import { register } from "../engine/registry"; import { RuntimeError, type BrowserLike, type RunContext } from "../engine/types"; /** * 采集/监听组件的浏览器会话解析: * 1. 流程中「打开网页」已建立的会话优先(内置或外部); * 2. 否则若节点 browserType 填 chrome/edge → 拉起对应外部浏览器; * 3. 其余情况退回内置运行浏览器。 */ async function crawlSession(ctx: RunContext, args: Record = {}): Promise { if (ctx.browser) return ctx.browser; const browserType = String(args.browserType ?? "").trim().toLowerCase(); if (browserType === "chrome" || browserType === "edge") { return externalBrowserSession({ mode: "launch", browserType, port: 9222 }); } return browserSession(); } const CRAWL_IDS = [ "WebBrowser_DataCollection", "WebBrowser_DataCollectionV1", "WebBrowser_DataCollectionV2", "WebBrowser_DataCollectionV3", "WebBrowser_DataCollectionV4", "WebBrowser_DataCollectionV5", "WebBrowser_DataCrawl", "WebBrowser_DataCrawl_web", "WebBrowser_DataCrawl_software", "WebBrowser_DataCrawler", "WebBrowser_BatchDataCrawl", "WebBrowser_BatchDataCrawlNew", "WebBrowser_BatchDataCrawlNewV1" ]; function parseList(value: unknown): Array<{ selector: string; label: string }> { if (value === undefined || value === null || value === "") return []; if (Array.isArray(value)) { return value.map(item => typeof item === "string" ? { selector: item, label: item } : { selector: String(item.selector ?? ""), label: String(item.label ?? item.selector ?? "") }).filter(item => item.selector); } const text = String(value).trim(); if (text.startsWith("[")) { try { const parsed = JSON.parse(text) as unknown; if (Array.isArray(parsed)) return parseList(parsed); } catch { /* fall through */ } } return [{ selector: text, label: text }]; } function firstOf(args: Record, keys: string[]): unknown { for (const key of keys) { if (args[key] !== undefined && args[key] !== null && args[key] !== "") return args[key]; } return undefined; } register(CRAWL_IDS, async (ctx, node, args) => { const browser = await crawlSession(ctx, args); const rowSelector = firstOf(args, ["selector_page", "page_element", "uiSelector_page", "row_selector", "selectorCode", "rowSelector"]); if (!rowSelector) { throw new RuntimeError("validation_error", "行选择器为空", { nodeId: node.id, componentId: node.componentId, recommendation: "填写每一行数据的选择器,例如 css:table tbody tr" }); } const rowSel = String(rowSelector).replace(/^css:/, "").trim(); const itemsRaw = firstOf(args, ["collect_items", "config", "columns", "column_selectors"]); const columns = parseList(itemsRaw); const pages = Math.max(1, Number(firstOf(args, ["collect_pages", "page_num", "pages", "repeat_count", "delay_pages"]) ?? 1) || 1); const nextRaw = firstOf(args, ["selector_more", "next_selector", "uiSelector_num"]); const nextSel = nextRaw ? String(nextRaw).replace(/^css:/, "").trim() : ""; const waitMs = Math.max(200, Number(firstOf(args, ["wait_time_page", "waitMs"]) ?? 1200) || 1200); const dedup = String(firstOf(args, ["deduplication"]) ?? "") === "1" || String(firstOf(args, ["deduplication"]) ?? "") === "true"; const script = "(async () => {" + "const rowSel = " + JSON.stringify(rowSel) + ";" + "const cols = " + JSON.stringify(columns.map(c => c.selector)) + ";" + "const pages = " + pages + ";" + "const nextSel = " + JSON.stringify(nextSel) + ";" + "const waitMs = " + waitMs + ";" + "const dedup = " + dedup + ";" + "const rows = []; const seen = new Set();" + "for (let p = 0; p < pages; p++) {" + " document.querySelectorAll(rowSel).forEach(r => {" + " const row = cols.length ? cols.map(cs => { const c = r.querySelector(cs); return ((c && (c.innerText || c.textContent)) || '').trim(); }) : [((r.innerText || r.textContent) || '').trim()];" + " const key = JSON.stringify(row);" + " if (!dedup || !seen.has(key)) { seen.add(key); rows.push(row); }" + " });" + " if (p < pages - 1 && nextSel) {" + " const btn = [...document.querySelectorAll(nextSel)].find(b => !b.disabled);" + " if (!btn) break;" + " btn.click();" + " await new Promise(r => setTimeout(r, waitMs));" + " }" + "}" + "return rows;" + "})()"; const rows = await browser.runJs(script) as unknown[][]; const labels = columns.map(c => c.label).filter(Boolean); ctx.log("已采集 " + (rows?.length ?? 0) + " 行" + (labels.length ? "(列:" + labels.join("、") + ")" : ""), "info", node.id); return rows ?? []; }); register("WebBrowser_StartWeblistening", async (ctx, node, args) => { const browser = await crawlSession(ctx, args); const started = browser.startNetworkListening?.(); ctx.log(started ? "网页监听已开启" : "网页监听已在运行中", "info", node.id); return { listening: true, started }; }); register("WebBrowser_GetWeblistening", async (ctx, node, args) => { const browser = await crawlSession(ctx, args); const url = String(args.url ?? "").trim(); const wildcard = String(args.iswildcard ?? "") === "1" || String(args.iswildcard ?? "") === "true"; const resourceType = String(args.web_resource ?? "").trim(); const captures = (await browser.getNetworkCaptures?.({ url, wildcard, resourceType })) ?? []; ctx.log("已获取 " + captures.length + " 条监听记录", "info", node.id); return captures; }); register("WebBrowser_StopWeblistening", async (ctx, node, args) => { const browser = await crawlSession(ctx, args); const stopped = browser.stopNetworkListening?.(); ctx.log(stopped ? "网页监听已停止" : "网页监听未在运行", "info", node.id); return { listening: false, stopped }; });