From 4ac9755416076cbb3f9a3f85df10bb3857cfb704 Mon Sep 17 00:00:00 2001 From: yu <191972707@qq.com> Date: Wed, 28 Oct 2026 17:53:44 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=8E=A5=E5=85=A5=E7=BD=91=E9=A1=B5?= =?UTF-8?q?=E6=89=B9=E4=B8=89=E2=80=94=E2=80=94=E6=95=B0=E6=8D=AE=E9=87=87?= =?UTF-8?q?=E9=9B=86=E5=AE=B6=E6=97=8F=EF=BC=8814=20id=EF=BC=89=E4=B8=8E?= =?UTF-8?q?=E7=BD=91=E9=A1=B5=E7=9B=91=E5=90=AC=E4=B8=89=E4=BB=B6=E5=A5=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/browser/session.ts | 59 +++++++++++++++++ src/main/capabilities/index.ts | 1 + src/main/capabilities/web-crawl.ts | 101 +++++++++++++++++++++++++++++ 3 files changed, 161 insertions(+) create mode 100644 src/main/capabilities/web-crawl.ts diff --git a/src/main/browser/session.ts b/src/main/browser/session.ts index 4fbb6ee..2273d93 100644 --- a/src/main/browser/session.ts +++ b/src/main/browser/session.ts @@ -820,6 +820,65 @@ export class BrowserSession implements BrowserLike { return true; } + startNetworkListening(): boolean { + const flags = globalThis as { __rpaNetListening?: boolean; __rpaNetListener?: unknown; __rpaNetCaptures?: Array<{ requestId: string; url: string; status: number; mimeType: string }> }; + if (!this.win.isDestroyed() && !this.win.webContents.debugger.isAttached()) this.win.webContents.debugger.attach("1.3"); + this.cdp("Network.enable").catch(() => {}); + if (flags.__rpaNetListening) return false; + flags.__rpaNetListening = true; + flags.__rpaNetCaptures = []; + const listener = (_event: unknown, method: string, params: Record) => { + if (method !== "Network.responseReceived") return; + const response = params.response as { url?: string; status?: number; mimeType?: string } | undefined; + const request = params.requestId as string | undefined; + if (!response || !request) return; + flags.__rpaNetCaptures?.push({ requestId: request, url: response.url ?? "", status: response.status ?? 0, mimeType: response.mimeType ?? "" }); + if ((flags.__rpaNetCaptures?.length ?? 0) > 2000) flags.__rpaNetCaptures?.shift(); + }; + flags.__rpaNetListener = listener; + this.win.webContents.debugger.on("message", listener); + return true; + } + + async getNetworkCaptures(options: { url?: string; wildcard?: boolean; resourceType?: string; body?: boolean } = {}): Promise>> { + const flags = globalThis as { __rpaNetListening?: boolean; __rpaNetCaptures?: Array<{ requestId: string; url: string; status: number; mimeType: string }> }; + let captures = flags.__rpaNetCaptures ?? []; + if (options.resourceType) { + const want = options.resourceType.toLowerCase(); + captures = captures.filter(c => c.mimeType.toLowerCase().includes(want)); + } + if (options.url) { + const keyword = options.url; + captures = options.wildcard + ? captures.filter(c => new RegExp("^" + keyword.replace(/[.*+?^${}()|[\]\\]/g, "\\ async currentUrl(): Promise {").replace(/\*/g, ".*")).test(c.url)) + : captures.filter(c => c.url.includes(keyword)); + } + const result: Array> = []; + for (const c of captures.slice(-200)) { + const item: Record = { url: c.url, status: c.status, mimeType: c.mimeType }; + if (options.body && /json|text|javascript/i.test(c.mimeType)) { + try { + const body = await this.cdp<{ body?: string }>("Network.getResponseBody", { requestId: c.requestId }); + item.body = body.body ?? ""; + } catch { item.body = ""; } + } + result.push(item); + } + return result; + } + + stopNetworkListening(): boolean { + const flags = globalThis as { __rpaNetListening?: boolean; __rpaNetListener?: unknown }; + if (!flags.__rpaNetListening) return false; + if (flags.__rpaNetListener) { + this.win.webContents.debugger.removeListener("message", flags.__rpaNetListener as never); + } + flags.__rpaNetListening = false; + flags.__rpaNetListener = undefined; + this.cdp("Network.disable").catch(() => {}); + return true; + } + async currentUrl(): Promise { return this.win.webContents.getURL(); } diff --git a/src/main/capabilities/index.ts b/src/main/capabilities/index.ts index e1b6611..2d6d010 100644 --- a/src/main/capabilities/index.ts +++ b/src/main/capabilities/index.ts @@ -11,6 +11,7 @@ import "./office"; import "./desktop"; import "./desktop-uia"; import "./web-batch2"; +import "./web-crawl"; import "./pdf"; import "./word"; import "./word-extended"; diff --git a/src/main/capabilities/web-crawl.ts b/src/main/capabilities/web-crawl.ts new file mode 100644 index 0000000..344d4c5 --- /dev/null +++ b/src/main/capabilities/web-crawl.ts @@ -0,0 +1,101 @@ +import { browserSession } from "../browser/session"; +import { register } from "../engine/registry"; +import { RuntimeError } from "../engine/types"; + +const CRAWL_IDS = [ + "WebBrowser_DataCollection", "WebBrowser_DataCollectionV1", "WebBrowser_DataCollectionV2", + "WebBrowser_DataCollectionV3", "WebBrowser_DataCollectionV4", "WebBrowser_DataCollectionV5", + "WebBrowser_DataCrawl", "WebBrowser_DataCrawl_web", "WebBrowser_DataCrawl_software", + "WebBrowser_DataCrawler", "WebBrowser_BatchDataCrawl", "WebBrowser_BatchDataCrawlNew", + "WebBrowser_BatchDataCrawlNewV1" +]; + +function parseList(value: unknown): Array<{ selector: string; label: string }> { + if (value === undefined || value === null || value === "") return []; + if (Array.isArray(value)) { + return value.map(item => typeof item === "string" ? { selector: item, label: item } : { selector: String(item.selector ?? ""), label: String(item.label ?? item.selector ?? "") }).filter(item => item.selector); + } + const text = String(value).trim(); + if (text.startsWith("[")) { + try { + const parsed = JSON.parse(text) as unknown; + if (Array.isArray(parsed)) return parseList(parsed); + } catch { /* fall through */ } + } + return [{ selector: text, label: text }]; +} + +function firstOf(args: Record, keys: string[]): unknown { + for (const key of keys) { + if (args[key] !== undefined && args[key] !== null && args[key] !== "") return args[key]; + } + return undefined; +} + +register(CRAWL_IDS, async (ctx, node, args) => { + const browser = await browserSession(); + const rowSelector = firstOf(args, ["selector_page", "page_element", "uiSelector_page", "row_selector", "selectorCode", "rowSelector"]); + if (!rowSelector) { + throw new RuntimeError("validation_error", "行选择器为空", { nodeId: node.id, componentId: node.componentId, recommendation: "填写每一行数据的选择器，例如 css:table tbody tr" }); + } + const rowSel = String(rowSelector).replace(/^css:/, "").trim(); + const itemsRaw = firstOf(args, ["collect_items", "config", "columns", "column_selectors"]); + const columns = parseList(itemsRaw); + const pages = Math.max(1, Number(firstOf(args, ["collect_pages", "page_num", "pages", "repeat_count", "delay_pages"]) ?? 1) || 1); + const nextRaw = firstOf(args, ["selector_more", "next_selector", "uiSelector_num"]); + const nextSel = nextRaw ? String(nextRaw).replace(/^css:/, "").trim() : ""; + const waitMs = Math.max(200, Number(firstOf(args, ["wait_time_page", "waitMs"]) ?? 1200) || 1200); + const dedup = String(firstOf(args, ["deduplication"]) ?? "") === "1" || String(firstOf(args, ["deduplication"]) ?? "") === "true"; + + const script = "(async () => {" + + "const rowSel = " + JSON.stringify(rowSel) + ";" + + "const cols = " + JSON.stringify(columns.map(c => c.selector)) + ";" + + "const pages = " + pages + ";" + + "const nextSel = " + JSON.stringify(nextSel) + ";" + + "const waitMs = " + waitMs + ";" + + "const dedup = " + dedup + ";" + + "const rows = []; const seen = new Set();" + + "for (let p = 0; p < pages; p++) {" + + " document.querySelectorAll(rowSel).forEach(r => {" + + " const row = cols.length ? cols.map(cs => { const c = r.querySelector(cs); return ((c && (c.innerText || c.textContent)) || '').trim(); }) : [((r.innerText || r.textContent) || '').trim()];" + + " const key = JSON.stringify(row);" + + " if (!dedup || !seen.has(key)) { seen.add(key); rows.push(row); }" + + " });" + + " if (p < pages - 1 && nextSel) {" + + " const btn = [...document.querySelectorAll(nextSel)].find(b => !b.disabled);" + + " if (!btn) break;" + + " btn.click();" + + " await new Promise(r => setTimeout(r, waitMs));" + + " }" + + "}" + + "return rows;" + + "})()"; + const rows = await browser.runJs(script) as unknown[][]; + const labels = columns.map(c => c.label).filter(Boolean); + ctx.log("已采集 " + (rows?.length ?? 0) + " 行" + (labels.length ? "（列：" + labels.join("、") + "）" : ""), "info", node.id); + return rows ?? []; +}); + +register("WebBrowser_StartWeblistening", async (ctx, node) => { + const browser = await browserSession(); + const started = browser.startNetworkListening(); + ctx.log(started ? "网页监听已开启" : "网页监听已在运行中", "info", node.id); + return { listening: true, started }; +}); + +register("WebBrowser_GetWeblistening", async (ctx, node, args) => { + const browser = await browserSession(); + const url = String(args.url ?? "").trim(); + const wildcard = String(args.iswildcard ?? "") === "1" || String(args.iswildcard ?? "") === "true"; + const resourceType = String(args.web_resource ?? "").trim(); + const captures = await browser.getNetworkCaptures({ url, wildcard, resourceType }); + ctx.log("已获取 " + captures.length + " 条监听记录", "info", node.id); + return captures; +}); + +register("WebBrowser_StopWeblistening", async (ctx, node) => { + const browser = await browserSession(); + const stopped = browser.stopNetworkListening(); + ctx.log(stopped ? "网页监听已停止" : "网页监听未在运行", "info", node.id); + return { listening: false, stopped }; +});