feat: 接入网页批三——数据采集家族（14 id）与网页监听三件套
This commit is contained in:
yu
parent
2ce0c13f7b
commit
4ac9755416
@ -820,6 +820,65 @@ export class BrowserSession implements BrowserLike {
|
||||
return true;
|
||||
}
|
||||
|
||||
startNetworkListening(): boolean {
|
||||
const flags = globalThis as { __rpaNetListening?: boolean; __rpaNetListener?: unknown; __rpaNetCaptures?: Array<{ requestId: string; url: string; status: number; mimeType: string }> };
|
||||
if (!this.win.isDestroyed() && !this.win.webContents.debugger.isAttached()) this.win.webContents.debugger.attach("1.3");
|
||||
this.cdp("Network.enable").catch(() => {});
|
||||
if (flags.__rpaNetListening) return false;
|
||||
flags.__rpaNetListening = true;
|
||||
flags.__rpaNetCaptures = [];
|
||||
const listener = (_event: unknown, method: string, params: Record<string, unknown>) => {
|
||||
if (method !== "Network.responseReceived") return;
|
||||
const response = params.response as { url?: string; status?: number; mimeType?: string } | undefined;
|
||||
const request = params.requestId as string | undefined;
|
||||
if (!response || !request) return;
|
||||
flags.__rpaNetCaptures?.push({ requestId: request, url: response.url ?? "", status: response.status ?? 0, mimeType: response.mimeType ?? "" });
|
||||
if ((flags.__rpaNetCaptures?.length ?? 0) > 2000) flags.__rpaNetCaptures?.shift();
|
||||
};
|
||||
flags.__rpaNetListener = listener;
|
||||
this.win.webContents.debugger.on("message", listener);
|
||||
return true;
|
||||
}
|
||||
|
||||
async getNetworkCaptures(options: { url?: string; wildcard?: boolean; resourceType?: string; body?: boolean } = {}): Promise<Array<Record<string, unknown>>> {
|
||||
const flags = globalThis as { __rpaNetListening?: boolean; __rpaNetCaptures?: Array<{ requestId: string; url: string; status: number; mimeType: string }> };
|
||||
let captures = flags.__rpaNetCaptures ?? [];
|
||||
if (options.resourceType) {
|
||||
const want = options.resourceType.toLowerCase();
|
||||
captures = captures.filter(c => c.mimeType.toLowerCase().includes(want));
|
||||
}
|
||||
if (options.url) {
|
||||
const keyword = options.url;
|
||||
captures = options.wildcard
|
||||
? captures.filter(c => new RegExp("^" + keyword.replace(/[.*+?^${}()|[\]\\]/g, "\\ async currentUrl(): Promise<string> {").replace(/\*/g, ".*")).test(c.url))
|
||||
: captures.filter(c => c.url.includes(keyword));
|
||||
}
|
||||
const result: Array<Record<string, unknown>> = [];
|
||||
for (const c of captures.slice(-200)) {
|
||||
const item: Record<string, unknown> = { url: c.url, status: c.status, mimeType: c.mimeType };
|
||||
if (options.body && /json|text|javascript/i.test(c.mimeType)) {
|
||||
try {
|
||||
const body = await this.cdp<{ body?: string }>("Network.getResponseBody", { requestId: c.requestId });
|
||||
item.body = body.body ?? "";
|
||||
} catch { item.body = ""; }
|
||||
}
|
||||
result.push(item);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
stopNetworkListening(): boolean {
|
||||
const flags = globalThis as { __rpaNetListening?: boolean; __rpaNetListener?: unknown };
|
||||
if (!flags.__rpaNetListening) return false;
|
||||
if (flags.__rpaNetListener) {
|
||||
this.win.webContents.debugger.removeListener("message", flags.__rpaNetListener as never);
|
||||
}
|
||||
flags.__rpaNetListening = false;
|
||||
flags.__rpaNetListener = undefined;
|
||||
this.cdp("Network.disable").catch(() => {});
|
||||
return true;
|
||||
}
|
||||
|
||||
async currentUrl(): Promise<string> {
|
||||
return this.win.webContents.getURL();
|
||||
}
|
||||
|
||||
@ -11,6 +11,7 @@ import "./office";
|
||||
import "./desktop";
|
||||
import "./desktop-uia";
|
||||
import "./web-batch2";
|
||||
import "./web-crawl";
|
||||
import "./pdf";
|
||||
import "./word";
|
||||
import "./word-extended";
|
||||
|
||||
101
src/main/capabilities/web-crawl.ts
Normal file
101
src/main/capabilities/web-crawl.ts
Normal file
@ -0,0 +1,101 @@
|
||||
import { browserSession } from "../browser/session";
|
||||
import { register } from "../engine/registry";
|
||||
import { RuntimeError } from "../engine/types";
|
||||
|
||||
const CRAWL_IDS = [
|
||||
"WebBrowser_DataCollection", "WebBrowser_DataCollectionV1", "WebBrowser_DataCollectionV2",
|
||||
"WebBrowser_DataCollectionV3", "WebBrowser_DataCollectionV4", "WebBrowser_DataCollectionV5",
|
||||
"WebBrowser_DataCrawl", "WebBrowser_DataCrawl_web", "WebBrowser_DataCrawl_software",
|
||||
"WebBrowser_DataCrawler", "WebBrowser_BatchDataCrawl", "WebBrowser_BatchDataCrawlNew",
|
||||
"WebBrowser_BatchDataCrawlNewV1"
|
||||
];
|
||||
|
||||
function parseList(value: unknown): Array<{ selector: string; label: string }> {
|
||||
if (value === undefined || value === null || value === "") return [];
|
||||
if (Array.isArray(value)) {
|
||||
return value.map(item => typeof item === "string" ? { selector: item, label: item } : { selector: String(item.selector ?? ""), label: String(item.label ?? item.selector ?? "") }).filter(item => item.selector);
|
||||
}
|
||||
const text = String(value).trim();
|
||||
if (text.startsWith("[")) {
|
||||
try {
|
||||
const parsed = JSON.parse(text) as unknown;
|
||||
if (Array.isArray(parsed)) return parseList(parsed);
|
||||
} catch { /* fall through */ }
|
||||
}
|
||||
return [{ selector: text, label: text }];
|
||||
}
|
||||
|
||||
function firstOf(args: Record<string, unknown>, keys: string[]): unknown {
|
||||
for (const key of keys) {
|
||||
if (args[key] !== undefined && args[key] !== null && args[key] !== "") return args[key];
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
register(CRAWL_IDS, async (ctx, node, args) => {
|
||||
const browser = await browserSession();
|
||||
const rowSelector = firstOf(args, ["selector_page", "page_element", "uiSelector_page", "row_selector", "selectorCode", "rowSelector"]);
|
||||
if (!rowSelector) {
|
||||
throw new RuntimeError("validation_error", "行选择器为空", { nodeId: node.id, componentId: node.componentId, recommendation: "填写每一行数据的选择器，例如 css:table tbody tr" });
|
||||
}
|
||||
const rowSel = String(rowSelector).replace(/^css:/, "").trim();
|
||||
const itemsRaw = firstOf(args, ["collect_items", "config", "columns", "column_selectors"]);
|
||||
const columns = parseList(itemsRaw);
|
||||
const pages = Math.max(1, Number(firstOf(args, ["collect_pages", "page_num", "pages", "repeat_count", "delay_pages"]) ?? 1) || 1);
|
||||
const nextRaw = firstOf(args, ["selector_more", "next_selector", "uiSelector_num"]);
|
||||
const nextSel = nextRaw ? String(nextRaw).replace(/^css:/, "").trim() : "";
|
||||
const waitMs = Math.max(200, Number(firstOf(args, ["wait_time_page", "waitMs"]) ?? 1200) || 1200);
|
||||
const dedup = String(firstOf(args, ["deduplication"]) ?? "") === "1" || String(firstOf(args, ["deduplication"]) ?? "") === "true";
|
||||
|
||||
const script = "(async () => {" +
|
||||
"const rowSel = " + JSON.stringify(rowSel) + ";" +
|
||||
"const cols = " + JSON.stringify(columns.map(c => c.selector)) + ";" +
|
||||
"const pages = " + pages + ";" +
|
||||
"const nextSel = " + JSON.stringify(nextSel) + ";" +
|
||||
"const waitMs = " + waitMs + ";" +
|
||||
"const dedup = " + dedup + ";" +
|
||||
"const rows = []; const seen = new Set();" +
|
||||
"for (let p = 0; p < pages; p++) {" +
|
||||
" document.querySelectorAll(rowSel).forEach(r => {" +
|
||||
" const row = cols.length ? cols.map(cs => { const c = r.querySelector(cs); return ((c && (c.innerText || c.textContent)) || '').trim(); }) : [((r.innerText || r.textContent) || '').trim()];" +
|
||||
" const key = JSON.stringify(row);" +
|
||||
" if (!dedup || !seen.has(key)) { seen.add(key); rows.push(row); }" +
|
||||
" });" +
|
||||
" if (p < pages - 1 && nextSel) {" +
|
||||
" const btn = [...document.querySelectorAll(nextSel)].find(b => !b.disabled);" +
|
||||
" if (!btn) break;" +
|
||||
" btn.click();" +
|
||||
" await new Promise(r => setTimeout(r, waitMs));" +
|
||||
" }" +
|
||||
"}" +
|
||||
"return rows;" +
|
||||
"})()";
|
||||
const rows = await browser.runJs(script) as unknown[][];
|
||||
const labels = columns.map(c => c.label).filter(Boolean);
|
||||
ctx.log("已采集 " + (rows?.length ?? 0) + " 行" + (labels.length ? "（列：" + labels.join("、") + "）" : ""), "info", node.id);
|
||||
return rows ?? [];
|
||||
});
|
||||
|
||||
register("WebBrowser_StartWeblistening", async (ctx, node) => {
|
||||
const browser = await browserSession();
|
||||
const started = browser.startNetworkListening();
|
||||
ctx.log(started ? "网页监听已开启" : "网页监听已在运行中", "info", node.id);
|
||||
return { listening: true, started };
|
||||
});
|
||||
|
||||
register("WebBrowser_GetWeblistening", async (ctx, node, args) => {
|
||||
const browser = await browserSession();
|
||||
const url = String(args.url ?? "").trim();
|
||||
const wildcard = String(args.iswildcard ?? "") === "1" || String(args.iswildcard ?? "") === "true";
|
||||
const resourceType = String(args.web_resource ?? "").trim();
|
||||
const captures = await browser.getNetworkCaptures({ url, wildcard, resourceType });
|
||||
ctx.log("已获取 " + captures.length + " 条监听记录", "info", node.id);
|
||||
return captures;
|
||||
});
|
||||
|
||||
register("WebBrowser_StopWeblistening", async (ctx, node) => {
|
||||
const browser = await browserSession();
|
||||
const stopped = browser.stopNetworkListening();
|
||||
ctx.log(stopped ? "网页监听已停止" : "网页监听未在运行", "info", node.id);
|
||||
return { listening: false, stopped };
|
||||
});
|
||||
Loading…
Reference in New Issue
Block a user