102 lines
5.0 KiB
TypeScript
102 lines
5.0 KiB
TypeScript
import { browserSession } from "../browser/session";
|
||
import { register } from "../engine/registry";
|
||
import { RuntimeError } from "../engine/types";
|
||
|
||
const CRAWL_IDS = [
|
||
"WebBrowser_DataCollection", "WebBrowser_DataCollectionV1", "WebBrowser_DataCollectionV2",
|
||
"WebBrowser_DataCollectionV3", "WebBrowser_DataCollectionV4", "WebBrowser_DataCollectionV5",
|
||
"WebBrowser_DataCrawl", "WebBrowser_DataCrawl_web", "WebBrowser_DataCrawl_software",
|
||
"WebBrowser_DataCrawler", "WebBrowser_BatchDataCrawl", "WebBrowser_BatchDataCrawlNew",
|
||
"WebBrowser_BatchDataCrawlNewV1"
|
||
];
|
||
|
||
function parseList(value: unknown): Array<{ selector: string; label: string }> {
|
||
if (value === undefined || value === null || value === "") return [];
|
||
if (Array.isArray(value)) {
|
||
return value.map(item => typeof item === "string" ? { selector: item, label: item } : { selector: String(item.selector ?? ""), label: String(item.label ?? item.selector ?? "") }).filter(item => item.selector);
|
||
}
|
||
const text = String(value).trim();
|
||
if (text.startsWith("[")) {
|
||
try {
|
||
const parsed = JSON.parse(text) as unknown;
|
||
if (Array.isArray(parsed)) return parseList(parsed);
|
||
} catch { /* fall through */ }
|
||
}
|
||
return [{ selector: text, label: text }];
|
||
}
|
||
|
||
function firstOf(args: Record<string, unknown>, keys: string[]): unknown {
|
||
for (const key of keys) {
|
||
if (args[key] !== undefined && args[key] !== null && args[key] !== "") return args[key];
|
||
}
|
||
return undefined;
|
||
}
|
||
|
||
register(CRAWL_IDS, async (ctx, node, args) => {
|
||
const browser = await browserSession();
|
||
const rowSelector = firstOf(args, ["selector_page", "page_element", "uiSelector_page", "row_selector", "selectorCode", "rowSelector"]);
|
||
if (!rowSelector) {
|
||
throw new RuntimeError("validation_error", "行选择器为空", { nodeId: node.id, componentId: node.componentId, recommendation: "填写每一行数据的选择器，例如 css:table tbody tr" });
|
||
}
|
||
const rowSel = String(rowSelector).replace(/^css:/, "").trim();
|
||
const itemsRaw = firstOf(args, ["collect_items", "config", "columns", "column_selectors"]);
|
||
const columns = parseList(itemsRaw);
|
||
const pages = Math.max(1, Number(firstOf(args, ["collect_pages", "page_num", "pages", "repeat_count", "delay_pages"]) ?? 1) || 1);
|
||
const nextRaw = firstOf(args, ["selector_more", "next_selector", "uiSelector_num"]);
|
||
const nextSel = nextRaw ? String(nextRaw).replace(/^css:/, "").trim() : "";
|
||
const waitMs = Math.max(200, Number(firstOf(args, ["wait_time_page", "waitMs"]) ?? 1200) || 1200);
|
||
const dedup = String(firstOf(args, ["deduplication"]) ?? "") === "1" || String(firstOf(args, ["deduplication"]) ?? "") === "true";
|
||
|
||
const script = "(async () => {" +
|
||
"const rowSel = " + JSON.stringify(rowSel) + ";" +
|
||
"const cols = " + JSON.stringify(columns.map(c => c.selector)) + ";" +
|
||
"const pages = " + pages + ";" +
|
||
"const nextSel = " + JSON.stringify(nextSel) + ";" +
|
||
"const waitMs = " + waitMs + ";" +
|
||
"const dedup = " + dedup + ";" +
|
||
"const rows = []; const seen = new Set();" +
|
||
"for (let p = 0; p < pages; p++) {" +
|
||
" document.querySelectorAll(rowSel).forEach(r => {" +
|
||
" const row = cols.length ? cols.map(cs => { const c = r.querySelector(cs); return ((c && (c.innerText || c.textContent)) || '').trim(); }) : [((r.innerText || r.textContent) || '').trim()];" +
|
||
" const key = JSON.stringify(row);" +
|
||
" if (!dedup || !seen.has(key)) { seen.add(key); rows.push(row); }" +
|
||
" });" +
|
||
" if (p < pages - 1 && nextSel) {" +
|
||
" const btn = [...document.querySelectorAll(nextSel)].find(b => !b.disabled);" +
|
||
" if (!btn) break;" +
|
||
" btn.click();" +
|
||
" await new Promise(r => setTimeout(r, waitMs));" +
|
||
" }" +
|
||
"}" +
|
||
"return rows;" +
|
||
"})()";
|
||
const rows = await browser.runJs(script) as unknown[][];
|
||
const labels = columns.map(c => c.label).filter(Boolean);
|
||
ctx.log("已采集 " + (rows?.length ?? 0) + " 行" + (labels.length ? "（列：" + labels.join("、") + "）" : ""), "info", node.id);
|
||
return rows ?? [];
|
||
});
|
||
|
||
register("WebBrowser_StartWeblistening", async (ctx, node) => {
|
||
const browser = await browserSession();
|
||
const started = browser.startNetworkListening();
|
||
ctx.log(started ? "网页监听已开启" : "网页监听已在运行中", "info", node.id);
|
||
return { listening: true, started };
|
||
});
|
||
|
||
register("WebBrowser_GetWeblistening", async (ctx, node, args) => {
|
||
const browser = await browserSession();
|
||
const url = String(args.url ?? "").trim();
|
||
const wildcard = String(args.iswildcard ?? "") === "1" || String(args.iswildcard ?? "") === "true";
|
||
const resourceType = String(args.web_resource ?? "").trim();
|
||
const captures = await browser.getNetworkCaptures({ url, wildcard, resourceType });
|
||
ctx.log("已获取 " + captures.length + " 条监听记录", "info", node.id);
|
||
return captures;
|
||
});
|
||
|
||
register("WebBrowser_StopWeblistening", async (ctx, node) => {
|
||
const browser = await browserSession();
|
||
const stopped = browser.stopNetworkListening();
|
||
ctx.log(stopped ? "网页监听已停止" : "网页监听未在运行", "info", node.id);
|
||
return { listening: false, stopped };
|
||
});
|