PRA-only-demo/src/main/capabilities/web-crawl.ts

import { browserSession, externalBrowserSession } from "../browser/session";
import { register } from "../engine/registry";
import { RuntimeError, type BrowserLike, type RunContext } from "../engine/types";

/**
 * 采集/监听组件的浏览器会话解析：
 * 1. 流程中「打开网页」已建立的会话优先（内置或外部）；
 * 2. 否则若节点 browserType 填 chrome/edge → 拉起对应外部浏览器；
 * 3. 其余情况退回内置运行浏览器。
 */
async function crawlSession(ctx: RunContext, args: Record<string, unknown> = {}): Promise<BrowserLike> {
  if (ctx.browser) return ctx.browser;
  const browserType = String(args.browserType ?? "").trim().toLowerCase();
  if (browserType === "chrome" || browserType === "edge") {
    return externalBrowserSession({ mode: "launch", browserType, port: 9222 });
  }
  return browserSession();
}

const CRAWL_IDS = [
  "WebBrowser_DataCollection", "WebBrowser_DataCollectionV1", "WebBrowser_DataCollectionV2",
  "WebBrowser_DataCollectionV3", "WebBrowser_DataCollectionV4", "WebBrowser_DataCollectionV5",
  "WebBrowser_DataCrawl", "WebBrowser_DataCrawl_web", "WebBrowser_DataCrawl_software",
  "WebBrowser_DataCrawler", "WebBrowser_BatchDataCrawl", "WebBrowser_BatchDataCrawlNew",
  "WebBrowser_BatchDataCrawlNewV1"
];

function parseList(value: unknown): Array<{ selector: string; label: string }> {
  if (value === undefined || value === null || value === "") return [];
  if (Array.isArray(value)) {
    return value.map(item => typeof item === "string" ? { selector: item, label: item } : { selector: String(item.selector ?? ""), label: String(item.label ?? item.selector ?? "") }).filter(item => item.selector);
  }
  const text = String(value).trim();
  if (text.startsWith("[")) {
    try {
      const parsed = JSON.parse(text) as unknown;
      if (Array.isArray(parsed)) return parseList(parsed);
    } catch { /* fall through */ }
  }
  return [{ selector: text, label: text }];
}

function firstOf(args: Record<string, unknown>, keys: string[]): unknown {
  for (const key of keys) {
    if (args[key] !== undefined && args[key] !== null && args[key] !== "") return args[key];
  }
  return undefined;
}

register(CRAWL_IDS, async (ctx, node, args) => {
  const browser = await crawlSession(ctx, args);
  const rowSelector = firstOf(args, ["selector_page", "page_element", "uiSelector_page", "row_selector", "selectorCode", "rowSelector"]);
  if (!rowSelector) {
    throw new RuntimeError("validation_error", "行选择器为空", { nodeId: node.id, componentId: node.componentId, recommendation: "填写每一行数据的选择器，例如 css:table tbody tr" });
  }
  const rowSel = String(rowSelector).replace(/^css:/, "").trim();
  const itemsRaw = firstOf(args, ["collect_items", "config", "columns", "column_selectors"]);
  const columns = parseList(itemsRaw);
  const pages = Math.max(1, Number(firstOf(args, ["collect_pages", "page_num", "pages", "repeat_count", "delay_pages"]) ?? 1) || 1);
  const nextRaw = firstOf(args, ["selector_more", "next_selector", "uiSelector_num"]);
  const nextSel = nextRaw ? String(nextRaw).replace(/^css:/, "").trim() : "";
  const waitMs = Math.max(200, Number(firstOf(args, ["wait_time_page", "waitMs"]) ?? 1200) || 1200);
  const dedup = String(firstOf(args, ["deduplication"]) ?? "") === "1" || String(firstOf(args, ["deduplication"]) ?? "") === "true";

  const script = "(async () => {" +
    "const rowSel = " + JSON.stringify(rowSel) + ";" +
    "const cols = " + JSON.stringify(columns.map(c => c.selector)) + ";" +
    "const pages = " + pages + ";" +
    "const nextSel = " + JSON.stringify(nextSel) + ";" +
    "const waitMs = " + waitMs + ";" +
    "const dedup = " + dedup + ";" +
    "const rows = []; const seen = new Set();" +
    "for (let p = 0; p < pages; p++) {" +
    "  document.querySelectorAll(rowSel).forEach(r => {" +
    "    const row = cols.length ? cols.map(cs => { const c = r.querySelector(cs); return ((c && (c.innerText || c.textContent)) || '').trim(); }) : [((r.innerText || r.textContent) || '').trim()];" +
    "    const key = JSON.stringify(row);" +
    "    if (!dedup || !seen.has(key)) { seen.add(key); rows.push(row); }" +
    "  });" +
    "  if (p < pages - 1 && nextSel) {" +
    "    const btn = [...document.querySelectorAll(nextSel)].find(b => !b.disabled);" +
    "    if (!btn) break;" +
    "    btn.click();" +
    "    await new Promise(r => setTimeout(r, waitMs));" +
    "  }" +
    "}" +
    "return rows;" +
    "})()";
  const rows = await browser.runJs(script) as unknown[][];
  const labels = columns.map(c => c.label).filter(Boolean);
  ctx.log("已采集 " + (rows?.length ?? 0) + " 行" + (labels.length ? "（列：" + labels.join("、") + "）" : ""), "info", node.id);
  return rows ?? [];
});

register("WebBrowser_StartWeblistening", async (ctx, node, args) => {
  const browser = await crawlSession(ctx, args);
  const started = browser.startNetworkListening?.();
  ctx.log(started ? "网页监听已开启" : "网页监听已在运行中", "info", node.id);
  return { listening: true, started };
});

register("WebBrowser_GetWeblistening", async (ctx, node, args) => {
  const browser = await crawlSession(ctx, args);
  const url = String(args.url ?? "").trim();
  const wildcard = String(args.iswildcard ?? "") === "1" || String(args.iswildcard ?? "") === "true";
  const resourceType = String(args.web_resource ?? "").trim();
  const captures = (await browser.getNetworkCaptures?.({ url, wildcard, resourceType })) ?? [];
  ctx.log("已获取 " + captures.length + " 条监听记录", "info", node.id);
  return captures;
});

register("WebBrowser_StopWeblistening", async (ctx, node, args) => {
  const browser = await crawlSession(ctx, args);
  const stopped = browser.stopNetworkListening?.();
  ctx.log(stopped ? "网页监听已停止" : "网页监听未在运行", "info", node.id);
  return { listening: false, stopped };
});