import { promises as fs } from "node:fs"; import path from "node:path"; import { register } from "../engine/registry"; import { RuntimeError, type CapabilityHandler } from "../engine/types"; type Table = unknown[][]; type IniDocument = Map>; function value(args: Record, keys: string[], fallback?: unknown): unknown { for (const key of keys) { const candidate = args[key]; if (candidate !== undefined && candidate !== null && candidate !== "") return candidate; } return fallback; } function text(candidate: unknown): string { return candidate == null ? "" : String(candidate); } function parse(candidate: unknown): unknown { if (typeof candidate !== "string") return candidate; const source = candidate.trim(); if (!source) return candidate; try { return JSON.parse(source); } catch { return candidate; } } function list(candidate: unknown): unknown[] { const parsed = parse(candidate); if (Array.isArray(parsed)) return parsed; if (parsed === undefined || parsed === null || parsed === "") return []; if (typeof parsed === "string") return parsed.split(/[,\uFF0C\n]/).map((item) => item.trim()).filter(Boolean); return [parsed]; } function table(candidate: unknown): Table { const parsed = parse(candidate); if (Array.isArray(parsed)) { if (!parsed.length) return []; if (parsed.every((row) => Array.isArray(row))) return parsed.map((row) => [...row as unknown[]]); if (parsed.every((row) => row && typeof row === "object" && !Array.isArray(row))) { const headers = [...new Set(parsed.flatMap((row) => Object.keys(row as Record)))]; return [headers, ...parsed.map((row) => headers.map((header) => (row as Record)[header]))]; } return parsed.map((item) => [item]); } if (parsed && typeof parsed === "object") { const source = parsed as Record; const headers = Object.keys(source); const columns = headers.map((header) => list(source[header])); const rowCount = Math.max(0, ...columns.map((column) => column.length)); return [headers, ...Array.from({ length: rowCount }, (_, row) => columns.map((column) => column[row]))]; } return parsed === undefined || parsed === null || parsed === "" ? [] : [[parsed]]; } function columnNames(candidate: unknown): string[] { return list(candidate).map((item) => text(item)).filter(Boolean); } function columnIndex(headers: unknown[], name: unknown): number { const source = text(name).trim(); const byName = headers.findIndex((header) => text(header) === source); if (byName >= 0) return byName; if (/^\d+$/.test(source)) return Math.max(0, Number(source) - 1); if (/^[A-Za-z]+$/.test(source)) { let result = 0; for (const character of source.toUpperCase()) result = result * 26 + character.charCodeAt(0) - 64; return Math.max(0, result - 1); } return -1; } function selectedIndexes(rows: Table, candidate: unknown): number[] { const headers = rows[0] ?? []; const requested = columnNames(candidate); return requested.map((name) => columnIndex(headers, name)).filter((index) => index >= 0); } function comparable(candidate: unknown): string | number | boolean | null { if (candidate === null || candidate === undefined) return null; if (typeof candidate === "number" || typeof candidate === "boolean") return candidate; const source = text(candidate).trim().replace(/^\(+|\)+$/g, "").trim(); if (/^-?\d+(\.\d+)?$/.test(source)) return Number(source); if (source === "true" || source === "false") return source === "true"; return source.replace(/^['"]|['"]$/g, ""); } function equal(left: unknown, right: unknown): boolean { return comparable(left) === comparable(right); } function collection(candidate: unknown): Set | unknown[] { if (candidate instanceof Set || Array.isArray(candidate)) return candidate; return list(candidate); } register("Collection_Create", () => []); register("Collection_HasItem", (_ctx, _node, args) => { const source = collection(args.collection); return source instanceof Set ? source.has(args.item) : source.some((item) => equal(item, args.item)); }); register("Collection_Count", (_ctx, _node, args) => { const source = collection(args.collection); return source instanceof Set ? source.size : source.length; }); register("Collection_Add", (_ctx, _node, args) => { const source = collection(args.collection); if (source instanceof Set) source.add(args.value); else if (!source.some((item) => equal(item, args.value))) source.push(args.value); return source; }); register("Collection_Remove", (_ctx, _node, args) => { const source = collection(args.collection); if (source instanceof Set) source.delete(args.value); else { const index = source.findIndex((item) => equal(item, args.value)); if (index >= 0) source.splice(index, 1); } return source; }); function requiredFile(args: Record, nodeId: string, componentId: string): string { const candidate = text(args.file).trim(); if (!candidate) { throw new RuntimeError("validation_error", "INI file path is empty", { nodeId, componentId, arg: "file", recommendation: "Select an INI file" }); } return path.resolve(candidate); } function parseIni(source: string): IniDocument { const document: IniDocument = new Map([["", new Map()]]); let section = ""; for (const rawLine of source.replace(/^\uFEFF/, "").split(/\r?\n/)) { const line = rawLine.trim(); if (!line || line.startsWith(";") || line.startsWith("#")) continue; const sectionMatch = line.match(/^\[([^\]]+)]$/); if (sectionMatch) { section = sectionMatch[1].trim(); if (!document.has(section)) document.set(section, new Map()); continue; } const separator = rawLine.indexOf("="); if (separator < 0) continue; document.get(section)!.set(rawLine.slice(0, separator).trim(), rawLine.slice(separator + 1).trim()); } return document; } function serializeIni(document: IniDocument): string { const lines: string[] = []; for (const [section, entries] of document) { if (section) { if (lines.length) lines.push(""); lines.push(`[${section}]`); } for (const [key, entryValue] of entries) lines.push(`${key}=${entryValue}`); } return `${lines.join("\r\n")}\r\n`; } async function loadIni(filePath: string): Promise { try { return parseIni(await fs.readFile(filePath, "utf8")); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return parseIni(""); throw error; } } async function saveIni(filePath: string, document: IniDocument): Promise { await fs.mkdir(path.dirname(filePath), { recursive: true }); await fs.writeFile(filePath, serializeIni(document), "utf8"); } register("Ini_Read", async (_ctx, node, args) => { const document = await loadIni(requiredFile(args, node.id, node.componentId)); return document.get(text(args.section))?.get(text(args.key)) ?? text(args.defaultValue); }); register("Ini_Write", async (_ctx, node, args) => { const filePath = requiredFile(args, node.id, node.componentId); const document = await loadIni(filePath); const section = text(args.section); if (!document.has(section)) document.set(section, new Map()); document.get(section)!.set(text(args.key), text(args.value)); await saveIni(filePath, document); return filePath; }); register("Ini_ListSection", async (_ctx, node, args) => { const document = await loadIni(requiredFile(args, node.id, node.componentId)); return [...document.keys()].filter(Boolean); }); register("Ini_ListKey", async (_ctx, node, args) => { const document = await loadIni(requiredFile(args, node.id, node.componentId)); return [...(document.get(text(args.section))?.keys() ?? [])]; }); register("Ini_DeleteKey", async (_ctx, node, args) => { const filePath = requiredFile(args, node.id, node.componentId); const document = await loadIni(filePath); const removed = document.get(text(args.section))?.delete(text(args.key)) ?? false; await saveIni(filePath, document); return removed; }); register("Ini_DeleteSection", async (_ctx, node, args) => { const filePath = requiredFile(args, node.id, node.componentId); const document = await loadIni(filePath); const removed = document.delete(text(args.section)); await saveIni(filePath, document); return removed; }); register("Log_Info", (ctx, node, args) => { const message = text(args.msg); ctx.log(message, "info", node.id); return message; }); register("Log_Error", (ctx, node, args) => { const message = text(args.msg); ctx.log(message, "error", node.id); return message; }); register("ExceptionUtil_Throws", (_ctx, node, args) => { throw new RuntimeError("user_exception", text(args.msg) || "User exception", { nodeId: node.id, componentId: node.componentId }); }); const createTable: CapabilityHandler = (_ctx, _node, args) => { const rows = table(value(args, ["data"])); const columns = columnNames(args.columnList); if (!columns.length) return rows; if (!rows.length) return [columns]; if (rows[0].length !== columns.length) throw new RuntimeError("validation_error", "Column count does not match data"); return [columns, ...rows]; }; register(["Bivartable_CreateDf", "Bivartable_CreateDfV1", "Bivartable_DoCreate"], createTable); function aggregate(args: Record, operation: "max" | "min" | "sum" | "avg"): unknown { const rows = table(args.df); if (rows.length < 2) return 0; const field = operation === "max" ? "maxvaluecols" : operation === "min" ? "minvaluecols" : operation === "sum" ? "sumvaluecols" : "avgvaluecols"; const indexes = selectedIndexes(rows, args[field]); if (!indexes.length) throw new RuntimeError("validation_error", "No aggregate column was found"); const groups = selectedIndexes(rows, args.groupby); const calculate = (body: unknown[][], index: number): number => { const numbers = body.map((row) => Number(row[index])).filter(Number.isFinite); if (!numbers.length) return 0; if (operation === "max") return Math.max(...numbers); if (operation === "min") return Math.min(...numbers); const sum = numbers.reduce((total, current) => total + current, 0); return operation === "avg" ? sum / numbers.length : sum; }; if (!groups.length) { const results = indexes.map((index) => calculate(rows.slice(1), index)); return results.length === 1 ? results[0] : Object.fromEntries(indexes.map((index, position) => [text(rows[0][index]), results[position]])); } const buckets = new Map(); for (const row of rows.slice(1)) { const key = JSON.stringify(groups.map((index) => row[index])); if (!buckets.has(key)) buckets.set(key, []); buckets.get(key)!.push(row); } return [ [...groups.map((index) => rows[0][index]), ...indexes.map((index) => rows[0][index])], ...[...buckets.entries()].map(([key, body]) => [...JSON.parse(key), ...indexes.map((index) => calculate(body, index))]) ]; } register("Bivartable_GetMaxValue", (_ctx, _node, args) => aggregate(args, "max")); register("Bivartable_GetMinValue", (_ctx, _node, args) => aggregate(args, "min")); register("Bivartable_GetSumValue", (_ctx, _node, args) => aggregate(args, "sum")); register("Bivartable_GetAvgValue", (_ctx, _node, args) => aggregate(args, "avg")); register("Bivartable_SummaryCalculationDf", (_ctx, _node, args) => { const method = text(args.method).toLowerCase(); const operation = method.includes("max") ? "max" : method.includes("min") ? "min" : method.includes("avg") || method.includes("mean") ? "avg" : "sum"; const field = operation === "max" ? "maxvaluecols" : operation === "min" ? "minvaluecols" : operation === "avg" ? "avgvaluecols" : "sumvaluecols"; return aggregate({ ...args, [field]: args.name }, operation); }); register("Bivartable_DeleteDuplicateRow", (_ctx, _node, args) => { const rows = table(args.df); if (rows.length < 2) return rows; const requested = text(args.change_type) === "name" ? args.col_name : args.col_num; const indexes = selectedIndexes(rows, requested); const compared = indexes.length ? indexes : rows[0].map((_item, index) => index); const keys = rows.slice(1).map((row) => JSON.stringify(compared.map((index) => row[index]))); const counts = new Map(); for (const key of keys) counts.set(key, (counts.get(key) ?? 0) + 1); const mode = text(args.mode).toLowerCase(); const seen = new Set(); const body = rows.slice(1).filter((_row, index) => { const key = keys[index]; if (mode === "false" || mode === "none" || mode.includes("all")) return counts.get(key) === 1; if (mode === "last") return keys.lastIndexOf(key) === index; if (seen.has(key)) return false; seen.add(key); return true; }); return [rows[0], ...body]; }); function mergeTables(args: Record): Table { const left = table(args.df1); const right = table(args.df2); if (!left.length) return right; if (!right.length) return left; const mode = text(value(args, ["merge_type", "howx"], "inner")).toLowerCase(); const common = left[0].filter((header) => right[0].some((candidate) => text(candidate) === text(header))).map(text); const sameKeys = columnNames(value(args, ["on_name", "onx"])); const leftKeys = columnNames(value(args, ["on_left", "left_onx"], sameKeys.length ? sameKeys : common)); const rightKeys = columnNames(value(args, ["on_right", "right_onx"], sameKeys.length ? sameKeys : common)); if (!leftKeys.length || leftKeys.length !== rightKeys.length) throw new RuntimeError("validation_error", "Invalid table join keys"); const leftIndexes = leftKeys.map((key) => columnIndex(left[0], key)); const rightIndexes = rightKeys.map((key) => columnIndex(right[0], key)); if ([...leftIndexes, ...rightIndexes].some((index) => index < 0)) throw new RuntimeError("validation_error", "Table join key does not exist"); const rightDataIndexes = right[0].map((_header, index) => index).filter((index) => !rightIndexes.includes(index)); const outputHeaders = [...left[0], ...rightDataIndexes.map((index) => { const header = text(right[0][index]); return left[0].some((candidate) => text(candidate) === header) ? `${header}_right` : right[0][index]; })]; const rightBuckets = new Map(); for (const row of right.slice(1)) { const key = JSON.stringify(rightIndexes.map((index) => row[index])); if (!rightBuckets.has(key)) rightBuckets.set(key, []); rightBuckets.get(key)!.push(row); } const matchedRight = new Set(); const output: Table = [outputHeaders]; for (const row of left.slice(1)) { const matches = rightBuckets.get(JSON.stringify(leftIndexes.map((index) => row[index]))) ?? []; if (matches.length) { for (const match of matches) { matchedRight.add(match); output.push([...row, ...rightDataIndexes.map((index) => match[index])]); } } else if (mode === "left" || mode === "outer") { output.push([...row, ...rightDataIndexes.map(() => null)]); } } if (mode === "right" || mode === "outer") { for (const row of right.slice(1)) { if (matchedRight.has(row)) continue; const leftPart = left[0].map((header) => { const keyPosition = leftKeys.indexOf(text(header)); return keyPosition >= 0 ? row[rightIndexes[keyPosition]] : null; }); output.push([...leftPart, ...rightDataIndexes.map((index) => row[index])]); } } return output; } register(["Bivartable_MergeDf", "Bivartable_DoMerge"], (_ctx, _node, args) => mergeTables(args)); function predicate(operator: string, expected: unknown): (actual: unknown) => boolean { const normalized = operator.trim().toLowerCase(); if (["2", "!=", "<>", "ne"].includes(normalized)) return (actual) => !equal(actual, expected); if (["3", ">", "gt"].includes(normalized)) return (actual) => Number(actual) > Number(expected); if (["4", ">=", "gte"].includes(normalized)) return (actual) => Number(actual) >= Number(expected); if (["5", "<", "lt"].includes(normalized)) return (actual) => Number(actual) < Number(expected); if (["6", "<=", "lte"].includes(normalized)) return (actual) => Number(actual) <= Number(expected); if (["7", "contains"].includes(normalized)) return (actual) => text(actual).includes(text(expected)); if (["8", "not contains"].includes(normalized)) return (actual) => !text(actual).includes(text(expected)); if (["9", "starts", "startswith"].includes(normalized)) return (actual) => text(actual).startsWith(text(expected)); if (["10", "ends", "endswith"].includes(normalized)) return (actual) => text(actual).endsWith(text(expected)); return (actual) => equal(actual, expected); } function expressionFilter(rows: Table, expression: string): unknown[][] { const conditions = [...expression.matchAll(/(?:\w+\s*)?\[\s*['"]([^'"]+)['"]\s*]\s*(==|!=|>=|<=|>|<)\s*([^&|]+)/g)]; if (!conditions.length) throw new RuntimeError("validation_error", "Unsupported filter expression"); const operators = [...expression.matchAll(/\s*([&|])\s*/g)].map((match) => match[1]); return rows.slice(1).filter((row) => { const results = conditions.map((match) => { const index = columnIndex(rows[0], match[1]); return index >= 0 && predicate(match[2], comparable(match[3]))(row[index]); }); return results.slice(1).reduce((result, current, index) => operators[index] === "|" ? result || current : result && current, results[0]); }); } function filterTable(args: Record): Table { const rows = table(args.df); if (rows.length < 2) return rows; const expression = text(value(args, ["rule", "filter"])).trim(); if (expression || text(args.filter_mode).toLowerCase().includes("advanced")) return [rows[0], ...expressionFilter(rows, expression)]; const index = columnIndex(rows[0], args.col_name); if (index < 0) throw new RuntimeError("validation_error", `Filter column does not exist: ${text(args.col_name)}`); const matches = predicate(text(args.choice || "1"), parse(args.criteria)); return [rows[0], ...rows.slice(1).filter((row) => matches(row[index]))]; } register(["Bivartable_FilterDf", "Bivartable_DoFilter"], (_ctx, _node, args) => filterTable(args));