393 lines
18 KiB
TypeScript
393 lines
18 KiB
TypeScript
import { promises as fs } from "node:fs";
|
|
import path from "node:path";
|
|
import { register } from "../engine/registry";
|
|
import { RuntimeError, type CapabilityHandler } from "../engine/types";
|
|
|
|
type Table = unknown[][];
|
|
type IniDocument = Map<string, Map<string, string>>;
|
|
|
|
function value(args: Record<string, unknown>, keys: string[], fallback?: unknown): unknown {
|
|
for (const key of keys) {
|
|
const candidate = args[key];
|
|
if (candidate !== undefined && candidate !== null && candidate !== "") return candidate;
|
|
}
|
|
return fallback;
|
|
}
|
|
|
|
function text(candidate: unknown): string {
|
|
return candidate == null ? "" : String(candidate);
|
|
}
|
|
|
|
function parse(candidate: unknown): unknown {
|
|
if (typeof candidate !== "string") return candidate;
|
|
const source = candidate.trim();
|
|
if (!source) return candidate;
|
|
try { return JSON.parse(source); } catch { return candidate; }
|
|
}
|
|
|
|
function list(candidate: unknown): unknown[] {
|
|
const parsed = parse(candidate);
|
|
if (Array.isArray(parsed)) return parsed;
|
|
if (parsed === undefined || parsed === null || parsed === "") return [];
|
|
if (typeof parsed === "string") return parsed.split(/[,\uFF0C\n]/).map((item) => item.trim()).filter(Boolean);
|
|
return [parsed];
|
|
}
|
|
|
|
function table(candidate: unknown): Table {
|
|
const parsed = parse(candidate);
|
|
if (Array.isArray(parsed)) {
|
|
if (!parsed.length) return [];
|
|
if (parsed.every((row) => Array.isArray(row))) return parsed.map((row) => [...row as unknown[]]);
|
|
if (parsed.every((row) => row && typeof row === "object" && !Array.isArray(row))) {
|
|
const headers = [...new Set(parsed.flatMap((row) => Object.keys(row as Record<string, unknown>)))];
|
|
return [headers, ...parsed.map((row) => headers.map((header) => (row as Record<string, unknown>)[header]))];
|
|
}
|
|
return parsed.map((item) => [item]);
|
|
}
|
|
if (parsed && typeof parsed === "object") {
|
|
const source = parsed as Record<string, unknown>;
|
|
const headers = Object.keys(source);
|
|
const columns = headers.map((header) => list(source[header]));
|
|
const rowCount = Math.max(0, ...columns.map((column) => column.length));
|
|
return [headers, ...Array.from({ length: rowCount }, (_, row) => columns.map((column) => column[row]))];
|
|
}
|
|
return parsed === undefined || parsed === null || parsed === "" ? [] : [[parsed]];
|
|
}
|
|
|
|
function columnNames(candidate: unknown): string[] {
|
|
return list(candidate).map((item) => text(item)).filter(Boolean);
|
|
}
|
|
|
|
function columnIndex(headers: unknown[], name: unknown): number {
|
|
const source = text(name).trim();
|
|
const byName = headers.findIndex((header) => text(header) === source);
|
|
if (byName >= 0) return byName;
|
|
if (/^\d+$/.test(source)) return Math.max(0, Number(source) - 1);
|
|
if (/^[A-Za-z]+$/.test(source)) {
|
|
let result = 0;
|
|
for (const character of source.toUpperCase()) result = result * 26 + character.charCodeAt(0) - 64;
|
|
return Math.max(0, result - 1);
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
function selectedIndexes(rows: Table, candidate: unknown): number[] {
|
|
const headers = rows[0] ?? [];
|
|
const requested = columnNames(candidate);
|
|
return requested.map((name) => columnIndex(headers, name)).filter((index) => index >= 0);
|
|
}
|
|
|
|
function comparable(candidate: unknown): string | number | boolean | null {
|
|
if (candidate === null || candidate === undefined) return null;
|
|
if (typeof candidate === "number" || typeof candidate === "boolean") return candidate;
|
|
const source = text(candidate).trim().replace(/^\(+|\)+$/g, "").trim();
|
|
if (/^-?\d+(\.\d+)?$/.test(source)) return Number(source);
|
|
if (source === "true" || source === "false") return source === "true";
|
|
return source.replace(/^['"]|['"]$/g, "");
|
|
}
|
|
|
|
function equal(left: unknown, right: unknown): boolean {
|
|
return comparable(left) === comparable(right);
|
|
}
|
|
|
|
function collection(candidate: unknown): Set<unknown> | unknown[] {
|
|
if (candidate instanceof Set || Array.isArray(candidate)) return candidate;
|
|
return list(candidate);
|
|
}
|
|
|
|
register("Collection_Create", () => []);
|
|
register("Collection_HasItem", (_ctx, _node, args) => {
|
|
const source = collection(args.collection);
|
|
return source instanceof Set ? source.has(args.item) : source.some((item) => equal(item, args.item));
|
|
});
|
|
register("Collection_Count", (_ctx, _node, args) => {
|
|
const source = collection(args.collection);
|
|
return source instanceof Set ? source.size : source.length;
|
|
});
|
|
register("Collection_Add", (_ctx, _node, args) => {
|
|
const source = collection(args.collection);
|
|
if (source instanceof Set) source.add(args.value);
|
|
else if (!source.some((item) => equal(item, args.value))) source.push(args.value);
|
|
return source;
|
|
});
|
|
register("Collection_Remove", (_ctx, _node, args) => {
|
|
const source = collection(args.collection);
|
|
if (source instanceof Set) source.delete(args.value);
|
|
else {
|
|
const index = source.findIndex((item) => equal(item, args.value));
|
|
if (index >= 0) source.splice(index, 1);
|
|
}
|
|
return source;
|
|
});
|
|
|
|
function requiredFile(args: Record<string, unknown>, nodeId: string, componentId: string): string {
|
|
const candidate = text(args.file).trim();
|
|
if (!candidate) {
|
|
throw new RuntimeError("validation_error", "INI file path is empty", {
|
|
nodeId, componentId, arg: "file", recommendation: "Select an INI file"
|
|
});
|
|
}
|
|
return path.resolve(candidate);
|
|
}
|
|
|
|
function parseIni(source: string): IniDocument {
|
|
const document: IniDocument = new Map([["", new Map()]]);
|
|
let section = "";
|
|
for (const rawLine of source.replace(/^\uFEFF/, "").split(/\r?\n/)) {
|
|
const line = rawLine.trim();
|
|
if (!line || line.startsWith(";") || line.startsWith("#")) continue;
|
|
const sectionMatch = line.match(/^\[([^\]]+)]$/);
|
|
if (sectionMatch) {
|
|
section = sectionMatch[1].trim();
|
|
if (!document.has(section)) document.set(section, new Map());
|
|
continue;
|
|
}
|
|
const separator = rawLine.indexOf("=");
|
|
if (separator < 0) continue;
|
|
document.get(section)!.set(rawLine.slice(0, separator).trim(), rawLine.slice(separator + 1).trim());
|
|
}
|
|
return document;
|
|
}
|
|
|
|
function serializeIni(document: IniDocument): string {
|
|
const lines: string[] = [];
|
|
for (const [section, entries] of document) {
|
|
if (section) {
|
|
if (lines.length) lines.push("");
|
|
lines.push(`[${section}]`);
|
|
}
|
|
for (const [key, entryValue] of entries) lines.push(`${key}=${entryValue}`);
|
|
}
|
|
return `${lines.join("\r\n")}\r\n`;
|
|
}
|
|
|
|
async function loadIni(filePath: string): Promise<IniDocument> {
|
|
try { return parseIni(await fs.readFile(filePath, "utf8")); }
|
|
catch (error) {
|
|
if ((error as NodeJS.ErrnoException).code === "ENOENT") return parseIni("");
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async function saveIni(filePath: string, document: IniDocument): Promise<void> {
|
|
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
|
await fs.writeFile(filePath, serializeIni(document), "utf8");
|
|
}
|
|
|
|
register("Ini_Read", async (_ctx, node, args) => {
|
|
const document = await loadIni(requiredFile(args, node.id, node.componentId));
|
|
return document.get(text(args.section))?.get(text(args.key)) ?? text(args.defaultValue);
|
|
});
|
|
register("Ini_Write", async (_ctx, node, args) => {
|
|
const filePath = requiredFile(args, node.id, node.componentId);
|
|
const document = await loadIni(filePath);
|
|
const section = text(args.section);
|
|
if (!document.has(section)) document.set(section, new Map());
|
|
document.get(section)!.set(text(args.key), text(args.value));
|
|
await saveIni(filePath, document);
|
|
return filePath;
|
|
});
|
|
register("Ini_ListSection", async (_ctx, node, args) => {
|
|
const document = await loadIni(requiredFile(args, node.id, node.componentId));
|
|
return [...document.keys()].filter(Boolean);
|
|
});
|
|
register("Ini_ListKey", async (_ctx, node, args) => {
|
|
const document = await loadIni(requiredFile(args, node.id, node.componentId));
|
|
return [...(document.get(text(args.section))?.keys() ?? [])];
|
|
});
|
|
register("Ini_DeleteKey", async (_ctx, node, args) => {
|
|
const filePath = requiredFile(args, node.id, node.componentId);
|
|
const document = await loadIni(filePath);
|
|
const removed = document.get(text(args.section))?.delete(text(args.key)) ?? false;
|
|
await saveIni(filePath, document);
|
|
return removed;
|
|
});
|
|
register("Ini_DeleteSection", async (_ctx, node, args) => {
|
|
const filePath = requiredFile(args, node.id, node.componentId);
|
|
const document = await loadIni(filePath);
|
|
const removed = document.delete(text(args.section));
|
|
await saveIni(filePath, document);
|
|
return removed;
|
|
});
|
|
|
|
register("Log_Info", (ctx, node, args) => {
|
|
const message = text(args.msg);
|
|
ctx.log(message, "info", node.id);
|
|
return message;
|
|
});
|
|
register("Log_Error", (ctx, node, args) => {
|
|
const message = text(args.msg);
|
|
ctx.log(message, "error", node.id);
|
|
return message;
|
|
});
|
|
register("ExceptionUtil_Throws", (_ctx, node, args) => {
|
|
throw new RuntimeError("user_exception", text(args.msg) || "User exception", {
|
|
nodeId: node.id, componentId: node.componentId
|
|
});
|
|
});
|
|
|
|
const createTable: CapabilityHandler = (_ctx, _node, args) => {
|
|
const rows = table(value(args, ["data"]));
|
|
const columns = columnNames(args.columnList);
|
|
if (!columns.length) return rows;
|
|
if (!rows.length) return [columns];
|
|
if (rows[0].length !== columns.length) throw new RuntimeError("validation_error", "Column count does not match data");
|
|
return [columns, ...rows];
|
|
};
|
|
register(["Bivartable_CreateDf", "Bivartable_CreateDfV1", "Bivartable_DoCreate"], createTable);
|
|
|
|
function aggregate(args: Record<string, unknown>, operation: "max" | "min" | "sum" | "avg"): unknown {
|
|
const rows = table(args.df);
|
|
if (rows.length < 2) return 0;
|
|
const field = operation === "max" ? "maxvaluecols" : operation === "min" ? "minvaluecols" : operation === "sum" ? "sumvaluecols" : "avgvaluecols";
|
|
const indexes = selectedIndexes(rows, args[field]);
|
|
if (!indexes.length) throw new RuntimeError("validation_error", "No aggregate column was found");
|
|
const groups = selectedIndexes(rows, args.groupby);
|
|
const calculate = (body: unknown[][], index: number): number => {
|
|
const numbers = body.map((row) => Number(row[index])).filter(Number.isFinite);
|
|
if (!numbers.length) return 0;
|
|
if (operation === "max") return Math.max(...numbers);
|
|
if (operation === "min") return Math.min(...numbers);
|
|
const sum = numbers.reduce((total, current) => total + current, 0);
|
|
return operation === "avg" ? sum / numbers.length : sum;
|
|
};
|
|
if (!groups.length) {
|
|
const results = indexes.map((index) => calculate(rows.slice(1), index));
|
|
return results.length === 1 ? results[0] : Object.fromEntries(indexes.map((index, position) => [text(rows[0][index]), results[position]]));
|
|
}
|
|
const buckets = new Map<string, unknown[][]>();
|
|
for (const row of rows.slice(1)) {
|
|
const key = JSON.stringify(groups.map((index) => row[index]));
|
|
if (!buckets.has(key)) buckets.set(key, []);
|
|
buckets.get(key)!.push(row);
|
|
}
|
|
return [
|
|
[...groups.map((index) => rows[0][index]), ...indexes.map((index) => rows[0][index])],
|
|
...[...buckets.entries()].map(([key, body]) => [...JSON.parse(key), ...indexes.map((index) => calculate(body, index))])
|
|
];
|
|
}
|
|
|
|
register("Bivartable_GetMaxValue", (_ctx, _node, args) => aggregate(args, "max"));
|
|
register("Bivartable_GetMinValue", (_ctx, _node, args) => aggregate(args, "min"));
|
|
register("Bivartable_GetSumValue", (_ctx, _node, args) => aggregate(args, "sum"));
|
|
register("Bivartable_GetAvgValue", (_ctx, _node, args) => aggregate(args, "avg"));
|
|
register("Bivartable_SummaryCalculationDf", (_ctx, _node, args) => {
|
|
const method = text(args.method).toLowerCase();
|
|
const operation = method.includes("max") ? "max" : method.includes("min") ? "min" : method.includes("avg") || method.includes("mean") ? "avg" : "sum";
|
|
const field = operation === "max" ? "maxvaluecols" : operation === "min" ? "minvaluecols" : operation === "avg" ? "avgvaluecols" : "sumvaluecols";
|
|
return aggregate({ ...args, [field]: args.name }, operation);
|
|
});
|
|
|
|
register("Bivartable_DeleteDuplicateRow", (_ctx, _node, args) => {
|
|
const rows = table(args.df);
|
|
if (rows.length < 2) return rows;
|
|
const requested = text(args.change_type) === "name" ? args.col_name : args.col_num;
|
|
const indexes = selectedIndexes(rows, requested);
|
|
const compared = indexes.length ? indexes : rows[0].map((_item, index) => index);
|
|
const keys = rows.slice(1).map((row) => JSON.stringify(compared.map((index) => row[index])));
|
|
const counts = new Map<string, number>();
|
|
for (const key of keys) counts.set(key, (counts.get(key) ?? 0) + 1);
|
|
const mode = text(args.mode).toLowerCase();
|
|
const seen = new Set<string>();
|
|
const body = rows.slice(1).filter((_row, index) => {
|
|
const key = keys[index];
|
|
if (mode === "false" || mode === "none" || mode.includes("all")) return counts.get(key) === 1;
|
|
if (mode === "last") return keys.lastIndexOf(key) === index;
|
|
if (seen.has(key)) return false;
|
|
seen.add(key);
|
|
return true;
|
|
});
|
|
return [rows[0], ...body];
|
|
});
|
|
|
|
function mergeTables(args: Record<string, unknown>): Table {
|
|
const left = table(args.df1);
|
|
const right = table(args.df2);
|
|
if (!left.length) return right;
|
|
if (!right.length) return left;
|
|
const mode = text(value(args, ["merge_type", "howx"], "inner")).toLowerCase();
|
|
const common = left[0].filter((header) => right[0].some((candidate) => text(candidate) === text(header))).map(text);
|
|
const sameKeys = columnNames(value(args, ["on_name", "onx"]));
|
|
const leftKeys = columnNames(value(args, ["on_left", "left_onx"], sameKeys.length ? sameKeys : common));
|
|
const rightKeys = columnNames(value(args, ["on_right", "right_onx"], sameKeys.length ? sameKeys : common));
|
|
if (!leftKeys.length || leftKeys.length !== rightKeys.length) throw new RuntimeError("validation_error", "Invalid table join keys");
|
|
const leftIndexes = leftKeys.map((key) => columnIndex(left[0], key));
|
|
const rightIndexes = rightKeys.map((key) => columnIndex(right[0], key));
|
|
if ([...leftIndexes, ...rightIndexes].some((index) => index < 0)) throw new RuntimeError("validation_error", "Table join key does not exist");
|
|
const rightDataIndexes = right[0].map((_header, index) => index).filter((index) => !rightIndexes.includes(index));
|
|
const outputHeaders = [...left[0], ...rightDataIndexes.map((index) => {
|
|
const header = text(right[0][index]);
|
|
return left[0].some((candidate) => text(candidate) === header) ? `${header}_right` : right[0][index];
|
|
})];
|
|
const rightBuckets = new Map<string, unknown[][]>();
|
|
for (const row of right.slice(1)) {
|
|
const key = JSON.stringify(rightIndexes.map((index) => row[index]));
|
|
if (!rightBuckets.has(key)) rightBuckets.set(key, []);
|
|
rightBuckets.get(key)!.push(row);
|
|
}
|
|
const matchedRight = new Set<unknown[]>();
|
|
const output: Table = [outputHeaders];
|
|
for (const row of left.slice(1)) {
|
|
const matches = rightBuckets.get(JSON.stringify(leftIndexes.map((index) => row[index]))) ?? [];
|
|
if (matches.length) {
|
|
for (const match of matches) {
|
|
matchedRight.add(match);
|
|
output.push([...row, ...rightDataIndexes.map((index) => match[index])]);
|
|
}
|
|
} else if (mode === "left" || mode === "outer") {
|
|
output.push([...row, ...rightDataIndexes.map(() => null)]);
|
|
}
|
|
}
|
|
if (mode === "right" || mode === "outer") {
|
|
for (const row of right.slice(1)) {
|
|
if (matchedRight.has(row)) continue;
|
|
const leftPart = left[0].map((header) => {
|
|
const keyPosition = leftKeys.indexOf(text(header));
|
|
return keyPosition >= 0 ? row[rightIndexes[keyPosition]] : null;
|
|
});
|
|
output.push([...leftPart, ...rightDataIndexes.map((index) => row[index])]);
|
|
}
|
|
}
|
|
return output;
|
|
}
|
|
register(["Bivartable_MergeDf", "Bivartable_DoMerge"], (_ctx, _node, args) => mergeTables(args));
|
|
|
|
function predicate(operator: string, expected: unknown): (actual: unknown) => boolean {
|
|
const normalized = operator.trim().toLowerCase();
|
|
if (["2", "!=", "<>", "ne"].includes(normalized)) return (actual) => !equal(actual, expected);
|
|
if (["3", ">", "gt"].includes(normalized)) return (actual) => Number(actual) > Number(expected);
|
|
if (["4", ">=", "gte"].includes(normalized)) return (actual) => Number(actual) >= Number(expected);
|
|
if (["5", "<", "lt"].includes(normalized)) return (actual) => Number(actual) < Number(expected);
|
|
if (["6", "<=", "lte"].includes(normalized)) return (actual) => Number(actual) <= Number(expected);
|
|
if (["7", "contains"].includes(normalized)) return (actual) => text(actual).includes(text(expected));
|
|
if (["8", "not contains"].includes(normalized)) return (actual) => !text(actual).includes(text(expected));
|
|
if (["9", "starts", "startswith"].includes(normalized)) return (actual) => text(actual).startsWith(text(expected));
|
|
if (["10", "ends", "endswith"].includes(normalized)) return (actual) => text(actual).endsWith(text(expected));
|
|
return (actual) => equal(actual, expected);
|
|
}
|
|
|
|
function expressionFilter(rows: Table, expression: string): unknown[][] {
|
|
const conditions = [...expression.matchAll(/(?:\w+\s*)?\[\s*['"]([^'"]+)['"]\s*]\s*(==|!=|>=|<=|>|<)\s*([^&|]+)/g)];
|
|
if (!conditions.length) throw new RuntimeError("validation_error", "Unsupported filter expression");
|
|
const operators = [...expression.matchAll(/\s*([&|])\s*/g)].map((match) => match[1]);
|
|
return rows.slice(1).filter((row) => {
|
|
const results = conditions.map((match) => {
|
|
const index = columnIndex(rows[0], match[1]);
|
|
return index >= 0 && predicate(match[2], comparable(match[3]))(row[index]);
|
|
});
|
|
return results.slice(1).reduce((result, current, index) => operators[index] === "|" ? result || current : result && current, results[0]);
|
|
});
|
|
}
|
|
|
|
function filterTable(args: Record<string, unknown>): Table {
|
|
const rows = table(args.df);
|
|
if (rows.length < 2) return rows;
|
|
const expression = text(value(args, ["rule", "filter"])).trim();
|
|
if (expression || text(args.filter_mode).toLowerCase().includes("advanced")) return [rows[0], ...expressionFilter(rows, expression)];
|
|
const index = columnIndex(rows[0], args.col_name);
|
|
if (index < 0) throw new RuntimeError("validation_error", `Filter column does not exist: ${text(args.col_name)}`);
|
|
const matches = predicate(text(args.choice || "1"), parse(args.criteria));
|
|
return [rows[0], ...rows.slice(1).filter((row) => matches(row[index]))];
|
|
}
|
|
register(["Bivartable_FilterDf", "Bivartable_DoFilter"], (_ctx, _node, args) => filterTable(args));
|