Function executeDeduplicateNode

src/nodeExecutor.ts:4490–4697

(
  nodeId: string,
  textValue: string,
  inputData: DBTablesNodeData,
  config?: unknown,
  executionContext?: FlowExecutionContext,
)

}

async function executeDeduplicateNode(
  nodeId: string,
  textValue: string,
  inputData: DBTablesNodeData,
  config?: unknown,
  executionContext?: FlowExecutionContext,
): Promise<NodeExecutionResult> {
  console.log(`Deduplicate Node ${nodeId}: Processing with input tables:`, {
    tableCount: inputData?.tables?.length,
    tableNames: inputData?.tables?.map((t) => t.name),
  });

  if (!inputData || inputData.type !== "TableSelection") {
    return { success: false, error: "Invalid input: Expected TableSelection data" };
  }

  // Parse configuration from textValue
  const parsed = getNodeConfig<{ dedupeByColumns?: string[]; dedupeAgainstVanilla?: boolean }>(config, textValue);
  if (!parsed) {
    return { success: false, error: "Invalid deduplicate configuration" };
  }
  const dedupeByColumns = parsed.dedupeByColumns || [];
  const dedupeAgainstVanilla = parsed.dedupeAgainstVanilla || false;

  console.log(
    `Deduplicate Node ${nodeId}: dedupeByColumns: [${dedupeByColumns.join(", ")}], dedupeAgainstVanilla: ${dedupeAgainstVanilla}`,
  );

  if (dedupeByColumns.length === 0) {
    return { success: false, error: "No dedupe by columns specified" };
  }

  const dedupedTables = [] as DBTablesNodeTable[];
  const inputTableToDupes = new Map<number, Set<number>>();
  const alreadyPresentRowHashes = new Set<number>();

  // If dedupeAgainstVanilla is enabled, build a set of vanilla row hashes
  const vanillaRowHashes = new Set<number>();
  if (dedupeAgainstVanilla) {
    console.log(`Deduplicate Node ${nodeId}: Building vanilla row hashes...`);

    // Get the base game pack path
    const baseGamePackName = gameToPackWithDBTablesName[appData.currentGame];
    if (baseGamePackName) {
      const baseGameFolder = appData.gamesToGameFolderPaths[appData.currentGame].dataFolder;
      if (baseGameFolder) {
        const baseGamePackPath = path.join(baseGameFolder, baseGamePackName);
        if (fs.existsSync(baseGamePackPath)) {
          // Get unique table names from input data
          const tableNamesToRead = new Set<string>();
          for (const tableData of inputData.tables) {
            tableNamesToRead.add(tableData.name);
          }

          console.log(
            `Deduplicate Node ${nodeId}: Reading vanilla tables: [${Array.from(tableNamesToRead).join(", ")}]`,
          );