Function calculateBulkOperations

scripts/update/processor.ts:71–112

* Calculates the bulk write operations (upserts/deletes) based on the diff * between old and new record arrays. * @param oldData Array of records from the old file version. * @param currentData Array of records from the current file version. * @param filepath Current path of the file (for loggin

(
  oldData: any[],
  currentData: any[],
  filepath: string
)

 * @returns An object containing the operations array and skipped record count.
 */
function calculateBulkOperations(
  oldData: any[],
  currentData: any[],
  filepath: string
): { operations: AnyBulkWriteOperation<Document>[]; skippedRecords: number } {
  const operations: AnyBulkWriteOperation<Document>[] = [];
  let skippedRecords = 0;

  const oldRecordsMap = new Map(oldData.map((r) => [r.index, r]));
  const newRecordsMap = new Map(currentData.map((r) => [r.index, r]));

  // Find Added/Modified records
  for (const [index, newRecord] of newRecordsMap.entries()) {
    if (index === undefined || index === null || index === '') {
      console.warn(`Record in ${filepath} is missing a valid 'index' field...`);
      skippedRecords++;
      continue;
    }
    const oldRecord = oldRecordsMap.get(index);
    if (!oldRecord || diff(oldRecord, newRecord)) {
      operations.push({
        [MONGO_OP_UPDATE_ONE]: {
          filter: { index: index },
          update: { $set: { ...newRecord, [FIELD_UPDATED_AT]: new Date().toISOString() } },
          upsert: true,
        },
      });
    }
  }

  // Find Deletions
  for (const [index] of oldRecordsMap.entries()) {
    // No need for oldRecord value here
    if (index === undefined || index === null || index === '') continue;
    if (!newRecordsMap.has(index)) {
      console.log(`  - Detected deletion for index: ${index}`);
      operations.push({ [MONGO_OP_DELETE_ONE]: { filter: { index: index } } });
    }
  }

  return { operations, skippedRecords };
}

/**
 * Executes the MongoDB bulk write operation.