Function checkInvalidVector

projects/app/src/service/common/system/cronTask.ts:99–151

(start: Date, end: Date)

}

export async function checkInvalidVector(start: Date, end: Date) {
  let deletedVectorAmount = 0;
  // 1. get all vector data
  const rows = await getVectorDataByTime(start, end);
  logger.info('Start cleaning invalid vector records', { totalVectors: rows.length, start, end });

  let index = 0;

  for await (const item of rows) {
    if (!item.teamId || !item.datasetId || !item.id) {
      logger.error('Invalid vector record encountered', { ...item });
      continue;
    }
    try {
      // 2. find dataset.data
      const hasData = await MongoDatasetData.countDocuments({
        teamId: item.teamId,
        datasetId: item.datasetId,
        'indexes.dataId': item.id
      });

      // 3. if not found, delete vector
      if (hasData === 0) {
        await deleteDatasetDataVector({
          teamId: item.teamId,
          id: item.id
        });
        logger.info('Deleted orphan vector record', {
          vectorId: item.id,
          teamId: item.teamId,
          datasetId: item.datasetId
        });
        deletedVectorAmount++;
      }

      index++;
      if (index % 100 === 0) {
        logger.debug('Invalid vector cleaning progress', {
          processedVectors: index,
          totalVectors: rows.length,
          deletedVectorAmount
        });
      }
    } catch (error) {
      logger.error('Failed to clean invalid vector record', { ...item, error });
    }
  }

  logger.info('Finished cleaning invalid vector records', {
    deletedVectorAmount,
    totalVectors: rows.length
  });
}