Function backfillInternalEvents

packages/admin-cli/src/upgrades.ts:574–929

({
  // defaults to 1 day in minutes
  intervalMinutes = 1440,
  workspaceIds,
  startDate: startDateOverride,
  endDate: endDateOverride,
  forceFullBackfill = false,
  // defaults to 10000 rows per batch within a time window
  limit = 10000,
  dryRun = false,
}: {
  intervalMinutes?: number;
  workspaceIds?: string[];
  startDate?: string;
  endDate?: string;
  forceFullBackfill?: boolean;
  limit?: number;
  dryRun?: boolean;
})

}

export async function backfillInternalEvents({
  // defaults to 1 day in minutes
  intervalMinutes = 1440,
  workspaceIds,
  startDate: startDateOverride,
  endDate: endDateOverride,
  forceFullBackfill = false,
  // defaults to 10000 rows per batch within a time window
  limit = 10000,
  dryRun = false,
}: {
  intervalMinutes?: number;
  workspaceIds?: string[];
  startDate?: string;
  endDate?: string;
  forceFullBackfill?: boolean;
  limit?: number;
  dryRun?: boolean;
}) {
  logger().info(
    dryRun
      ? "Analyzing internal events backfill (dry run)"
      : "Backfilling internal events",
  );

  // Determine start date
  let startDate: Date;
  if (startDateOverride) {
    startDate = new Date(startDateOverride);
    logger().info(
      { startDate, override: startDateOverride },
      "Using manual start date override",
    );
  } else if (forceFullBackfill) {
    // Skip internal_events check and always use min from user_events_v2
    logger().info(
      "Force full backfill enabled, skipping internal_events check",
    );
    try {
      const userEventsQb = new ClickHouseQueryBuilder();
      const userEventsWorkspaceFilter = workspaceIds
        ? `AND workspace_id IN ${userEventsQb.addQueryValue(workspaceIds, "Array(String)")}`
        : "";

      const userEventsResult = await query({
        query: `SELECT min(processing_time) as min_time FROM user_events_v2 WHERE event_type = 'track' AND startsWith(event, 'DF') ${userEventsWorkspaceFilter}`,
        query_params: userEventsQb.getQueries(),
        clickhouse_settings: { wait_end_of_query: 1 },
      });

      const minTimeResult = await userEventsResult.json<{ min_time: string }>();
      const minTime = minTimeResult[0]?.min_time;

      logger().debug(
        { minTimeResult, minTime },
        "Raw min time result from user_events_v2 (force full backfill)",
      );