Function readAndProcessEvents

src/data/transform.ts:206–262

()

}

async function readAndProcessEvents() {
  console.log('Reading events from:', RAW_FILE);

  const fileStream = createReadStream(RAW_FILE);
  const rl = createInterface({
    input: fileStream,
    crlfDelay: Infinity,
  });

  let count = 0;
  let accumulated = '';

  for await (const line of rl) {
    const trimmed = line.trim();
    if (!trimmed) continue;

    // If line starts with {, it's a new JSON object
    if (trimmed.startsWith('{')) {
      // Try to parse any accumulated content first
      if (accumulated) {
        try {
          const event = JSON.parse(accumulated) as GitHubEvent;
          processEvent(event);
          count++;
        } catch {
          // Skip malformed accumulated content
        }
      }
      accumulated = line;
    } else {
      // This line is a continuation of previous (embedded newline in JSON string)
      // The newline needs to be escaped for JSON.parse to accept it
      accumulated += '\\n' + line;
    }

    if (count % 100000 === 0 && count > 0) {
      console.log(`Processed ${count} events...`);
    }
  }

  // Don't forget the last accumulated entry
  if (accumulated) {
    try {
      const event = JSON.parse(accumulated) as GitHubEvent;
      processEvent(event);
      count++;
    } catch {
      // Skip malformed
    }
  }

  console.log(`\nTotal events processed: ${count}`);
  console.log(`Unique repos: ${repos.size}`);
  console.log(`Issues: ${issues.size}`);
  console.log(`Pull requests: ${pulls.size}`);
  console.log(`Users: ${users.size}`);
}