Function main

apps/cursor/src/scripts/extract-from-github.ts:395–561

()

}

async function main() {
  const args = parseArgs();
  console.log("Args:", args);

  await mkdir(dirname(args.output), { recursive: true });

  const candidates = await discoverCandidates(args);

  console.log(`Total unique candidates: ${candidates.size}`);

  const resumeIds = args.resume ? await loadResumeIds(args.output) : new Set();
  if (args.resume) {
    console.log(`Resume: skipping ${resumeIds.size} already-extracted repos.`);
  }

  const outStream = createWriteStream(args.output, {
    flags: args.resume ? "a" : "w",
  });
  const failedStream = createWriteStream(`${args.output}.failed.jsonl`, {
    flags: args.resume ? "a" : "w",
  });

  const summary = {
    total_candidates: candidates.size,
    after_filter: 0,
    parsed_ok: 0,
    parse_failed: 0,
    skipped_fork: 0,
    skipped_archived: 0,
    skipped_unreadable: 0,
    skipped_low_stars: 0,
    skipped_resume: 0,
    by_source: {} as Record<string, number>,
    by_component_type: {} as Record<string, number>,
  };

  let processed = 0;
  for (const [key, candidate] of candidates) {
    if (processed >= args.limit) break;
    processed++;

    const url = `https://github.com/${candidate.owner}/${candidate.repo}`;
    const prefix = `[${processed}/${Math.min(candidates.size, args.limit)}] ${key}`;

    let meta: GitHubRepoMeta | null;
    try {
      meta = await fetchGitHubRepoMeta(candidate.owner, candidate.repo);
    } catch (err) {
      console.warn(
        `${prefix} repo meta fetch failed: ${err instanceof Error ? err.message : err}`,
      );
      summary.skipped_unreadable++;
      failedStream.write(
        `${JSON.stringify({
          url,
          source: candidate.source,
          reason: "meta_fetch_failed",
        })}\n`,