Function processDocuments

scripts/pipeline/pdf-processor.ts:238–401

(options: {
  inputDir?: string;
  dataSetIds?: number[];
  fileTypes?: string[];
  maxFiles?: number;
  outputDir?: string;
  reprocessEmpty?: boolean;
})

}

export async function processDocuments(options: {
  inputDir?: string;
  dataSetIds?: number[];
  fileTypes?: string[];
  maxFiles?: number;
  outputDir?: string;
  reprocessEmpty?: boolean;
}): Promise<ExtractedDocument[]> {
  console.log("\n=== PDF Text Extractor ===\n");

  const {
    inputDir = DOWNLOADS_DIR,
    maxFiles = Infinity,
    outputDir = EXTRACTED_DIR,
  } = options;

  if (!fs.existsSync(outputDir)) fs.mkdirSync(outputDir, { recursive: true });
  if (!fs.existsSync(DATA_DIR)) fs.mkdirSync(DATA_DIR, { recursive: true });

  // Collect files to process per data set
  const dirs = fs.existsSync(inputDir)
    ? fs.readdirSync(inputDir)
        .filter(d => d.startsWith("data-set-") && fs.statSync(path.join(inputDir, d)).isDirectory())
        .sort()
    : [];

  const log = loadLog();

  // Pre-pass: remove low-quality extractions so they get re-extracted with OCR
  if (options.reprocessEmpty) {
    console.log("  Scanning for low-quality extractions to reprocess...");
    let removedCount = 0;

    for (const dir of dirs) {
      const dsMatch = dir.match(/data-set-(\d+)/);
      if (!dsMatch) continue;
      const dsId = parseInt(dsMatch[1], 10);
      if (options.dataSetIds && !options.dataSetIds.includes(dsId)) continue;

      const dsOutputDir = path.join(outputDir, `ds${dsId}`);
      if (!fs.existsSync(dsOutputDir)) continue;

      const jsonFiles = fs.readdirSync(dsOutputDir).filter(f => f.endsWith(".json"));
      for (const jsonFile of jsonFiles) {
        const jsonPath = path.join(dsOutputDir, jsonFile);
        try {
          const doc: ExtractedDocument = JSON.parse(fs.readFileSync(jsonPath, "utf-8"));
          if (doc.method === "ocr") continue; // Already OCR'd, skip
          if (isTextQualityPoor(doc.text, doc.pageCount)) {
            log.totalPages -= doc.pageCount;
            log.totalChars -= doc.text.length;
            log.totalProcessed--;
            fs.unlinkSync(jsonPath);
            removedCount++;
          }
        } catch {
          // Skip malformed JSON
        }