Function getFileTokenScores

packages/code-map/src/parse.ts:63–133

(
  projectRoot: string,
  filePaths: string[],
  readFile?: SourceReader,
)

}

export async function getFileTokenScores(
  projectRoot: string,
  filePaths: string[],
  readFile?: SourceReader,
): Promise<FileTokenData> {
  const startTime = Date.now()
  const tokenScores: Record<string, Record<string, number>> = {}
  const externalCalls: Record<string, number> = {}
  const fileCallsMap = new Map<string, string[]>()
  let parsedFiles = 0
  let totalParsedBytes = 0

  for (const filePath of filePaths) {
    if (
      parsedFiles >= MAX_PARSE_FILES ||
      totalParsedBytes >= MAX_TOTAL_PARSE_BYTES
    ) {
      break
    }

    const fullPath = path.join(projectRoot, filePath)
    const languageConfig = await getLanguageConfig(fullPath)
    if (!languageConfig) continue

    const parsed = await parseTokensForScoring({
      filePath,
      fullPath,
      languageConfig,
      readFile,
      remainingBytes: MAX_TOTAL_PARSE_BYTES - totalParsedBytes,
    })
    if (parsed.skipped) continue

    parsedFiles++
    totalParsedBytes += parsed.bytes

    const { scores, calls } = scoreFileTokens(fullPath, parsed)
    tokenScores[filePath] = scores
    fileCallsMap.set(filePath, calls)

    for (const call of calls) {
      if (!scores[call]) {
        externalCalls[call] = (externalCalls[call] ?? 0) + 1
      }
    }
  }

  const tokenCallers = buildTokenCallers(tokenScores, fileCallsMap)
  boostScoresByExternalCalls(tokenScores, externalCalls)

  if (DEBUG_PARSING) {
    const endTime = Date.now()
    console.log(`Parsed ${filePaths.length} files in ${endTime - startTime}ms`)

    try {
      fs.writeFileSync(
        '../debug/debug-parse.json',
        JSON.stringify({