Function parseTokensForScoring

packages/code-map/src/parse.ts:150–180

(params: {
  filePath: string
  fullPath: string
  languageConfig: LanguageConfig
  readFile?: SourceReader
  remainingBytes: number
})

}

async function parseTokensForScoring(params: {
  filePath: string
  fullPath: string
  languageConfig: LanguageConfig
  readFile?: SourceReader
  remainingBytes: number
}): Promise<ParsedTokensForScoring> {
  const { filePath, fullPath, languageConfig, readFile, remainingBytes } =
    params

  if (!readFile) {
    return parseTokensWithLimits(fullPath, languageConfig, undefined, {
      maxBytes: MAX_PARSE_FILE_BYTES,
      remainingBytes,
    })
  }

  try {
    const source = await readFile(filePath)
    return parseTokensWithLimits(filePath, languageConfig, () => source, {
      maxBytes: MAX_PARSE_FILE_BYTES,
      remainingBytes,
    })
  } catch (e) {
    if (DEBUG_PARSING) {
      console.error(`Error reading source: ${e}`)
      console.log(filePath)
    }
    return emptyParsedTokens(false)
  }
}

function parseTokensWithLimits(
  filePath: string,