Function countFileLinesAndTokens

src/integrations/misc/line-counter.ts:71–167

(
	filePath: string,
	options: LineAndTokenCountOptions = {},
)

 * @returns A promise that resolves to line count, token estimate, and completion status
 */
export async function countFileLinesAndTokens(
	filePath: string,
	options: LineAndTokenCountOptions = {},
): Promise<LineAndTokenCountResult> {
	const { budgetTokens, chunkLines = 256 } = options

	// Check if file exists
	try {
		await fs.promises.access(filePath, fs.constants.F_OK)
	} catch (error) {
		throw new Error(`File not found: ${filePath}`)
	}

	return new Promise((resolve, reject) => {
		let lineCount = 0
		let tokenEstimate = 0
		let lineBuffer: string[] = []
		let complete = true
		let isProcessing = false
		let shouldClose = false

		const readStream = createReadStream(filePath)
		const rl = createInterface({
			input: readStream,
			crlfDelay: Infinity,
		})

		const processBuffer = async () => {
			if (lineBuffer.length === 0) return

			const bufferText = lineBuffer.join("\n")
			lineBuffer = [] // Clear buffer before processing

			try {
				const contentBlocks: Anthropic.Messages.ContentBlockParam[] = [{ type: "text", text: bufferText }]
				const chunkTokens = await countTokens(contentBlocks)
				tokenEstimate += chunkTokens
			} catch (error) {
				// On tokenizer error, use conservative estimate: 2 char ≈ 1 token
				tokenEstimate += Math.ceil(bufferText.length / 2)
			}

			// Check if we've exceeded budget
			if (budgetTokens !== undefined && tokenEstimate > budgetTokens) {
				complete = false
				shouldClose = true
				rl.close()
				readStream.destroy()
			}
		}

		rl.on("line", (line) => {
			lineCount++
			lineBuffer.push(line)

			// Process buffer when it reaches chunk size
			if (lineBuffer.length >= chunkLines && !isProcessing) {
				isProcessing = true