File quantize-tokenizer.test.ts

test/quantize-tokenizer.test.ts:None–None

import { describe, it, expect } from 'vitest';
import { quantize, cosineQuantized } from '../src/context/sqlite-index.js';
import { heuristicTokens, countTokens, countTokensJson } from '../src/utils/tokenizer.js';