Function deterministicEmbedding

benchmark/scale-eval.ts:28–42

(text: string, dims = 384)

}

function deterministicEmbedding(text: string, dims = 384): Float32Array {
  const arr = new Float32Array(dims);
  const words = text.toLowerCase().split(/\W+/).filter(w => w.length > 2);
  for (const word of words) {
    for (let i = 0; i < word.length; i++) {
      const idx = (word.charCodeAt(i) * 31 + i * 17) % dims;
      arr[idx] += 1;
      const idx2 = (word.charCodeAt(i) * 37 + i * 13 + word.length * 7) % dims;
      arr[idx2] += 0.5;
    }
  }
  const norm = Math.sqrt(arr.reduce((s, v) => s + v * v, 0));
  if (norm > 0) for (let i = 0; i < dims; i++) arr[i] /= norm;
  return arr;
}

function estimateTokens(text: string): number {
  return Math.ceil(text.length / 4);