Function tokenize

scripts/migrate-locale.ts:81–83

(s: string)

}

function tokenize(s: string): string[] {
  return normalize(s).split(/\s+/).filter(t => t.length > 1);
}

// Noise tokens that add no signal
const NOISE = new Set(["the", "a", "an", "and", "or", "of", "to", "in",