Function tokenize

site/js/common/search.ts:228–243

(string: string, tokens: Set<string>, reverse?: boolean)

// Tokenize a string into words, optionally reversing the order
const tokenize = (string: string, tokens: Set<string>, reverse?: boolean) => {
  const words = string.replaceAll(/[^a-zA-Z0-9]/g, ' ').split(/\s+/);
  if (reverse) {
    words.reverse();
  }
  words.forEach((word) => {
    [word, ...word.replaceAll(/([a-z])([A-Z])/g, '$1 $2').split(' ')].forEach(
      (wordPart) => {
        const token = wordPart.toLowerCase();
        if (token.length > 3 || SHORT_WORDS.includes(token)) {
          tokens.add(token);
        }
      },
    );
  });
};

// Get the weighting of a word in a string based on its position
const getWeighting = (tokens: string, word: string) => {