Function tokenizeSegment

apps/cli/src/tooling.ts:339–346

(input: string)

};

const tokenizeSegment = (input: string): ReadonlyArray<string> =>
  input
    .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
    .replace(/[._-]+/g, " ")
    .trim()
    .toLowerCase()
    .split(/\s+/)
    .filter((token) => token.length > 0);

const tokenVariants = (input: string): ReadonlyArray<string> => {
  const token = input.toLowerCase();