Function tokenizeCodeEditorLine

packages/core/src/widgets/codeEditorSyntax.ts:1022–1034

(
  line: string,
  context: Readonly<{ language?: CodeEditorSyntaxLanguage }> = {},
)

}

export function tokenizeCodeEditorLine(
  line: string,
  context: Readonly<{ language?: CodeEditorSyntaxLanguage }> = {},
): readonly CodeEditorSyntaxToken[] {
  const source = typeof line === "string" ? line : "";
  if (source.length === 0) return Object.freeze([]);

  const language = normalizeLanguage(context.language);
  if (language === "plain") return plainLine(source);

  const grammar = GRAMMARS[language] ?? GRAMMARS.plain;
  return tokenizeLineWithGrammar(source, grammar, language);
}

/**
 * Sanitizes custom tokenizer output and guarantees full line coverage.