Function tokenizeLineWithGrammar

packages/core/src/widgets/codeEditorSyntax.ts:932–1015

(
  line: string,
  grammar: LanguageGrammar,
  language: CanonicalLanguage,
)

}

function tokenizeLineWithGrammar(
  line: string,
  grammar: LanguageGrammar,
  language: CanonicalLanguage,
): readonly CodeEditorSyntaxToken[] {
  const scanner: Scanner = { source: line, index: 0, tokens: [] };

  while (scanner.index < line.length) {
    const ch = asChar(line, scanner.index);
    const next = asChar(line, scanner.index + 1);

    if (isWhitespace(ch)) {
      consumeWhitespace(scanner);
      continue;
    }

    if (grammar.lineComment === "#" && ch === "#") {
      pushToken(scanner.tokens, line.slice(scanner.index), "comment");
      break;
    }

    if (grammar.lineComment === "//" && ch === "/" && next === "/") {
      pushToken(scanner.tokens, line.slice(scanner.index), "comment");
      break;
    }

    if (grammar.blockComments && ch === "/" && next === "*") {
      const start = scanner.index;
      scanner.index += 2;
      while (scanner.index < line.length) {
        const a = asChar(line, scanner.index);
        const b = asChar(line, scanner.index + 1);
        scanner.index += 1;
        if (a === "*" && b === "/") {
          scanner.index += 1;
          break;
        }
      }
      pushToken(scanner.tokens, line.slice(start, scanner.index), "comment");
      continue;
    }

    if (language === "csharp" && ch === "@" && next === '"') {
      pushToken(scanner.tokens, consumeCSharpVerbatimString(scanner), "string");
      continue;
    }

    if (
      (ch === "'" && grammar.allowSingleQuotedStrings) ||
      (ch === '"' && grammar.allowDoubleQuotedStrings) ||
      (ch === "`" && grammar.allowBacktickStrings)
    ) {
      pushToken(scanner.tokens, consumeString(scanner, ch), "string");
      continue;
    }

    if (isDigit(ch) || (ch === "-" && isDigit(next))) {
      pushToken(scanner.tokens, consumeNumber(scanner), "number");