Function tokenize

packages/monaco/src/textmate.ts:67–88

(line, state)

            return INITIAL;
        },
        tokenize(line, state) {
            const tokenizerState = state as StateStack;
            const textMateResult = grammar.tokenizeLine(line, tokenizerState, 500);
            if (textMateResult.stoppedEarly) {
                return {
                    endState: state,
                    tokens: textMateResult.tokens.map(t => ({ ...t, scopes: t.scopes.reverse().join(' ') }))
                };
            }

            let endState: StateStack;
            if (state.equals(textMateResult.ruleStack)) {
                endState = tokenizerState;
            } else {
                endState = textMateResult.ruleStack;
            }

            return {
                endState,
                tokens: textMateResult.tokens.map(t => ({ ...t, scopes: t.scopes.reverse().join(' ') }))
            };
        }
    });
}