Function tokenizerFac

packages/muya/src/inlineRenderer/lexer.ts:25–641

(src: string, beginRules: BeginRules | null, inlineRules: InlineRules, pos = 0, top: boolean, labels: Labels, options: ITokenizerFacOptions)

    = /title|textarea|style|xmp|iframe|noembed|noframes|script|plaintext/i;

function tokenizerFac(src: string, beginRules: BeginRules | null, inlineRules: InlineRules, pos = 0, top: boolean, labels: Labels, options: ITokenizerFacOptions) {
    const originSrc = src;
    const tokens: Token[] = [];
    let pending = '';
    let pendingStartPos = pos;
    const { superSubScript, footnote } = options;
    const pushPending = () => {
        if (pending) {
            tokens.push({
                type: 'text',
                parent: tokens,
                raw: pending,
                content: pending,
                range: {
                    start: pendingStartPos,
                    end: pos,
                },
            });
        }

        pendingStartPos = pos;
        pending = '';
    };

    if (beginRules && pos === 0) {
        const beginRuleKeys = [
            'header',
            'hr',
            'code_fence',
            'multiple_math',
        ] as const;

        for (const ruleName of beginRuleKeys) {
            const to = beginRules[ruleName].exec(src);

            if (to) {
                const token = {
                    type: ruleName,
                    raw: to[0],
                    parent: tokens,
                    marker: to[1],
                    content: to[2] || '',
                    backlash: to[3] || '',
                    range: {
                        start: pos,
                        end: pos + to[0].length,
                    },
                };
                tokens.push(token);
                src = src.substring(to[0].length);
                pos = pos + to[0].length;
                break;
            }
        }
        const def = beginRules.reference_definition.exec(src);
        if (def && isLengthEven(def[3])) {
            const token = {
                type: 'reference_definition' as const,