Method getNextToken

src/tokenizer.ts:128–192

()

    }

    getNextToken() {
        if (this.buffer.length === 0) {

            const comments: Comment[] = this.scanner.scanComments();
            if (this.scanner.trackComment) {
                for (let i = 0; i < comments.length; ++i) {
                    const e: Comment = comments[i];
                    const value = this.scanner.source.slice(e.slice[0], e.slice[1]);
                    const comment: BufferEntry = {
                        type: e.multiLine ? 'BlockComment' : 'LineComment',
                        value: value
                    };
                    if (this.trackRange) {
                        comment.range = e.range;
                    }
                    if (this.trackLoc) {
                        comment.loc = e.loc;
                    }
                    this.buffer.push(comment);
                }
            }

            if (!this.scanner.eof()) {
                let loc;

                if (this.trackLoc) {
                    loc = {
                        start: {
                            line: this.scanner.lineNumber,
                            column: this.scanner.index - this.scanner.lineStart
                        },
                        end: {}
                    };
                }

                const startRegex = (this.scanner.source[this.scanner.index] === '/') && this.reader.isRegexStart();
                const token = startRegex ? this.scanner.scanRegExp() : this.scanner.lex();
                this.reader.push(token);

                const entry: BufferEntry = {
                    type: TokenName[token.type],
                    value: this.scanner.source.slice(token.start, token.end)
                };
                if (this.trackRange) {
                    entry.range = [token.start, token.end];
                }
                if (this.trackLoc) {
                    loc.end = {
                        line: this.scanner.lineNumber,
                        column: this.scanner.index - this.scanner.lineStart
                    };
                    entry.loc = loc;
                }
                if (token.type === Token.RegularExpression) {
                    const pattern = token.pattern as string;
                    const flags = token.flags as string;
                    entry.regex = { pattern, flags };
                }