Method _tokenize

source/python.js:1922–2057

()

                return position + 1;
            }
            _tokenize() {
                if (this._token.type !== '\n') {
                    this._skipWhitespace();
                }
                if (this._token.type === 'dedent') {
                    this._indentation.pop();
                    this._outdent--;
                    if (this._outdent > 0) {
                        this._token = { type: 'dedent', value: '' };
                        return;
                    }
                }
                if (this._token.type === '\n') {
                    let indent = '';
                    let i = this._position;
                    while (i < this._text.length) {
                        const c = this._text[i];
                        if (ast._Tokenizer._isSpace(c)) {
                            indent += c;
                            i++;
                        } else if (ast._Tokenizer._isNewline(c)) {
                            indent = '';
                            i = this._newLine(i);
                            this._position = i;
                            this.linepos = i;
                            this.lineno += 1;
                        } else if (c === '#') {
                            indent = '';
                            while (i < this._text.length && !ast._Tokenizer._isNewline(this._text[i])) {
                                i++;
                            }
                            continue;
                        } else {
                            break;
                        }
                    }
                    let type = null;
                    if (indent.length > 0) {
                        const current = this._indentation.length > 0 ? this._indentation[this._indentation.length - 1] : '';
                        if (indent.length > current.length) {
                            type = 'indent';
                            this._indentation.push(indent);
                        } else if (indent.length > 0 && indent.length < current.length) {
                            type = 'dedent';
                            this._outdent = 0;
                            for (let j = this._indentation.length - 1; j >= 0 && indent.length < this._indentation[j].length; j--) {
                                this._outdent++;
                            }
                        } else {
                            this._position += indent.length;
                        }
                    } else if (i >= this._text.length) {
                        this._token = { type: 'eof', value: '' };
                        return;
                    } else if (this._indentation.length > 0) {
                        type = 'dedent';
                        this._outdent = this._indentation.length;
                    }