Method next

crates/stdlib/src/_tokenize.rs:108–146

(zelf: &Py<Self>, vm: &VirtualMachine)

    impl IterNext for PyTokenizerIter {
        fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> {
            let mut state = zelf.state.read().clone();

            loop {
                match &mut state.phase {
                    TokenizerPhase::Reading { source } => {
                        let line = zelf.readline(vm)?;
                        if line.is_empty() {
                            let accumulated = core::mem::take(source);
                            let parsed = parse_unchecked_source(&accumulated, PySourceType::Python);
                            let tokens: Vec<Token> = parsed.tokens().iter().copied().collect();
                            let errors: Vec<ParseError> = parsed.errors().to_vec();
                            let line_index = LineIndex::from_source_text(&accumulated);
                            let implicit_nl = !accumulated.ends_with('\n');
                            state.phase = TokenizerPhase::Yielding {
                                source: accumulated,
                                tokens,
                                errors,
                                index: 0,
                                line_index,
                                need_implicit_nl: implicit_nl,
                                pending_fstring_parts: Vec::new(),
                                pending_empty_fstring_middle: None,
                            };
                        } else {
                            source.push_str(&line);
                        }
                    }
                    TokenizerPhase::Yielding { .. } => {
                        let result = emit_next_token(&mut state, zelf.extra_tokens, vm)?;
                        *zelf.state.write() = state;
                        return Ok(result);
                    }
                    TokenizerPhase::Done => {
                        return Ok(PyIterReturn::StopIteration(None));
                    }
                }
            }
        }
    }

    /// Emit the next token from the Yielding phase.