Function emit_next_token

crates/stdlib/src/_tokenize.rs:150–407

Emit the next token from the Yielding phase.

(
        state: &mut TokenizerState,
        extra_tokens: bool,
        vm: &VirtualMachine,
    )

    /// Emit the next token from the Yielding phase.
    fn emit_next_token(
        state: &mut TokenizerState,
        extra_tokens: bool,
        vm: &VirtualMachine,
    ) -> PyResult<PyIterReturn> {
        let TokenizerPhase::Yielding {
            source,
            tokens,
            errors,
            index,
            line_index,
            need_implicit_nl,
            pending_fstring_parts,
            pending_empty_fstring_middle,
        } = &mut state.phase
        else {
            unreachable!()
        };

        // Emit pending empty FSTRING_MIDDLE (for format spec nesting)
        if let Some((mid_type, mid_line, mid_col, mid_line_str)) =
            pending_empty_fstring_middle.take()
        {
            return Ok(PyIterReturn::Return(make_token_tuple(
                vm,
                mid_type,
                "",
                mid_line,
                mid_col as isize,
                mid_line,
                mid_col as isize,
                &mid_line_str,
            )));
        }

        // Emit any pending fstring sub-tokens first
        if let Some((tok_type, tok_str, sl, sc, el, ec)) = pending_fstring_parts.pop() {
            let offset: usize = source
                .lines()
                .take(sl.saturating_sub(1))
                .map(|l| l.len() + 1)
                .sum();
            let full_line = source.full_line_str(TextSize::from(offset.min(source.len()) as u32));
            return Ok(PyIterReturn::Return(make_token_tuple(
                vm,
                tok_type,
                &tok_str,
                sl,
                sc as isize,
                el,
                ec as isize,
                full_line,
            )));
        }

        let source_len = TextSize::from(source.len() as u32);

        while *index < tokens.len() {