Method next

atomic-core/src/diff/token/tokenizer.rs:509–538

(&mut self)

    type Item = Token<'a>;

    fn next(&mut self) -> Option<Self::Item> {
        if self.is_finished() {
            return None;
        }

        let b = self.peek()?;

        // Determine token type and read it
        let token = if Self::is_newline(b) {
            self.read_newline()
        } else if Self::is_whitespace(b) {
            self.read_whitespace()
        } else if self.config.recognize_comments && b == b'/' && self.peek_at(1) == Some(b'/') {
            self.read_comment()
        } else if self.config.recognize_strings && (b == b'"' || b == b'\'') {
            self.read_string(b)
        } else if self.config.recognize_numbers && Self::is_number_start(b) {
            self.read_number()
        } else if Self::is_word_char(b) && !b.is_ascii_digit() {
            self.read_word()
        } else if self.config.recognize_operators && Self::is_operator(b) {
            self.read_operator()
        } else if Self::is_punctuation(b) {
            self.read_punctuation()
        } else {
            self.read_other()
        };

        Some(token)
    }

    fn size_hint(&self) -> (usize, Option<usize>) {
        let remaining = self.content.len() - self.position;