Reads the next token from the input stream. Note that this returns errors only on fatal I/O conditions. EOF and malformed tokens are both returned as the special token types `Token::Eof` and `Token::Bad` respectively.
(&mut self)
| 669 | /// Note that this returns errors only on fatal I/O conditions. EOF and malformed tokens are |
| 670 | /// both returned as the special token types `Token::Eof` and `Token::Bad` respectively. |
| 671 | pub fn read(&mut self) -> io::Result<TokenSpan> { |
| 672 | if let Some(Ok(ch_span)) = self.input.peek() |
| 673 | && ch_span.pos == (LineCol { line: 1, col: 1 }) |
| 674 | && ch_span.ch == '#' |
| 675 | { |
| 676 | let first = self.input.next().unwrap()?; |
| 677 | match self.input.peek() { |
| 678 | Some(Ok(ch_span)) if ch_span.ch == '!' => { |
| 679 | self.input.next().unwrap()?; |
| 680 | let _end = self.consume_rest_of_line()?; |
| 681 | return self.read(); |
| 682 | } |
| 683 | Some(Ok(_)) | None => { |
| 684 | return self.handle_bad_read("Unknown character: #", first.pos); |
| 685 | } |
| 686 | Some(Err(_)) => return Err(self.input.next().unwrap().unwrap_err()), |
| 687 | } |
| 688 | } |
| 689 | |
| 690 | let ch_span = self.advance_and_read_next()?; |
| 691 | if ch_span.is_none() { |
| 692 | let last_pos = self.input.next_pos(); |
| 693 | return Ok(TokenSpan::new(Token::Eof, last_pos, 0)); |
| 694 | } |
| 695 | let ch_span = ch_span.unwrap(); |
| 696 | match ch_span.ch { |
| 697 | '\n' | ':' => Ok(TokenSpan::new(Token::Eol, ch_span.pos, 1)), |
| 698 | '\'' => self.consume_rest_of_line(), |
| 699 | |
| 700 | '"' => self.consume_text(ch_span), |
| 701 | |
| 702 | ';' => Ok(TokenSpan::new(Token::Semicolon, ch_span.pos, 1)), |
| 703 | ',' => Ok(TokenSpan::new(Token::Comma, ch_span.pos, 1)), |
| 704 | |
| 705 | '(' => Ok(TokenSpan::new(Token::LeftParen, ch_span.pos, 1)), |
| 706 | ')' => Ok(TokenSpan::new(Token::RightParen, ch_span.pos, 1)), |
| 707 | |
| 708 | '+' => Ok(TokenSpan::new(Token::Plus, ch_span.pos, 1)), |
| 709 | '-' => Ok(TokenSpan::new(Token::Minus, ch_span.pos, 1)), |
| 710 | '*' => Ok(TokenSpan::new(Token::Multiply, ch_span.pos, 1)), |
| 711 | '/' => Ok(TokenSpan::new(Token::Divide, ch_span.pos, 1)), |
| 712 | '^' => Ok(TokenSpan::new(Token::Exponent, ch_span.pos, 1)), |
| 713 | |
| 714 | '=' => Ok(TokenSpan::new(Token::Equal, ch_span.pos, 1)), |
| 715 | '<' | '>' => self.consume_operator(ch_span), |
| 716 | |
| 717 | '@' => self.consume_label(ch_span), |
| 718 | |
| 719 | '&' => self.consume_integer_with_base(ch_span.pos), |
| 720 | |
| 721 | ch if ch.is_ascii_digit() => self.consume_number(ch_span), |
| 722 | ch if ch.is_word() => self.consume_symbol(ch_span), |
| 723 | ch => self.handle_bad_read(format!("Unknown character: {}", ch), ch_span.pos), |
| 724 | } |
| 725 | } |
| 726 | |
| 727 | /// Returns a peekable adaptor for this lexer. |
| 728 | pub fn peekable(self) -> PeekableLexer<'a> { |