()
| 732 | |
| 733 | #[test] |
| 734 | fn test_tokenizer_position() { |
| 735 | let mut tokenizer = Tokenizer::new(b"ab cd"); |
| 736 | assert_eq!(tokenizer.position(), 0); |
| 737 | |
| 738 | tokenizer.next(); // "ab" |
| 739 | assert_eq!(tokenizer.position(), 2); |
| 740 | |
| 741 | tokenizer.next(); // " " |
| 742 | assert_eq!(tokenizer.position(), 3); |
| 743 | } |
| 744 | |
| 745 | #[test] |
| 746 | fn test_tokenizer_is_finished() { |