()
| 744 | |
| 745 | #[test] |
| 746 | fn test_tokenizer_is_finished() { |
| 747 | let mut tokenizer = Tokenizer::new(b"ab"); |
| 748 | assert!(!tokenizer.is_finished()); |
| 749 | |
| 750 | tokenizer.next(); // consume "ab" |
| 751 | assert!(tokenizer.is_finished()); |
| 752 | } |
| 753 | |
| 754 | #[test] |
| 755 | fn test_tokenizer_size_hint() { |