Function test_tokenizer_remaining

atomic-core/src/diff/token/tests.rs:725–731

()

    #[test]
    fn test_tokenizer_remaining() {
        let mut tokenizer = Tokenizer::new(b"hello world");
        assert_eq!(tokenizer.remaining(), b"hello world");

        tokenizer.next(); // consume "hello"
        assert_eq!(tokenizer.remaining(), b" world");
    }

    #[test]
    fn test_tokenizer_position() {