Function test_tokenizer_unicode_in_other

atomic-core/src/diff/token/tests.rs:701–707

()

    #[test]
    fn test_tokenizer_unicode_in_other() {
        let code = "café".as_bytes();
        let tokens: Vec<Token> = Tokenizer::new(code).collect();

        // ASCII part is Word, then 'é' bytes are Other
        assert!(!tokens.is_empty());
    }

    // Tokenizer convenience method tests