Create a new tokenizer with default configuration. Uses full code-aware tokenization by default. # Arguments `content` - The bytes to tokenize # Example ```rust use atomic_core::diff::token::Tokenizer; let tokenizer = Tokenizer::new(b"hello world"); let tokens: Vec<_> = tokenizer.collect(); assert_eq!(tokens.len(), 3); ```
(content: &'a [u8])
| 63 | /// assert_eq!(tokens.len(), 3); |
| 64 | /// ``` |
| 65 | pub fn new(content: &'a [u8]) -> Self { |
| 66 | Self::with_config(content, TokenizerConfig::default()) |
| 67 | } |
| 68 | |
| 69 | /// Create a new tokenizer with custom configuration. |
| 70 | /// |
nothing calls this directly
no outgoing calls
no test coverage detected