Returns the default configuration with all features enabled. This is equivalent to `TokenizerConfig::code()` and provides full code-aware tokenization suitable for most programming languages.
()
| 63 | /// This is equivalent to `TokenizerConfig::code()` and provides |
| 64 | /// full code-aware tokenization suitable for most programming languages. |
| 65 | fn default() -> Self { |
| 66 | Self { |
| 67 | merge_whitespace: true, |
| 68 | recognize_operators: true, |
| 69 | recognize_strings: true, |
| 70 | recognize_numbers: true, |
| 71 | recognize_comments: true, |
| 72 | } |
| 73 | } |
| 74 | } |
| 75 | |
| 76 | impl TokenizerConfig { |
nothing calls this directly
no outgoing calls
no test coverage detected