Create a minimal configuration for faster, simpler tokenization.
()
| 494 | |
| 495 | /// Create a minimal configuration for faster, simpler tokenization. |
| 496 | pub fn minimal() -> Self { |
| 497 | Self { |
| 498 | tokenizer_config: TokenizerConfig::minimal(), |
| 499 | ignore_whitespace_changes: false, |
| 500 | filter_whitespace: false, |
| 501 | } |
| 502 | } |
| 503 | |
| 504 | /// Create a configuration that ignores whitespace. |
| 505 | pub fn ignoring_whitespace() -> Self { |
nothing calls this directly
no outgoing calls
no test coverage detected