Method with_code_aware

atomic-core/src/record/workflow/crdt/tokenize/rules.rs:80–83

Sets whether to use code-aware tokenization. When enabled, the tokenizer recognizes multi-character operators, string literals, numeric literals, and comments.

(mut self, aware: bool)

    /// When enabled, the tokenizer recognizes multi-character operators,
    /// string literals, numeric literals, and comments.
    pub fn with_code_aware(mut self, aware: bool) -> Self {
        self.code_aware = aware;
        self
    }

    /// Sets whether to include newline tokens.
    pub fn with_include_newlines(mut self, include: bool) -> Self {