Function tokenize

atomic-core/src/merge/three_way.rs:83–88

Tokenize a byte slice into owned [`MergeToken`]s. Uses the project's code-aware [`Tokenizer`] so that operators, strings, numbers, and comments are recognised as single tokens.

(content: &[u8])

/// Uses the project's code-aware [`Tokenizer`] so that operators,
/// strings, numbers, and comments are recognised as single tokens.
pub fn tokenize(content: &[u8]) -> Vec<MergeToken> {
    Tokenizer::tokenize_all(content)
        .into_iter()
        .map(|t| MergeToken::new(t.content()))
        .collect()
}

/// Perform a three-way merge of token sequences.
///