Function tokenize_content_for_display

atomic-repository/src/semantic_regen.rs:394–421

Tokenize file content into FileOps for display purposes. This is the **on-the-fly fallback** — when `diff`/`log`/`blame` needs semantic data but the SEMANTIC section hasn't been regenerated yet, this function tokenizes the content directly without persisting to redb. This is the same tokenization pipeline used during `record`, but standalone and stateless. # Arguments `path` - File path (for t

(path: &str, content: &[u8], encoding: Encoding)

/// assert!(ops.line_count() > 0);
/// ```
pub fn tokenize_content_for_display(path: &str, content: &[u8], encoding: Encoding) -> FileOps {
    let placeholder_change_id = NodeId::new(0);
    let mut builder = CrdtChangeBuilder::new(placeholder_change_id);

    let enc = if encoding == Encoding::Binary {
        None
    } else {
        Some(encoding)
    };

    builder.add_file_with_content(path, content, enc);

    let result = builder.finish();
    let (file_ops_list, _, _) = result.into_parts();

    // Extract the single FileOps for this file
    file_ops_list
        .into_iter()
        .next()
        .map(|builder_ops| builder_ops.into_change_ops())
        .unwrap_or_else(|| {
            FileOps::create(
                TrunkId::new(placeholder_change_id, 0),
                path.to_string(),
                Some(encoding),
            )
        })
}

/// Detect the encoding of file content.
///