Function tokenize_single_file

atomic-repository/src/semantic_regen.rs:612–674

Tokenize a single file's content. This runs on a rayon worker thread for parallel regeneration.

(path: &str, content: &[u8], skip_binary: bool)

///
/// This runs on a rayon worker thread for parallel regeneration.
fn tokenize_single_file(path: &str, content: &[u8], skip_binary: bool) -> FileRegenResult {
    if content.is_empty() {
        return FileRegenResult {
            path: path.to_string(),
            file_ops: FileOps::create(TrunkId::new(NodeId::new(0), 0), path.to_string(), None),
            crdt_stats: CrdtBuildStats::default(),
            skipped: true,
            error: None,
        };
    }

    let encoding = detect_encoding(content);

    if encoding == Encoding::Binary && skip_binary {
        return FileRegenResult {
            path: path.to_string(),
            file_ops: FileOps::create(
                TrunkId::new(NodeId::new(0), 0),
                path.to_string(),
                Some(Encoding::Binary),
            ),
            crdt_stats: CrdtBuildStats::default(),
            skipped: true,
            error: None,
        };
    }

    // Tokenize using the same pipeline as record
    let placeholder_change_id = NodeId::new(0);
    let mut builder = CrdtChangeBuilder::new(placeholder_change_id);

    let enc = if encoding == Encoding::Binary {
        None
    } else {
        Some(encoding)
    };

    builder.add_file_with_content(path, content, enc);

    let result = builder.finish();
    let crdt_stats = result.stats().clone();
    let (file_ops_list, _, _) = result.into_parts();

    let file_ops = file_ops_list
        .into_iter()
        .next()
        .map(|builder_ops| builder_ops.into_change_ops())
        .unwrap_or_else(|| {
            FileOps::create(
                TrunkId::new(placeholder_change_id, 0),
                path.to_string(),
                enc,
            )
        });

    FileRegenResult {
        path: path.to_string(),
        file_ops,