Method tokenize_all

atomic-core/src/record/workflow/crdt/tokenize/mod.rs:89–99

Tokenizes all content and returns the result with statistics. This eagerly processes all lines and collects statistics.

(&self)

    ///
    /// This eagerly processes all lines and collects statistics.
    pub fn tokenize_all(&self) -> (Vec<TokenizedLine>, TokenStats) {
        let mut lines = Vec::new();
        let mut stats = TokenStats::new();

        for line in self.lines() {
            stats.add_line(&line);
            lines.push(line);
        }

        (lines, stats)
    }

    /// Checks if the content appears to be binary.
    ///