Method decode

src/tokenizer.rs:197–202

(&self, tokens: &[u32])

    }

    pub fn decode(&self, tokens: &[u32]) -> String {
        let text: String = tokens.iter().map(|t| self.decoder[t].as_str()).collect();
        let decoded_bytes: Vec<u8> = text.chars().map(|c| self.byte_decoder[&c]).collect();

        String::from_utf8_lossy(&decoded_bytes[..]).replace("</w>", " ")
    }
}

#[cfg(test)]