Method decode

src/token.rs:41–44

(&self, tokens: &[i64], skip_special_tokens: bool)

    }

    pub fn decode(&self, tokens: &[i64], skip_special_tokens: bool) -> String {
        let clean_spaces = false;
        self.spm.decode(tokens, skip_special_tokens, clean_spaces)
    }

    pub fn vocab_size(&self, include_special_tokens: bool) -> usize {
        let vocab = self.spm.vocab();