Code
Hub
Workspaces
Following
Trending
Connect
MCP
copy
Create free account
hub
/
github.com/ankane/tokenizers-ruby
/ decode
Method
decode
ext/tokenizers/src/decoders.rs:38–40 ·
view source on GitHub ↗
(&self, tokens: Vec<String>)
Source
from the content-addressed store, hash-verified
36
37
impl RbDecoder {
38
pub fn decode(&self, tokens: Vec<String>) -> RbResult<String> {
39
self.decoder.decode(tokens).map_err(RbError::from)
40
}
41
}
42
43
macro_rules! getter {
Callers
6
test_from_pretrained_bert
Method · 0.45
test_from_pretrained_gpt2
Method · 0.45
test_char_bpe_tokenizer
Method · 0.45
test_encode_add_special_tokens
Method · 0.45
test_decode_with_special_tokens
Method · 0.45
test_getters
Method · 0.45
Calls
no outgoing calls
Tested by
6
test_from_pretrained_bert
Method · 0.36
test_from_pretrained_gpt2
Method · 0.36
test_char_bpe_tokenizer
Method · 0.36
test_encode_add_special_tokens
Method · 0.36
test_decode_with_special_tokens
Method · 0.36
test_getters
Method · 0.36