Method
decode_batch
(
ruby: &Ruby,
rb_self: &Self,
sequences: Vec<Vec<u32>>,
skip_special_tokens: bool,
)
Source from the content-addressed store, hash-verified
| 602 | } |
| 603 | |
| 604 | pub fn decode_batch( |
| 605 | ruby: &Ruby, |
| 606 | rb_self: &Self, |
| 607 | sequences: Vec<Vec<u32>>, |
| 608 | skip_special_tokens: bool, |
| 609 | ) -> RbResult<Vec<String>> { |
| 610 | ruby.detach(|| { |
| 611 | let slices = sequences.iter().map(|v| &v[..]).collect::<Vec<&[u32]>>(); |
| 612 | rb_self |
| 613 | .tokenizer |
| 614 | .read() |
| 615 | .unwrap() |
| 616 | .decode_batch(&slices, skip_special_tokens) |
| 617 | }) |
| 618 | .map_err(RbError::from) |
| 619 | } |
| 620 | |
| 621 | pub fn token_to_id(&self, token: String) -> RbResult<Option<u32>> { |
| 622 | Ok(self.read_inner()?.token_to_id(&token)) |