Method
encode_batch
(
ruby: &Ruby,
rb_self: &Self,
input: RArray,
is_pretokenized: bool,
add_special_tokens: bool,
)
Source from the content-addressed store, hash-verified
| 540 | } |
| 541 | |
| 542 | pub fn encode_batch( |
| 543 | ruby: &Ruby, |
| 544 | rb_self: &Self, |
| 545 | input: RArray, |
| 546 | is_pretokenized: bool, |
| 547 | add_special_tokens: bool, |
| 548 | ) -> RbResult<RArray> { |
| 549 | let input: Vec<tk::EncodeInput> = input |
| 550 | .into_iter() |
| 551 | .map(|o| { |
| 552 | let input: tk::EncodeInput = if is_pretokenized { |
| 553 | PreTokenizedEncodeInput::try_convert(o)?.into() |
| 554 | } else { |
| 555 | TextEncodeInput::try_convert(o)?.into() |
| 556 | }; |
| 557 | Ok(input) |
| 558 | }) |
| 559 | .collect::<RbResult<Vec<tk::EncodeInput>>>()?; |
| 560 | ruby.detach(|| { |
| 561 | rb_self |
| 562 | .tokenizer |
| 563 | .read() |
| 564 | .unwrap() |
| 565 | .encode_batch_char_offsets(input, add_special_tokens) |
| 566 | }) |
| 567 | .map(|encodings| ruby.ary_from_iter(encodings.into_iter().map(Into::<RbEncoding>::into))) |
| 568 | .map_err(RbError::from) |
| 569 | } |
| 570 | |
| 571 | pub fn encode_batch_fast( |
| 572 | ruby: &Ruby, |