| 510 | } |
| 511 | |
| 512 | pub fn encode( |
| 513 | &self, |
| 514 | sequence: Value, |
| 515 | pair: Option<Value>, |
| 516 | is_pretokenized: bool, |
| 517 | add_special_tokens: bool, |
| 518 | ) -> RbResult<RbEncoding> { |
| 519 | let sequence: tk::InputSequence = if is_pretokenized { |
| 520 | PreTokenizedInputSequence::try_convert(sequence)?.into() |
| 521 | } else { |
| 522 | TextInputSequence::try_convert(sequence)?.into() |
| 523 | }; |
| 524 | let input = match pair { |
| 525 | Some(pair) => { |
| 526 | let pair: tk::InputSequence = if is_pretokenized { |
| 527 | PreTokenizedInputSequence::try_convert(pair)?.into() |
| 528 | } else { |
| 529 | TextInputSequence::try_convert(pair)?.into() |
| 530 | }; |
| 531 | tk::EncodeInput::Dual(sequence, pair) |
| 532 | } |
| 533 | None => tk::EncodeInput::Single(sequence), |
| 534 | }; |
| 535 | |
| 536 | self.read_inner()? |
| 537 | .encode_char_offsets(input, add_special_tokens) |
| 538 | .map(|v| RbEncoding { encoding: v }) |
| 539 | .map_err(RbError::from) |
| 540 | } |
| 541 | |
| 542 | pub fn encode_batch( |
| 543 | ruby: &Ruby, |