MCPcopy Create free account
hub / github.com/ankane/tokenizers-ruby / enable_padding

Method enable_padding

ext/tokenizers/src/tokenizer.rs:428–481  ·  view source on GitHub ↗

TODO support more kwargs

(ruby: &Ruby, rb_self: &Self, kwargs: RHash)

Source from the content-addressed store, hash-verified

426
427 // TODO support more kwargs
428 pub fn enable_padding(ruby: &Ruby, rb_self: &Self, kwargs: RHash) -> RbResult<()> {
429 let mut params = PaddingParams::default();
430
431 let value: Value = kwargs.delete(ruby.to_symbol("direction"))?;
432 if !value.is_nil() {
433 let dir_str = String::try_convert(value)?;
434 params.direction = match dir_str.as_str() {
435 "left" => PaddingDirection::Left,
436 "right" => PaddingDirection::Right,
437 _ => {
438 return Err(Error::new(
439 ruby.exception_arg_error(),
440 "The direction value must be 'left' or 'right'",
441 ))
442 }
443 }
444 }
445
446 let value: Value = kwargs.delete(ruby.to_symbol("pad_to_multiple_of"))?;
447 if !value.is_nil() {
448 params.pad_to_multiple_of = TryConvert::try_convert(value)?;
449 }
450
451 let value: Value = kwargs.delete(ruby.to_symbol("pad_id"))?;
452 if !value.is_nil() {
453 params.pad_id = TryConvert::try_convert(value)?;
454 }
455
456 let value: Value = kwargs.delete(ruby.to_symbol("pad_type_id"))?;
457 if !value.is_nil() {
458 params.pad_type_id = TryConvert::try_convert(value)?;
459 }
460
461 let value: Value = kwargs.delete(ruby.to_symbol("pad_token"))?;
462 if !value.is_nil() {
463 params.pad_token = TryConvert::try_convert(value)?;
464 }
465
466 let value: Value = kwargs.delete(ruby.to_symbol("length"))?;
467 if value.is_nil() {
468 params.strategy = PaddingStrategy::BatchLongest;
469 } else {
470 params.strategy = PaddingStrategy::Fixed(TryConvert::try_convert(value)?);
471 }
472
473 if !kwargs.is_empty() {
474 // TODO improve message
475 return Err(Error::new(ruby.exception_arg_error(), "unknown keyword"));
476 }
477
478 rb_self.write_inner()?.with_padding(Some(params));
479
480 Ok(())
481 }
482
483 pub fn no_padding(&self) -> RbResult<()> {
484 self.write_inner()?.with_padding(None);

Callers 2

test_paddingMethod · 0.45
test_worksMethod · 0.45

Calls 1

write_innerMethod · 0.80

Tested by 2

test_paddingMethod · 0.36
test_worksMethod · 0.36