| 184 | end |
| 185 | |
| 186 | def test_padding |
| 187 | tokenizer = Tokenizers.from_pretrained("bert-base-cased") |
| 188 | assert_nil tokenizer.padding |
| 189 | |
| 190 | tokenizer.enable_padding |
| 191 | default_padding = {"length"=>nil, "pad_id"=>0, "pad_type_id"=>0, "pad_token"=>"[PAD]", "pad_to_multiple_of"=>nil, "direction"=>"right"} |
| 192 | assert_equal default_padding, tokenizer.padding |
| 193 | |
| 194 | tokenizer.enable_padding(length: 1024, direction: "left", pad_to_multiple_of: 256, pad_id: 29000, pad_type_id: 1, pad_token: "[SSS]") |
| 195 | configured_padding = {"length"=>1024, "pad_id"=>29000, "pad_type_id"=>1, "pad_token"=>"[SSS]", "pad_to_multiple_of"=>256, "direction"=>"left"} |
| 196 | assert_equal configured_padding, tokenizer.padding |
| 197 | |
| 198 | tokenizer.no_padding |
| 199 | assert_nil tokenizer.padding |
| 200 | end |
| 201 | |
| 202 | def test_truncation |
| 203 | tokenizer = Tokenizers.from_pretrained("bert-base-cased") |
nothing calls this directly
no test coverage detected