| 113 | end |
| 114 | |
| 115 | def test_encode_batch |
| 116 | tokenizer = Tokenizers.from_pretrained("bert-base-cased") |
| 117 | encoded = tokenizer.encode_batch(["I can feel the magic, can you?"]) |
| 118 | assert_equal 1, encoded.size |
| 119 | expected_ids = [101, 146, 1169, 1631, 1103, 3974, 117, 1169, 1128, 136, 102] |
| 120 | assert_equal expected_ids, encoded[0].ids |
| 121 | assert_equal ["I can feel the magic, can you?"], tokenizer.decode_batch(encoded.map(&:ids)) |
| 122 | end |
| 123 | |
| 124 | def test_encode_batch_fast |
| 125 | tokenizer = Tokenizers.from_pretrained("bert-base-cased") |
nothing calls this directly
no test coverage detected