| 67 | end |
| 68 | |
| 69 | def test_encode_add_special_tokens |
| 70 | tokenizer = Tokenizers.from_pretrained("bert-base-cased") |
| 71 | |
| 72 | # encode |
| 73 | encoded = tokenizer.encode("I can feel the magic, can you?", add_special_tokens: false) |
| 74 | expected_ids = [146, 1169, 1631, 1103, 3974, 117, 1169, 1128, 136] |
| 75 | expected_tokens = ["I", "can", "feel", "the", "magic", ",", "can", "you", "?"] |
| 76 | assert_equal expected_ids, encoded.ids |
| 77 | assert_equal expected_tokens, encoded.tokens |
| 78 | |
| 79 | # decode |
| 80 | assert_equal "I can feel the magic, can you?", tokenizer.decode(encoded.ids) |
| 81 | end |
| 82 | |
| 83 | def test_encode_multibyte_offsets |
| 84 | tokenizer = Tokenizers.from_pretrained("gpt2") |
nothing calls this directly
no test coverage detected