| 51 | end |
| 52 | |
| 53 | def test_char_bpe_tokenizer |
| 54 | vocab = "test/support/roberta-base-vocab.json" |
| 55 | merges = "test/support/roberta-base-merges.txt" |
| 56 | tokenizer = Tokenizers::CharBPETokenizer.new(vocab, merges) |
| 57 | |
| 58 | # encode |
| 59 | encoded = tokenizer.encode("I can feel the magic, can you?", add_special_tokens: false) |
| 60 | expected_ids = [3, 3245, 3, 33763, 3, 212, 3, 119, 18879, 3, 3, 3245, 3, 9839, 3, 3] |
| 61 | expected_tokens = ["<unk>", "ca", "<unk>", "fee", "<unk>", "th", "<unk>", "m", "agi", "<unk>", "<unk>", "ca", "<unk>", "yo", "<unk>", "<unk>"] |
| 62 | assert_equal expected_ids, encoded.ids |
| 63 | assert_equal expected_tokens, encoded.tokens |
| 64 | |
| 65 | # decode |
| 66 | assert_equal "cafeethmagicayo", tokenizer.decode(encoded.ids) |
| 67 | end |
| 68 | |
| 69 | def test_encode_add_special_tokens |
| 70 | tokenizer = Tokenizers.from_pretrained("bert-base-cased") |