| 90 | end |
| 91 | |
| 92 | def test_encode_pair_encoding |
| 93 | tokenizer = Tokenizers.from_pretrained("bert-base-cased") |
| 94 | question = "Am I allowed to pass two text arguments?" |
| 95 | answer = "Yes I am!" |
| 96 | encoded = tokenizer.encode(question, answer) |
| 97 | expected_tokens = ["[CLS]", "Am", "I", "allowed", "to", "pass", "two", "text", "arguments", "?", "[SEP]", "Yes", "I", "am", "!", "[SEP]"] |
| 98 | assert_equal expected_tokens, encoded.tokens |
| 99 | end |
| 100 | |
| 101 | def test_encode_pretokenized_encoding |
| 102 | tokenizer = Tokenizers.from_pretrained("bert-base-cased") |
nothing calls this directly
no test coverage detected