Method test_tokenization

tests/test_modeling_bart.py:478–487

(self)

    @slow
    def test_tokenization(self):
        tokenizer = BartTokenizer.from_pretrained("facebook/bart-large")
        examples = [" Hello world", " DomDramg"]  # need leading spaces for equality
        fairseq_results = [
            torch.Tensor([0, 20920, 232, 2]),
            torch.Tensor([0, 11349, 495, 4040, 571, 2]),
        ]
        for ex, desired_result in zip(examples, fairseq_results):
            bart_toks = tokenizer.encode(ex, return_tensors="pt")
            _assert_tensors_equal(desired_result.long(), bart_toks, prefix=ex)

    def test_generate_fp16(self):
        config, input_ids, batch_size = self._get_config_and_data()