Method test_decode_batch

test/tokenizer_test.rb:139–150

  end

  def test_decode_batch
    tokenizer = Tokenizers.from_pretrained("bert-base-cased")

    string_1 = "I can feel the magic, can you?"
    token_ids_1 = [101, 146, 1169, 1631, 1103, 3974, 117, 1169, 1128, 136, 102]

    string_2 = "Am I allowed to pass two text arguments?"
    token_ids_2 = [101, 7277, 146, 2148, 1106, 2789, 1160, 3087, 9989, 136, 102]

    assert_equal [string_1, string_2], tokenizer.decode_batch([token_ids_1, token_ids_2])
    assert_equal ["[CLS] #{string_1} [SEP]", "[CLS] #{string_2} [SEP]"], tokenizer.decode_batch([token_ids_1, token_ids_2], skip_special_tokens: false)
  end

  def test_id_token_conversion
    tokenizer = Tokenizers.from_pretrained("bert-base-cased")