Method test_from_pretrained_bert

test/tokenizer_test.rb:4–16

class TokenizerTest < Minitest::Test
  def test_from_pretrained_bert
    tokenizer = Tokenizers.from_pretrained("bert-base-cased")

    # encode
    encoded = tokenizer.encode("I can feel the magic, can you?")
    expected_ids = [101, 146, 1169, 1631, 1103, 3974, 117, 1169, 1128, 136, 102]
    expected_tokens = ["[CLS]", "I", "can", "feel", "the", "magic", ",", "can", "you", "?", "[SEP]"]
    assert_equal expected_ids, encoded.ids
    assert_equal expected_tokens, encoded.tokens

    # decode
    assert_equal "I can feel the magic, can you?", tokenizer.decode(encoded.ids)
  end

  def test_from_pretrained_gpt2
    tokenizer = Tokenizers.from_pretrained("gpt2")