Method test_id_token_conversion

test/tokenizer_test.rb:152–156

  end

  def test_id_token_conversion
    tokenizer = Tokenizers.from_pretrained("bert-base-cased")
    assert_equal 1169, tokenizer.token_to_id("can")
    assert_equal "magic", tokenizer.id_to_token(3974)
  end

  def test_vocab_size
    tokenizer = Tokenizers.from_pretrained("bert-base-cased")