| 1694 | }; |
| 1695 | |
| 1696 | void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { |
| 1697 | struct gguf_context * ctx = ml.meta.get(); |
| 1698 | |
| 1699 | // determine vocab type |
| 1700 | { |
| 1701 | ml.get_key(LLM_KV_TOKENIZER_MODEL, tokenizer_model); |
| 1702 | ml.get_key(LLM_KV_TOKENIZER_PRE, tokenizer_pre, false); |
| 1703 | |
| 1704 | ml.get_key(LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, n_token_types, false); |
| 1705 | |
| 1706 | if (tokenizer_model == "no_vocab" || tokenizer_model == "none") { |
| 1707 | type = LLAMA_VOCAB_TYPE_NONE; |
| 1708 | |
| 1709 | // default special tokens |
| 1710 | special_bos_id = LLAMA_TOKEN_NULL; |
| 1711 | special_eos_id = LLAMA_TOKEN_NULL; |
| 1712 | special_unk_id = LLAMA_TOKEN_NULL; |
| 1713 | special_sep_id = LLAMA_TOKEN_NULL; |
| 1714 | special_pad_id = LLAMA_TOKEN_NULL; |
| 1715 | special_mask_id = LLAMA_TOKEN_NULL; |
| 1716 | linefeed_id = LLAMA_TOKEN_NULL; |
| 1717 | |
| 1718 | // read vocab size from metadata |
| 1719 | uint32_t n_tokens = 0; |
| 1720 | if (ml.get_key(LLM_KV_VOCAB_SIZE, n_tokens, false)) { |
| 1721 | LLAMA_LOG_WARN("%s: adding %u dummy tokens\n", __func__, n_tokens); |
| 1722 | id_to_token.resize(n_tokens); |
| 1723 | } |
| 1724 | |
| 1725 | return; |
| 1726 | } |
| 1727 | |
| 1728 | if (tokenizer_model == "llama") { |
| 1729 | type = LLAMA_VOCAB_TYPE_SPM; |
| 1730 | |
| 1731 | // default special tokens |
| 1732 | special_bos_id = 1; |
| 1733 | special_eos_id = 2; |
| 1734 | special_unk_id = 0; |
| 1735 | special_sep_id = LLAMA_TOKEN_NULL; |
| 1736 | special_pad_id = LLAMA_TOKEN_NULL; |
| 1737 | special_mask_id = LLAMA_TOKEN_NULL; |
| 1738 | } else if (tokenizer_model == "bert") { |
| 1739 | type = LLAMA_VOCAB_TYPE_WPM; |
| 1740 | |
| 1741 | // default special tokens |
| 1742 | special_bos_id = 101; |
| 1743 | special_eos_id = LLAMA_TOKEN_NULL; |
| 1744 | special_unk_id = 100; |
| 1745 | special_sep_id = 102; |
| 1746 | special_pad_id = 0; |
| 1747 | special_mask_id = 103; |
| 1748 | |
| 1749 | add_sep = true; |
| 1750 | } else if (tokenizer_model == "gpt2") { |
| 1751 | type = LLAMA_VOCAB_TYPE_BPE; |
| 1752 | |
| 1753 | // read bpe merges and populate bpe ranks |
nothing calls this directly
no test coverage detected