| 19 | } |
| 20 | |
| 21 | void LanguageModel::initialize(ModelReader& model_reader) { |
| 22 | if (binary_version() < 6) { |
| 23 | config["unk_token"] = get_attribute_with_default<std::string>("unk_token", "<unk>"); |
| 24 | config["bos_token"] = get_attribute_with_default<std::string>("bos_token", "<s>"); |
| 25 | config["eos_token"] = get_attribute_with_default<std::string>("eos_token", "</s>"); |
| 26 | } |
| 27 | |
| 28 | VocabularyInfo vocab_info; |
| 29 | vocab_info.unk_token = config["unk_token"]; |
| 30 | vocab_info.bos_token = config["bos_token"]; |
| 31 | vocab_info.eos_token = config["eos_token"]; |
| 32 | |
| 33 | _vocabulary = load_vocabulary(model_reader, "vocabulary", std::move(vocab_info)); |
| 34 | if (!_vocabulary) |
| 35 | throw std::runtime_error("Cannot load the vocabulary from the model directory"); |
| 36 | } |
| 37 | |
| 38 | |
| 39 | std::vector<ScoringResult> |
nothing calls this directly
no test coverage detected