| 175 | }; |
| 176 | |
| 177 | DictTrie::PrecomputedDict LoadMergedJiebaDict(const std::string& dictPath) { |
| 178 | const std::shared_ptr<MarisaDict> dict = |
| 179 | SerializableDict::NewFromFile<MarisaDict>(dictPath); |
| 180 | const LexiconPtr lexicon = dict->GetLexicon(); |
| 181 | |
| 182 | std::vector<RawMergedEntry> entries; |
| 183 | entries.reserve(lexicon->Length()); |
| 184 | double base_freq_sum = 0.0; |
| 185 | bool saw_metadata = false; |
| 186 | |
| 187 | for (size_t i = 0; i < lexicon->Length(); ++i) { |
| 188 | const DictEntry* entry = lexicon->At(i); |
| 189 | const std::vector<std::string> values = entry->Values(); |
| 190 | RawMergedEntry raw = {entry->Key(), "", 0.0, false, false}; |
| 191 | |
| 192 | if (values.size() == 3) { |
| 193 | saw_metadata = true; |
| 194 | raw.tag = values[1]; |
| 195 | if (values[2] == "base") { |
| 196 | if (!ParsePositiveDouble(values[0], &raw.freq)) { |
| 197 | throw std::runtime_error("invalid merged jieba base frequency for: " + |
| 198 | raw.word); |
| 199 | } |
| 200 | raw.contributes_to_base_sum = true; |
| 201 | } else if (values[2] == "user_freq") { |
| 202 | if (!ParseNonNegativeDouble(values[0], &raw.freq)) { |
| 203 | throw std::runtime_error( |
| 204 | "invalid merged jieba user frequency for: " + raw.word); |
| 205 | } |
| 206 | if (raw.freq == 0.0) { |
| 207 | raw.uses_default_weight = true; |
| 208 | } |
| 209 | } else if (values[2] == "user_default") { |
| 210 | raw.uses_default_weight = true; |
| 211 | } else { |
| 212 | throw std::runtime_error("unknown merged jieba entry kind for: " + |
| 213 | raw.word); |
| 214 | } |
| 215 | } else if (values.size() == 2) { |
| 216 | // Backward-compatible fallback for earlier generated dictionaries. |
| 217 | raw.tag = values[1]; |
| 218 | if (!ParsePositiveDouble(values[0], &raw.freq)) { |
| 219 | throw std::runtime_error("invalid merged jieba frequency for: " + |
| 220 | raw.word); |
| 221 | } |
| 222 | raw.contributes_to_base_sum = true; |
| 223 | } else if (values.size() == 1) { |
| 224 | raw.tag = values[0]; |
| 225 | raw.uses_default_weight = true; |
| 226 | } else { |
| 227 | throw std::runtime_error("invalid merged jieba values for: " + raw.word); |
| 228 | } |
| 229 | |
| 230 | if (raw.contributes_to_base_sum) { |
| 231 | base_freq_sum += raw.freq; |
| 232 | } |
| 233 | entries.push_back(raw); |
| 234 | } |
no test coverage detected