| 340 | } |
| 341 | |
| 342 | util::Status SentencePieceProcessor::LoadVocabulary(absl::string_view filename, |
| 343 | int threshold) { |
| 344 | auto input = filesystem::NewReadableFile(filename); |
| 345 | RETURN_IF_ERROR(input->status()); |
| 346 | |
| 347 | std::string line; |
| 348 | std::vector<std::string> vocab; |
| 349 | |
| 350 | while (input->ReadLine(&line)) { |
| 351 | const std::vector<std::string> v = absl::StrSplit(line, '\t'); |
| 352 | CHECK_GE_OR_RETURN(v.size(), 1); |
| 353 | CHECK_OR_RETURN(!v[0].empty()); |
| 354 | int32_t freq = 1; |
| 355 | if (v.size() >= 2) { |
| 356 | CHECK_OR_RETURN(absl::SimpleAtoi(v[1], &freq)) |
| 357 | << "Could not parse the frequency"; |
| 358 | } |
| 359 | if (freq >= threshold) { |
| 360 | vocab.emplace_back(v[0]); |
| 361 | } |
| 362 | } |
| 363 | |
| 364 | return SetVocabulary(ToPieceArray(vocab)); |
| 365 | } |
| 366 | |
| 367 | #define CHECK_OR_RETURN_STATUS_STL(container) \ |
| 368 | RETURN_IF_ERROR(status()); \ |