| 323 | } |
| 324 | |
| 325 | util::Status TrainerInterface::LoadSentences() { |
| 326 | RETURN_IF_ERROR(status()); |
| 327 | CHECK_OR_RETURN(sentences_.empty()); |
| 328 | CHECK_OR_RETURN(required_chars_.empty()); |
| 329 | CHECK_OR_RETURN(trainer_spec_.input_format().empty() || |
| 330 | trainer_spec_.input_format() == "text" || |
| 331 | trainer_spec_.input_format() == "tsv") |
| 332 | << "Supported formats are 'text' and 'tsv'."; |
| 333 | |
| 334 | CHECK_OR_RETURN( |
| 335 | (sentence_iterator_ != nullptr && trainer_spec_.input().empty()) || |
| 336 | (sentence_iterator_ == nullptr && !trainer_spec_.input().empty())) |
| 337 | << "SentenceIterator and trainer_spec.input() must be exclusive."; |
| 338 | |
| 339 | CHECK_OR_RETURN( |
| 340 | (output_model_proto_ != nullptr && |
| 341 | trainer_spec_.model_prefix().empty()) || |
| 342 | (output_model_proto_ == nullptr && !trainer_spec_.model_prefix().empty())) |
| 343 | << "ModelProto and trainer_spec.model_prefix() must be exclusive."; |
| 344 | |
| 345 | const bool is_tsv = trainer_spec_.input_format() == "tsv"; |
| 346 | |
| 347 | SentenceSelector selector(&sentences_, trainer_spec_); |
| 348 | random::ReservoirSampler<std::string> test_sentence_sampler( |
| 349 | &self_test_samples_, trainer_spec_.self_test_sample_size()); |
| 350 | |
| 351 | int too_long_lines = 0; |
| 352 | |
| 353 | std::unique_ptr<SentenceIterator> sentence_iterator_impl; |
| 354 | if (sentence_iterator_ == nullptr) { |
| 355 | LOG(INFO) << "SentenceIterator is not specified. Using " |
| 356 | "MultiFileSentenceIterator."; |
| 357 | sentence_iterator_impl = |
| 358 | std::make_unique<MultiFileSentenceIterator>(std::vector<std::string>( |
| 359 | trainer_spec_.input().begin(), trainer_spec_.input().end())); |
| 360 | sentence_iterator_ = sentence_iterator_impl.get(); |
| 361 | } |
| 362 | |
| 363 | for (; !sentence_iterator_->done(); sentence_iterator_->Next()) { |
| 364 | int64_t freq = 1; |
| 365 | std::string sentence = sentence_iterator_->value(); |
| 366 | |
| 367 | if (is_tsv) { |
| 368 | const std::vector<std::string> v = absl::StrSplit(sentence, '\t'); |
| 369 | CHECK_EQ_OR_RETURN(v.size(), 2) |
| 370 | << "Input format must be: word <tab> freq. " << sentence; |
| 371 | sentence = v[0]; |
| 372 | CHECK_OR_RETURN(absl::SimpleAtoi(v[1], &freq)) |
| 373 | << "Could not parse the frequency"; |
| 374 | CHECK_GE_OR_RETURN(freq, 1); |
| 375 | } |
| 376 | |
| 377 | if (sentence.empty()) continue; |
| 378 | |
| 379 | if (static_cast<int>(sentence.size()) > |
| 380 | trainer_spec_.max_sentence_length()) { |
| 381 | if (too_long_lines == 0) { |
| 382 | LOG(WARNING) << "Found too long line (" << sentence.size() << " > " |