| 96 | } |
| 97 | |
| 98 | void Next() override { |
| 99 | if (!valid_) return; |
| 100 | |
| 101 | string line; |
| 102 | status_ = input_buffer_->ReadLine(&line); |
| 103 | if (!status_.ok()) { |
| 104 | if (errors::IsOutOfRange(status_) && vocab_size_ != -1 && |
| 105 | next_id_ != vocab_size_) { |
| 106 | status_ = errors::InvalidArgument("Invalid vocab_size in ", filename_, |
| 107 | ": expected ", vocab_size_, |
| 108 | " but got ", next_id_); |
| 109 | } |
| 110 | valid_ = false; |
| 111 | return; |
| 112 | } |
| 113 | if (vocab_size_ != -1 && next_id_ >= vocab_size_) { |
| 114 | LOG(WARNING) << "Truncated " << filename_ << " before its end at " |
| 115 | << vocab_size_ << " records."; |
| 116 | LOG(WARNING) << "next_id_ : " << next_id_; |
| 117 | status_ = errors::OutOfRange("Finished reading ", vocab_size_, |
| 118 | " of lines from ", filename_); |
| 119 | valid_ = false; |
| 120 | return; |
| 121 | } |
| 122 | if (line.empty()) { |
| 123 | status_ = errors::InvalidArgument("Invalid content in ", filename_, |
| 124 | ": empty line found at position ", |
| 125 | input_buffer_->Tell(), "."); |
| 126 | valid_ = false; |
| 127 | return; |
| 128 | } |
| 129 | |
| 130 | std::vector<string> tokens; |
| 131 | if (!ignore_split_) { |
| 132 | tokens = str_util::Split(line, delimiter_); |
| 133 | if (std::max(key_index_, value_index_) >= tokens.size()) { |
| 134 | status_ = errors::InvalidArgument( |
| 135 | "Invalid number of columns in ", filename_, " line ", next_id_, |
| 136 | " (", line, ") : expected ", std::max(key_index_, value_index_), |
| 137 | " got ", tokens.size()); |
| 138 | valid_ = false; |
| 139 | return; |
| 140 | } |
| 141 | } |
| 142 | status_ = SetValue(line, tokens, key_index_, &key_); |
| 143 | if (!status_.ok()) { |
| 144 | valid_ = false; |
| 145 | return; |
| 146 | } |
| 147 | status_ = SetValue(line, tokens, value_index_, &value_); |
| 148 | if (!status_.ok()) { |
| 149 | valid_ = false; |
| 150 | return; |
| 151 | } |
| 152 | |
| 153 | next_id_++; |
| 154 | } |
| 155 | |