| 132 | } |
| 133 | |
| 134 | inline Status HdfsSequenceScanner::GetRecord(uint8_t** record_ptr, |
| 135 | int64_t* record_len) { |
| 136 | // There are 2 cases: |
| 137 | // - Record-compressed -- like a regular record, but the data is compressed. |
| 138 | // - Uncompressed. |
| 139 | RETURN_IF_ERROR(ReadBlockHeader()); |
| 140 | |
| 141 | // We don't look at the keys, only the values. |
| 142 | RETURN_IF_FALSE(stream_->SkipBytes(current_key_length_, &parse_status_)); |
| 143 | |
| 144 | if (header_->is_compressed) { |
| 145 | int64_t in_size = current_block_length_ - current_key_length_; |
| 146 | if (in_size < 0) { |
| 147 | stringstream ss; |
| 148 | ss << stream_->filename() << " Invalid record size: " << in_size; |
| 149 | return Status(ss.str()); |
| 150 | } |
| 151 | uint8_t* compressed_data; |
| 152 | RETURN_IF_FALSE( |
| 153 | stream_->ReadBytes(in_size, &compressed_data, &parse_status_)); |
| 154 | |
| 155 | int64_t len; |
| 156 | { |
| 157 | SCOPED_TIMER(decompress_timer_); |
| 158 | RETURN_IF_ERROR(decompressor_->ProcessBlock(false, in_size, compressed_data, |
| 159 | &len, &unparsed_data_buffer_)); |
| 160 | VLOG_FILE << "Decompressed " << in_size << " to " << len; |
| 161 | } |
| 162 | *record_ptr = unparsed_data_buffer_; |
| 163 | // Read the length of the record. |
| 164 | int size = ReadWriteUtil::GetVLong(*record_ptr, record_len, in_size); |
| 165 | if (size == -1) { |
| 166 | stringstream ss; |
| 167 | ss << stream_->filename() << " Invalid record size: " << in_size; |
| 168 | return Status(ss.str()); |
| 169 | } |
| 170 | *record_ptr += size; |
| 171 | } else { |
| 172 | // Uncompressed records |
| 173 | RETURN_IF_FALSE(stream_->ReadVLong(record_len, &parse_status_)); |
| 174 | if (*record_len < 0) { |
| 175 | stringstream ss; |
| 176 | ss << stream_->filename() << " Invalid record length: " << *record_len; |
| 177 | return Status(ss.str()); |
| 178 | } |
| 179 | RETURN_IF_FALSE( |
| 180 | stream_->ReadBytes(*record_len, record_ptr, &parse_status_)); |
| 181 | } |
| 182 | return Status::OK(); |
| 183 | } |
| 184 | |
| 185 | // Process block compressed sequence files. This is the most used sequence file |
| 186 | // format. The general strategy is to process the data in large chunks to minimize |