MCPcopy Create free account
hub / github.com/apache/arrow / ReadBatch

Method ReadBatch

cpp/src/parquet/column_reader.cc:1191–1230  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

1189
1190template <typename DType>
1191int64_t TypedColumnReaderImpl<DType>::ReadBatch(int64_t batch_size, int16_t* def_levels,
1192 int16_t* rep_levels, T* values,
1193 int64_t* values_read) {
1194 // HasNext might invoke ReadNewPage until a data page with
1195 // `available_values_current_page() > 0` is found.
1196 if (!HasNext()) {
1197 *values_read = 0;
1198 return 0;
1199 }
1200
1201 // TODO(wesm): keep reading data pages until batch_size is reached, or the
1202 // row group is finished
1203 int64_t num_def_levels = 0;
1204 // Number of non-null values to read within `num_def_levels`.
1205 int64_t non_null_values_to_read = 0;
1206 ReadLevels(batch_size, def_levels, rep_levels, &num_def_levels,
1207 &non_null_values_to_read);
1208 // Should not return more values than available in the current data page,
1209 // since currently, ReadLevels would only consume level from current
1210 // data page.
1211 if (ARROW_PREDICT_FALSE(num_def_levels > this->available_values_current_page())) {
1212 throw ParquetException(kErrorRepDefLevelNotMatchesNumValues);
1213 }
1214 if (non_null_values_to_read != 0) {
1215 *values_read = this->ReadValues(non_null_values_to_read, values);
1216 } else {
1217 *values_read = 0;
1218 }
1219 // Adjust total_values, since if max_def_level_ == 0, num_def_levels would
1220 // be 0 and `values_read` would adjust to `available_values_current_page()`.
1221 int64_t total_values = std::max<int64_t>(num_def_levels, *values_read);
1222 int64_t expected_values = std::min(batch_size, this->available_values_current_page());
1223 if (total_values == 0 && expected_values > 0) {
1224 std::stringstream ss;
1225 ss << "Read 0 values, expected " << expected_values;
1226 ParquetException::EofException(ss.str());
1227 }
1228 this->ConsumeBufferedValues(total_values);
1229 return total_values;
1230}
1231
1232template <typename DType>
1233int64_t TypedColumnReaderImpl<DType>::Skip(int64_t num_values_to_skip) {

Callers 15

ReadFunction · 0.45
ReadOptionalFunction · 0.45
ReadColumnValuesFunction · 0.45
AssertColumnValuesFunction · 0.45
AssertColumnValuesEqualFunction · 0.45
TEST_FFunction · 0.45
CheckReadBatchesMethod · 0.45
TESTFunction · 0.45
TEST_PFunction · 0.45
TestWriteAndReadMethod · 0.45
ReadColumnMethod · 0.45
ReadColumnFullyMethod · 0.45

Calls 6

HasNextFunction · 0.85
ParquetExceptionFunction · 0.85
ReadValuesMethod · 0.80
strMethod · 0.80
ConsumeBufferedValuesMethod · 0.80

Tested by 15

ReadColumnValuesFunction · 0.36
AssertColumnValuesFunction · 0.36
AssertColumnValuesEqualFunction · 0.36
TEST_FFunction · 0.36
CheckReadBatchesMethod · 0.36
TESTFunction · 0.36
TEST_PFunction · 0.36
TestWriteAndReadMethod · 0.36
ReadColumnMethod · 0.36
ReadColumnFullyMethod · 0.36
TESTFunction · 0.36
CheckResultsMethod · 0.36