MCPcopy Create free account
hub / github.com/apache/arrow / ReadBatch

Method ReadBatch

cpp/src/parquet/column_reader.cc:1084–1123  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

1082
1083template <typename DType>
1084int64_t TypedColumnReaderImpl<DType>::ReadBatch(int64_t batch_size, int16_t* def_levels,
1085 int16_t* rep_levels, T* values,
1086 int64_t* values_read) {
1087 // HasNext might invoke ReadNewPage until a data page with
1088 // `available_values_current_page() > 0` is found.
1089 if (!HasNext()) {
1090 *values_read = 0;
1091 return 0;
1092 }
1093
1094 // TODO(wesm): keep reading data pages until batch_size is reached, or the
1095 // row group is finished
1096 int64_t num_def_levels = 0;
1097 // Number of non-null values to read within `num_def_levels`.
1098 int64_t non_null_values_to_read = 0;
1099 ReadLevels(batch_size, def_levels, rep_levels, &num_def_levels,
1100 &non_null_values_to_read);
1101 // Should not return more values than available in the current data page,
1102 // since currently, ReadLevels would only consume level from current
1103 // data page.
1104 if (ARROW_PREDICT_FALSE(num_def_levels > this->available_values_current_page())) {
1105 throw ParquetException(kErrorRepDefLevelNotMatchesNumValues);
1106 }
1107 if (non_null_values_to_read != 0) {
1108 *values_read = this->ReadValues(non_null_values_to_read, values);
1109 } else {
1110 *values_read = 0;
1111 }
1112 // Adjust total_values, since if max_def_level_ == 0, num_def_levels would
1113 // be 0 and `values_read` would adjust to `available_values_current_page()`.
1114 int64_t total_values = std::max<int64_t>(num_def_levels, *values_read);
1115 int64_t expected_values = std::min(batch_size, this->available_values_current_page());
1116 if (total_values == 0 && expected_values > 0) {
1117 std::stringstream ss;
1118 ss << "Read 0 values, expected " << expected_values;
1119 ParquetException::EofException(ss.str());
1120 }
1121 this->ConsumeBufferedValues(total_values);
1122 return total_values;
1123}
1124
1125template <typename DType>
1126void TypedColumnReaderImpl<DType>::InitScratchForSkip() {

Callers 15

ReadFunction · 0.45
ReadOptionalFunction · 0.45
ReadColumnValuesFunction · 0.45
AssertColumnValuesFunction · 0.45
AssertColumnValuesEqualFunction · 0.45
TEST_FFunction · 0.45
CheckReadBatchesMethod · 0.45
TESTFunction · 0.45
TEST_PFunction · 0.45
TestWriteAndReadMethod · 0.45
ReadColumnMethod · 0.45
ReadColumnFullyMethod · 0.45

Calls 6

HasNextFunction · 0.85
ParquetExceptionFunction · 0.85
ReadValuesMethod · 0.80
strMethod · 0.80
ConsumeBufferedValuesMethod · 0.80

Tested by 15

ReadColumnValuesFunction · 0.36
AssertColumnValuesFunction · 0.36
AssertColumnValuesEqualFunction · 0.36
TEST_FFunction · 0.36
CheckReadBatchesMethod · 0.36
TESTFunction · 0.36
TEST_PFunction · 0.36
TestWriteAndReadMethod · 0.36
ReadColumnMethod · 0.36
ReadColumnFullyMethod · 0.36
TESTFunction · 0.36
CheckResultsMethod · 0.36