MCPcopy Create free account
hub / github.com/apache/arrow / ReadBatch

Method ReadBatch

cpp/src/parquet/column_reader.cc:1099–1138  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

1097
1098template <typename DType>
1099int64_t TypedColumnReaderImpl<DType>::ReadBatch(int64_t batch_size, int16_t* def_levels,
1100 int16_t* rep_levels, T* values,
1101 int64_t* values_read) {
1102 // HasNext might invoke ReadNewPage until a data page with
1103 // `available_values_current_page() > 0` is found.
1104 if (!HasNext()) {
1105 *values_read = 0;
1106 return 0;
1107 }
1108
1109 // TODO(wesm): keep reading data pages until batch_size is reached, or the
1110 // row group is finished
1111 int64_t num_def_levels = 0;
1112 // Number of non-null values to read within `num_def_levels`.
1113 int64_t non_null_values_to_read = 0;
1114 ReadLevels(batch_size, def_levels, rep_levels, &num_def_levels,
1115 &non_null_values_to_read);
1116 // Should not return more values than available in the current data page,
1117 // since currently, ReadLevels would only consume level from current
1118 // data page.
1119 if (ARROW_PREDICT_FALSE(num_def_levels > this->available_values_current_page())) {
1120 throw ParquetException(kErrorRepDefLevelNotMatchesNumValues);
1121 }
1122 if (non_null_values_to_read != 0) {
1123 *values_read = this->ReadValues(non_null_values_to_read, values);
1124 } else {
1125 *values_read = 0;
1126 }
1127 // Adjust total_values, since if max_def_level_ == 0, num_def_levels would
1128 // be 0 and `values_read` would adjust to `available_values_current_page()`.
1129 int64_t total_values = std::max<int64_t>(num_def_levels, *values_read);
1130 int64_t expected_values = std::min(batch_size, this->available_values_current_page());
1131 if (total_values == 0 && expected_values > 0) {
1132 std::stringstream ss;
1133 ss << "Read 0 values, expected " << expected_values;
1134 ParquetException::EofException(ss.str());
1135 }
1136 this->ConsumeBufferedValues(total_values);
1137 return total_values;
1138}
1139
1140template <typename DType>
1141void TypedColumnReaderImpl<DType>::InitScratchForSkip() {

Callers 15

ReadFunction · 0.45
ReadOptionalFunction · 0.45
ReadColumnValuesFunction · 0.45
AssertColumnValuesFunction · 0.45
AssertColumnValuesEqualFunction · 0.45
TEST_FFunction · 0.45
CheckReadBatchesMethod · 0.45
TESTFunction · 0.45
TEST_PFunction · 0.45
TestWriteAndReadMethod · 0.45
ReadColumnMethod · 0.45
ReadColumnFullyMethod · 0.45

Calls 6

HasNextFunction · 0.85
ParquetExceptionFunction · 0.85
ReadValuesMethod · 0.80
strMethod · 0.80
ConsumeBufferedValuesMethod · 0.80

Tested by 15

ReadColumnValuesFunction · 0.36
AssertColumnValuesFunction · 0.36
AssertColumnValuesEqualFunction · 0.36
TEST_FFunction · 0.36
CheckReadBatchesMethod · 0.36
TESTFunction · 0.36
TEST_PFunction · 0.36
TestWriteAndReadMethod · 0.36
ReadColumnMethod · 0.36
ReadColumnFullyMethod · 0.36
TESTFunction · 0.36
CheckResultsMethod · 0.36