MCPcopy Create free account
hub / github.com/apache/arrow / ReadBatchWithDictionary

Method ReadBatchWithDictionary

cpp/src/parquet/column_reader.cc:1143–1188  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

1141
1142template <typename DType>
1143int64_t TypedColumnReaderImpl<DType>::ReadBatchWithDictionary(
1144 int64_t batch_size, int16_t* def_levels, int16_t* rep_levels, int32_t* indices,
1145 int64_t* indices_read, const T** dict, int32_t* dict_len) {
1146 bool has_dict_output = dict != nullptr && dict_len != nullptr;
1147 // Similar logic as ReadValues to get pages.
1148 if (!HasNext()) {
1149 *indices_read = 0;
1150 if (has_dict_output) {
1151 *dict = nullptr;
1152 *dict_len = 0;
1153 }
1154 return 0;
1155 }
1156
1157 // Verify the current data page is dictionary encoded.
1158 if (this->current_encoding_ != Encoding::RLE_DICTIONARY) {
1159 std::stringstream ss;
1160 ss << "Data page is not dictionary encoded. Encoding: "
1161 << EncodingToString(this->current_encoding_);
1162 throw ParquetException(ss.str());
1163 }
1164
1165 // Get dictionary pointer and length.
1166 if (has_dict_output) {
1167 GetDictionary(dict, dict_len);
1168 }
1169
1170 // Similar logic as ReadValues to get def levels and rep levels.
1171 int64_t num_def_levels = 0;
1172 int64_t indices_to_read = 0;
1173 ReadLevels(batch_size, def_levels, rep_levels, &num_def_levels, &indices_to_read);
1174
1175 // Read dictionary indices.
1176 *indices_read = ReadDictionaryIndices(indices_to_read, indices);
1177 int64_t total_indices = std::max<int64_t>(num_def_levels, *indices_read);
1178 // Some callers use a batch size of 0 just to get the dictionary.
1179 int64_t expected_values = std::min(batch_size, this->available_values_current_page());
1180 if (total_indices == 0 && expected_values > 0) {
1181 std::stringstream ss;
1182 ss << "Read 0 values, expected " << expected_values;
1183 ParquetException::EofException(ss.str());
1184 }
1185 this->ConsumeBufferedValues(total_indices);
1186
1187 return total_indices;
1188}
1189
1190template <typename DType>
1191int64_t TypedColumnReaderImpl<DType>::ReadBatch(int64_t batch_size, int16_t* def_levels,

Callers 2

TESTFunction · 0.80
TEST_FFunction · 0.80

Calls 6

HasNextFunction · 0.85
EncodingToStringFunction · 0.85
ParquetExceptionFunction · 0.85
strMethod · 0.80
ConsumeBufferedValuesMethod · 0.80

Tested by 2

TESTFunction · 0.64
TEST_FFunction · 0.64