MCPcopy Create free account
hub / github.com/apache/arrow / ReadBatchWithDictionary

Method ReadBatchWithDictionary

cpp/src/parquet/column_reader.cc:1051–1096  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

1049
1050template <typename DType>
1051int64_t TypedColumnReaderImpl<DType>::ReadBatchWithDictionary(
1052 int64_t batch_size, int16_t* def_levels, int16_t* rep_levels, int32_t* indices,
1053 int64_t* indices_read, const T** dict, int32_t* dict_len) {
1054 bool has_dict_output = dict != nullptr && dict_len != nullptr;
1055 // Similar logic as ReadValues to get pages.
1056 if (!HasNext()) {
1057 *indices_read = 0;
1058 if (has_dict_output) {
1059 *dict = nullptr;
1060 *dict_len = 0;
1061 }
1062 return 0;
1063 }
1064
1065 // Verify the current data page is dictionary encoded.
1066 if (this->current_encoding_ != Encoding::RLE_DICTIONARY) {
1067 std::stringstream ss;
1068 ss << "Data page is not dictionary encoded. Encoding: "
1069 << EncodingToString(this->current_encoding_);
1070 throw ParquetException(ss.str());
1071 }
1072
1073 // Get dictionary pointer and length.
1074 if (has_dict_output) {
1075 GetDictionary(dict, dict_len);
1076 }
1077
1078 // Similar logic as ReadValues to get def levels and rep levels.
1079 int64_t num_def_levels = 0;
1080 int64_t indices_to_read = 0;
1081 ReadLevels(batch_size, def_levels, rep_levels, &num_def_levels, &indices_to_read);
1082
1083 // Read dictionary indices.
1084 *indices_read = ReadDictionaryIndices(indices_to_read, indices);
1085 int64_t total_indices = std::max<int64_t>(num_def_levels, *indices_read);
1086 // Some callers use a batch size of 0 just to get the dictionary.
1087 int64_t expected_values = std::min(batch_size, this->available_values_current_page());
1088 if (total_indices == 0 && expected_values > 0) {
1089 std::stringstream ss;
1090 ss << "Read 0 values, expected " << expected_values;
1091 ParquetException::EofException(ss.str());
1092 }
1093 this->ConsumeBufferedValues(total_indices);
1094
1095 return total_indices;
1096}
1097
1098template <typename DType>
1099int64_t TypedColumnReaderImpl<DType>::ReadBatch(int64_t batch_size, int16_t* def_levels,

Callers 2

TESTFunction · 0.80
TEST_FFunction · 0.80

Calls 6

HasNextFunction · 0.85
EncodingToStringFunction · 0.85
ParquetExceptionFunction · 0.85
strMethod · 0.80
ConsumeBufferedValuesMethod · 0.80

Tested by 2

TESTFunction · 0.64
TEST_FFunction · 0.64