MCPcopy Create free account
hub / github.com/apache/arrow / ReadBatchWithDictionary

Method ReadBatchWithDictionary

cpp/src/parquet/column_reader.cc:1036–1081  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

1034
1035template <typename DType>
1036int64_t TypedColumnReaderImpl<DType>::ReadBatchWithDictionary(
1037 int64_t batch_size, int16_t* def_levels, int16_t* rep_levels, int32_t* indices,
1038 int64_t* indices_read, const T** dict, int32_t* dict_len) {
1039 bool has_dict_output = dict != nullptr && dict_len != nullptr;
1040 // Similar logic as ReadValues to get pages.
1041 if (!HasNext()) {
1042 *indices_read = 0;
1043 if (has_dict_output) {
1044 *dict = nullptr;
1045 *dict_len = 0;
1046 }
1047 return 0;
1048 }
1049
1050 // Verify the current data page is dictionary encoded.
1051 if (this->current_encoding_ != Encoding::RLE_DICTIONARY) {
1052 std::stringstream ss;
1053 ss << "Data page is not dictionary encoded. Encoding: "
1054 << EncodingToString(this->current_encoding_);
1055 throw ParquetException(ss.str());
1056 }
1057
1058 // Get dictionary pointer and length.
1059 if (has_dict_output) {
1060 GetDictionary(dict, dict_len);
1061 }
1062
1063 // Similar logic as ReadValues to get def levels and rep levels.
1064 int64_t num_def_levels = 0;
1065 int64_t indices_to_read = 0;
1066 ReadLevels(batch_size, def_levels, rep_levels, &num_def_levels, &indices_to_read);
1067
1068 // Read dictionary indices.
1069 *indices_read = ReadDictionaryIndices(indices_to_read, indices);
1070 int64_t total_indices = std::max<int64_t>(num_def_levels, *indices_read);
1071 // Some callers use a batch size of 0 just to get the dictionary.
1072 int64_t expected_values = std::min(batch_size, this->available_values_current_page());
1073 if (total_indices == 0 && expected_values > 0) {
1074 std::stringstream ss;
1075 ss << "Read 0 values, expected " << expected_values;
1076 ParquetException::EofException(ss.str());
1077 }
1078 this->ConsumeBufferedValues(total_indices);
1079
1080 return total_indices;
1081}
1082
1083template <typename DType>
1084int64_t TypedColumnReaderImpl<DType>::ReadBatch(int64_t batch_size, int16_t* def_levels,

Callers 2

TESTFunction · 0.80
TEST_FFunction · 0.80

Calls 6

HasNextFunction · 0.85
EncodingToStringFunction · 0.85
ParquetExceptionFunction · 0.85
strMethod · 0.80
ConsumeBufferedValuesMethod · 0.80

Tested by 2

TESTFunction · 0.64
TEST_FFunction · 0.64