MCPcopy Create free account
hub / github.com/apache/arrow / DefRepLevelsToListInfo

Function DefRepLevelsToListInfo

cpp/src/parquet/level_conversion.cc:42–118  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

40
41template <typename OffsetType>
42void DefRepLevelsToListInfo(const int16_t* def_levels, const int16_t* rep_levels,
43 int64_t num_def_levels, LevelInfo level_info,
44 ValidityBitmapInputOutput* output, OffsetType* offsets) {
45 OffsetType* orig_pos = offsets;
46 optional<::arrow::internal::FirstTimeBitmapWriter> valid_bits_writer;
47 if (output->valid_bits) {
48 valid_bits_writer.emplace(output->valid_bits, output->valid_bits_offset,
49 output->values_read_upper_bound);
50 }
51 for (int x = 0; x < num_def_levels; x++) {
52 // Skip items that belong to empty or null ancestor lists and further nested lists.
53 if (def_levels[x] < level_info.repeated_ancestor_def_level ||
54 rep_levels[x] > level_info.rep_level) {
55 continue;
56 }
57
58 if (rep_levels[x] == level_info.rep_level) {
59 // A continuation of an existing list.
60 // offsets can be null for structs with repeated children (we don't need to know
61 // offsets until we get to the children).
62 if (offsets != nullptr) {
63 if (ARROW_PREDICT_FALSE(*offsets == std::numeric_limits<OffsetType>::max())) {
64 throw ParquetException("List index overflow.");
65 }
66 *offsets += 1;
67 }
68 } else {
69 if (ARROW_PREDICT_FALSE(
70 (valid_bits_writer.has_value() &&
71 valid_bits_writer->position() >= output->values_read_upper_bound) ||
72 (offsets - orig_pos) >= output->values_read_upper_bound)) {
73 std::stringstream ss;
74 ss << "Definition levels exceeded upper bound: "
75 << output->values_read_upper_bound;
76 throw ParquetException(ss.str());
77 }
78
79 // current_rep < list rep_level i.e. start of a list (ancestor empty lists are
80 // filtered out above).
81 // offsets can be null for structs with repeated children (we don't need to know
82 // offsets until we get to the children).
83 if (offsets != nullptr) {
84 ++offsets;
85 // Use cumulative offsets because variable size lists are more common than
86 // fixed size lists so it should be cheaper to make these cumulative and
87 // subtract when validating fixed size lists.
88 *offsets = *(offsets - 1);
89 if (def_levels[x] >= level_info.def_level) {
90 if (ARROW_PREDICT_FALSE(*offsets == std::numeric_limits<OffsetType>::max())) {
91 throw ParquetException("List index overflow.");
92 }
93 *offsets += 1;
94 }
95 }
96
97 if (valid_bits_writer.has_value()) {
98 // the level_info def level for lists reflects element present level.
99 // the prior level distinguishes between empty lists.

Callers

nothing calls this directly

Calls 7

ParquetExceptionFunction · 0.85
strMethod · 0.80
positionMethod · 0.45
SetMethod · 0.45
ClearMethod · 0.45
NextMethod · 0.45
FinishMethod · 0.45

Tested by

no test coverage detected