| 44 | |
| 45 | template <typename OffsetType> |
| 46 | void DefRepLevelsToListInfo(const int16_t* def_levels, const int16_t* rep_levels, |
| 47 | int64_t num_def_levels, LevelInfo level_info, |
| 48 | ValidityBitmapInputOutput* output, OffsetType* offsets) { |
| 49 | OffsetType* orig_pos = offsets; |
| 50 | optional<::arrow::internal::FirstTimeBitmapWriter> valid_bits_writer; |
| 51 | if (output->valid_bits) { |
| 52 | valid_bits_writer.emplace(output->valid_bits, output->valid_bits_offset, |
| 53 | output->values_read_upper_bound); |
| 54 | } |
| 55 | for (int x = 0; x < num_def_levels; x++) { |
| 56 | // Skip items that belong to empty or null ancestor lists and further nested lists. |
| 57 | if (def_levels[x] < level_info.repeated_ancestor_def_level || |
| 58 | rep_levels[x] > level_info.rep_level) { |
| 59 | continue; |
| 60 | } |
| 61 | |
| 62 | if (rep_levels[x] == level_info.rep_level) { |
| 63 | // A continuation of an existing list. |
| 64 | // offsets can be null for structs with repeated children (we don't need to know |
| 65 | // offsets until we get to the children). |
| 66 | if (offsets != nullptr) { |
| 67 | if (ARROW_PREDICT_FALSE(*offsets == std::numeric_limits<OffsetType>::max())) { |
| 68 | throw ParquetException("List index overflow."); |
| 69 | } |
| 70 | *offsets += 1; |
| 71 | } |
| 72 | } else { |
| 73 | if (ARROW_PREDICT_FALSE( |
| 74 | (valid_bits_writer.has_value() && |
| 75 | valid_bits_writer->position() >= output->values_read_upper_bound) || |
| 76 | (offsets - orig_pos) >= output->values_read_upper_bound)) { |
| 77 | std::stringstream ss; |
| 78 | ss << "Definition levels exceeded upper bound: " |
| 79 | << output->values_read_upper_bound; |
| 80 | throw ParquetException(ss.str()); |
| 81 | } |
| 82 | |
| 83 | // current_rep < list rep_level i.e. start of a list (ancestor empty lists are |
| 84 | // filtered out above). |
| 85 | // offsets can be null for structs with repeated children (we don't need to know |
| 86 | // offsets until we get to the children). |
| 87 | if (offsets != nullptr) { |
| 88 | ++offsets; |
| 89 | // Use cumulative offsets because variable size lists are more common than |
| 90 | // fixed size lists so it should be cheaper to make these cumulative and |
| 91 | // subtract when validating fixed size lists. |
| 92 | *offsets = *(offsets - 1); |
| 93 | if (def_levels[x] >= level_info.def_level) { |
| 94 | if (ARROW_PREDICT_FALSE(*offsets == std::numeric_limits<OffsetType>::max())) { |
| 95 | throw ParquetException("List index overflow."); |
| 96 | } |
| 97 | *offsets += 1; |
| 98 | } |
| 99 | } |
| 100 | |
| 101 | if (valid_bits_writer.has_value()) { |
| 102 | // the level_info def level for lists reflects element present level. |
| 103 | // the prior level distinguishes between empty lists. |