MCPcopy Create free account
hub / github.com/apache/arrow / MakePages

Function MakePages

cpp/src/parquet/test_util.h:535–596  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

533// Generates pages from randomly generated data
534template <typename Type>
535inline int MakePages(const ColumnDescriptor* d, int num_pages, int levels_per_page,
536 std::vector<int16_t>& def_levels, std::vector<int16_t>& rep_levels,
537 std::vector<typename Type::c_type>& values,
538 std::vector<uint8_t>& buffer,
539 std::vector<std::shared_ptr<Page>>& pages,
540 Encoding::type encoding = Encoding::PLAIN, uint32_t seed = 0) {
541 int num_levels = levels_per_page * num_pages;
542 int num_values = 0;
543 int16_t zero = 0;
544 int16_t max_def_level = d->max_definition_level();
545 int16_t max_rep_level = d->max_repetition_level();
546 std::vector<int> values_per_page(num_pages, levels_per_page);
547 // Create definition levels
548 if (max_def_level > 0 && num_levels != 0) {
549 def_levels.resize(num_levels);
550 random_numbers(num_levels, seed, zero, max_def_level, def_levels.data());
551 for (int p = 0; p < num_pages; p++) {
552 int num_values_per_page = 0;
553 for (int i = 0; i < levels_per_page; i++) {
554 if (def_levels[i + p * levels_per_page] == max_def_level) {
555 num_values_per_page++;
556 num_values++;
557 }
558 }
559 values_per_page[p] = num_values_per_page;
560 }
561 } else {
562 num_values = num_levels;
563 }
564 // Create repetition levels
565 if (max_rep_level > 0 && num_levels != 0) {
566 rep_levels.resize(num_levels);
567 // Using a different seed so that def_levels and rep_levels are different.
568 random_numbers(num_levels, seed + 789, zero, max_rep_level, rep_levels.data());
569 // The generated levels are random. Force the very first page to start with a new
570 // record.
571 rep_levels[0] = 0;
572 // For a null value, rep_levels and def_levels are both 0.
573 // If we have a repeated value right after this, it needs to start with
574 // rep_level = 0 to indicate a new record.
575 for (int i = 0; i < num_levels - 1; ++i) {
576 if (rep_levels[i] == 0 && def_levels[i] == 0) {
577 rep_levels[i + 1] = 0;
578 }
579 }
580 }
581 // Create values
582 values.resize(num_values);
583 if (encoding == Encoding::PLAIN) {
584 InitValues<typename Type::c_type>(num_values, values, buffer);
585 PaginatePlain<Type>(d, values, def_levels, max_def_level, rep_levels, max_rep_level,
586 levels_per_page, values_per_page, pages);
587 } else if (encoding == Encoding::RLE_DICTIONARY ||
588 encoding == Encoding::PLAIN_DICTIONARY) {
589 // Calls InitValues and repeats the data
590 InitDictValues<typename Type::c_type>(num_values, levels_per_page, values, buffer);
591 PaginateDict<Type>(d, values, def_levels, max_def_level, rep_levels, max_rep_level,
592 levels_per_page, values_per_page, pages);

Callers

nothing calls this directly

Calls 3

random_numbersFunction · 0.85
resizeMethod · 0.80
dataMethod · 0.45

Tested by

no test coverage detected