Method TestFullRoundtrip

cpp/src/parquet/statistics_test.cc:422–481

  }

  void TestFullRoundtrip(int64_t num_values, int64_t null_count) {
    this->GenerateData(num_values);

    // compute statistics for the whole batch
    auto expected_stats = MakeStatistics<TestType>(this->schema_.Column(0));
    expected_stats->Update(this->values_ptr_, num_values - null_count, null_count);

    auto sink = CreateOutputStream();
    auto gnode = std::static_pointer_cast<GroupNode>(this->node_);
    std::shared_ptr<WriterProperties> writer_properties =
        WriterProperties::Builder().enable_statistics("column")->build();
    auto file_writer = ParquetFileWriter::Open(sink, gnode, writer_properties);
    auto row_group_writer = file_writer->AppendRowGroup();
    auto column_writer =
        static_cast<TypedColumnWriter<TestType>*>(row_group_writer->NextColumn());

    // simulate the case when data comes from multiple buffers,
    // in which case special care is necessary for FLBA/ByteArray types
    for (int i = 0; i < 2; i++) {
      int64_t batch_num_values = i ? num_values - num_values / 2 : num_values / 2;
      int64_t batch_null_count = i ? null_count : 0;
      DCHECK(null_count <= num_values);  // avoid too much headache
      std::vector<int16_t> definition_levels(batch_null_count, 0);
      definition_levels.insert(definition_levels.end(),
                               batch_num_values - batch_null_count, 1);
      auto beg = this->values_.begin() + i * num_values / 2;
      auto end = beg + batch_num_values;
      std::vector<c_type> batch = GetDeepCopy(std::vector<c_type>(beg, end));
      c_type* batch_values_ptr = GetValuesPointer(batch);
      column_writer->WriteBatch(batch_num_values, definition_levels.data(), nullptr,
                                batch_values_ptr);
      DeepFree(batch);
    }
    column_writer->Close();
    row_group_writer->Close();
    file_writer->Close();

    ASSERT_OK_AND_ASSIGN(auto buffer, sink->Finish());
    auto source = std::make_shared<::arrow::io::BufferReader>(buffer);
    auto file_reader = ParquetFileReader::Open(source);
    auto rg_reader = file_reader->RowGroup(0);
    auto column_chunk = rg_reader->metadata()->ColumnChunk(0);
    if (!column_chunk->is_stats_set()) return;
    std::shared_ptr<Statistics> stats = column_chunk->statistics();
    // check values after serialization + deserialization
    EXPECT_EQ(null_count, stats->null_count());
    EXPECT_EQ(num_values - null_count, stats->num_values());
    EXPECT_TRUE(expected_stats->HasMinMax());
    EXPECT_EQ(expected_stats->EncodeMin(), stats->EncodeMin());
    EXPECT_EQ(expected_stats->EncodeMax(), stats->EncodeMax());

    std::shared_ptr<EncodedStatistics> enc_stats = column_chunk->encoded_statistics();
    EXPECT_EQ(null_count, enc_stats->null_count);
    EXPECT_TRUE(enc_stats->has_min);
    EXPECT_TRUE(enc_stats->has_max);
    EXPECT_EQ(expected_stats->EncodeMin(), enc_stats->min());
    EXPECT_EQ(expected_stats->EncodeMax(), enc_stats->max());
    EXPECT_EQ(enc_stats->is_min_value_exact, std::make_optional(true));