Function WriteInBatches

cpp/examples/arrow/parquet_read_write.cc:127–166

}

arrow::Status WriteInBatches(std::string path_to_file) {
  // #include "parquet/arrow/writer.h"
  // #include "arrow/util/type_fwd.h"
  using parquet::ArrowWriterProperties;
  using parquet::WriterProperties;

  // Data is in RBR
  std::shared_ptr<arrow::RecordBatchReader> batch_stream;
  ARROW_ASSIGN_OR_RAISE(batch_stream, GetRBR());

  // Choose compression
  std::shared_ptr<WriterProperties> props =
      WriterProperties::Builder().compression(arrow::Compression::SNAPPY)->build();

  // Opt to store Arrow schema for easier reads back into Arrow
  std::shared_ptr<ArrowWriterProperties> arrow_props =
      ArrowWriterProperties::Builder().store_schema()->build();

  // Create a writer
  std::shared_ptr<arrow::io::FileOutputStream> outfile;
  ARROW_ASSIGN_OR_RAISE(outfile, arrow::io::FileOutputStream::Open(path_to_file));
  std::unique_ptr<parquet::arrow::FileWriter> writer;
  ARROW_ASSIGN_OR_RAISE(
      writer, parquet::arrow::FileWriter::Open(*batch_stream->schema().get(),
                                               arrow::default_memory_pool(), outfile,
                                               props, arrow_props));

  // Write each batch as a row_group
  for (arrow::Result<std::shared_ptr<arrow::RecordBatch>> maybe_batch : *batch_stream) {
    ARROW_ASSIGN_OR_RAISE(auto batch, maybe_batch);
    ARROW_ASSIGN_OR_RAISE(auto table,
                          arrow::Table::FromRecordBatches(batch->schema(), {batch}));
    ARROW_RETURN_NOT_OK(writer->WriteTable(*table.get(), batch->num_rows()));
  }

  // Write file footer and close
  ARROW_RETURN_NOT_OK(writer->Close());

  return arrow::Status::OK();
}

arrow::Status RunExamples(const std::string& path_to_file) {
  ARROW_RETURN_NOT_OK(WriteFullFile(path_to_file));