| 521 | } // ConvertToRecordBatch |
| 522 | |
| 523 | arrow::Status DoRowConversion(int32_t num_rows, int32_t batch_size) { |
| 524 | //(Doc section: Convert to Arrow) |
| 525 | // Write JSON records |
| 526 | std::vector<std::string> json_records = { |
| 527 | R"({"pk": 1, "date_created": "2020-10-01", "data": {"deleted": true, "metrics": [{"key": "x", "value": 1}]}})", |
| 528 | R"({"pk": 2, "date_created": "2020-10-03", "data": {"deleted": false, "metrics": []}})", |
| 529 | R"({"pk": 3, "date_created": "2020-10-05", "data": {"deleted": false, "metrics": [{"key": "x", "value": 33}, {"key": "x", "value": 42}]}})"}; |
| 530 | |
| 531 | std::vector<rapidjson::Document> records; |
| 532 | records.reserve(num_rows); |
| 533 | for (int32_t i = 0; i < num_rows; ++i) { |
| 534 | rapidjson::Document document; |
| 535 | document.Parse(json_records[i % json_records.size()]); |
| 536 | records.push_back(std::move(document)); |
| 537 | } |
| 538 | |
| 539 | for (const rapidjson::Document& doc : records) { |
| 540 | rapidjson::StringBuffer sb; |
| 541 | rapidjson::Writer<rapidjson::StringBuffer> writer(sb); |
| 542 | doc.Accept(writer); |
| 543 | std::cout << sb.GetString() << std::endl; |
| 544 | } |
| 545 | auto tags_schema = arrow::list(arrow::struct_({ |
| 546 | arrow::field("key", arrow::utf8()), |
| 547 | arrow::field("value", arrow::int64()), |
| 548 | })); |
| 549 | auto schema = arrow::schema( |
| 550 | {arrow::field("pk", arrow::int64()), arrow::field("date_created", arrow::utf8()), |
| 551 | arrow::field("data", arrow::struct_({arrow::field("deleted", arrow::boolean()), |
| 552 | arrow::field("metrics", tags_schema)}))}); |
| 553 | |
| 554 | // Convert records into a table |
| 555 | ARROW_ASSIGN_OR_RAISE(std::shared_ptr<arrow::RecordBatch> batch, |
| 556 | ConvertToRecordBatch(records, schema)); |
| 557 | |
| 558 | ARROW_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Table> table, |
| 559 | arrow::Table::FromRecordBatches({batch})); |
| 560 | |
| 561 | // Print table |
| 562 | std::cout << table->ToString() << std::endl; |
| 563 | ARROW_RETURN_NOT_OK(table->ValidateFull()); |
| 564 | //(Doc section: Convert to Arrow) |
| 565 | |
| 566 | //(Doc section: Convert to Rows) |
| 567 | // Create converter |
| 568 | ArrowToDocumentConverter to_doc_converter; |
| 569 | |
| 570 | // Convert table into document (row) iterator |
| 571 | arrow::Iterator<rapidjson::Document> document_iter = |
| 572 | to_doc_converter.ConvertToIterator(table, batch_size); |
| 573 | |
| 574 | // Print each row |
| 575 | for (arrow::Result<rapidjson::Document> doc_result : document_iter) { |
| 576 | ARROW_ASSIGN_OR_RAISE(rapidjson::Document doc, std::move(doc_result)); |
| 577 | |
| 578 | assert(doc.HasMember("pk")); |
| 579 | assert(doc["pk"].IsInt64()); |
| 580 | assert(doc.HasMember("date_created")); |
no test coverage detected