(Doc section: Reading and writing partitioned data #2) Read an entire dataset, but with partitioning information.
| 276 | // (Doc section: Reading and writing partitioned data #2) |
| 277 | // Read an entire dataset, but with partitioning information. |
| 278 | arrow::Result<std::shared_ptr<arrow::Table>> ScanPartitionedDataset( |
| 279 | const std::shared_ptr<fs::FileSystem>& filesystem, |
| 280 | const std::shared_ptr<ds::FileFormat>& format, const std::string& base_dir) { |
| 281 | fs::FileSelector selector; |
| 282 | selector.base_dir = base_dir; |
| 283 | selector.recursive = true; // Make sure to search subdirectories |
| 284 | ds::FileSystemFactoryOptions options; |
| 285 | // We'll use Hive-style partitioning. We'll let Arrow Datasets infer the partition |
| 286 | // schema. |
| 287 | options.partitioning = ds::HivePartitioning::MakeFactory(); |
| 288 | ARROW_ASSIGN_OR_RAISE(auto factory, ds::FileSystemDatasetFactory::Make( |
| 289 | filesystem, selector, format, options)); |
| 290 | ARROW_ASSIGN_OR_RAISE(auto dataset, factory->Finish()); |
| 291 | // Print out the fragments |
| 292 | ARROW_ASSIGN_OR_RAISE(auto fragments, dataset->GetFragments()); |
| 293 | for (const auto& fragment : fragments) { |
| 294 | std::cout << "Found fragment: " << (*fragment)->ToString() << std::endl; |
| 295 | std::cout << "Partition expression: " |
| 296 | << (*fragment)->partition_expression().ToString() << std::endl; |
| 297 | } |
| 298 | ARROW_ASSIGN_OR_RAISE(auto scan_builder, dataset->NewScan()); |
| 299 | ARROW_ASSIGN_OR_RAISE(auto scanner, scan_builder->Finish()); |
| 300 | return scanner->ToTable(); |
| 301 | } |
| 302 | // (Doc section: Reading and writing partitioned data #2) |
| 303 | |
| 304 | // (Doc section: Reading and writing partitioned data #3) |
no test coverage detected