| 76 | } |
| 77 | |
| 78 | arrow::Result<std::shared_ptr<ds::Dataset>> GetDatasetFromDirectory( |
| 79 | std::shared_ptr<fs::FileSystem> fs, std::shared_ptr<ds::ParquetFileFormat> format, |
| 80 | std::string dir) { |
| 81 | // Find all files under `path` |
| 82 | fs::FileSelector s; |
| 83 | s.base_dir = dir; |
| 84 | s.recursive = true; |
| 85 | |
| 86 | ds::FileSystemFactoryOptions options; |
| 87 | // The factory will try to build a child dataset. |
| 88 | ARROW_ASSIGN_OR_RAISE(auto factory, |
| 89 | ds::FileSystemDatasetFactory::Make(fs, s, format, options)); |
| 90 | |
| 91 | // Try to infer a common schema for all files. |
| 92 | ARROW_ASSIGN_OR_RAISE(auto schema, factory->Inspect(conf.inspect_options)); |
| 93 | // Caller can optionally decide another schema as long as it is compatible |
| 94 | // with the previous one, e.g. `factory->Finish(compatible_schema)`. |
| 95 | ARROW_ASSIGN_OR_RAISE(auto child, factory->Finish(conf.finish_options)); |
| 96 | |
| 97 | ds::DatasetVector children{conf.repeat, child}; |
| 98 | auto dataset = ds::UnionDataset::Make(std::move(schema), std::move(children)); |
| 99 | |
| 100 | return dataset; |
| 101 | } |
| 102 | |
| 103 | arrow::Result<std::shared_ptr<ds::Dataset>> GetParquetDatasetFromMetadata( |
| 104 | std::shared_ptr<fs::FileSystem> fs, std::shared_ptr<ds::ParquetFileFormat> format, |
no test coverage detected