| 498 | } |
| 499 | |
| 500 | Status EnsureDestinationValid(const FileSystemDatasetWriteOptions& options) { |
| 501 | if (options.existing_data_behavior == ExistingDataBehavior::kError) { |
| 502 | fs::FileSelector selector; |
| 503 | selector.base_dir = options.base_dir; |
| 504 | selector.recursive = true; |
| 505 | Result<std::vector<fs::FileInfo>> maybe_files = |
| 506 | options.filesystem->GetFileInfo(selector); |
| 507 | if (!maybe_files.ok()) { |
| 508 | // If the path doesn't exist then continue |
| 509 | return Status::OK(); |
| 510 | } |
| 511 | if (maybe_files->size() > 0) { |
| 512 | return Status::Invalid( |
| 513 | "Could not write to ", options.base_dir, |
| 514 | " as the directory is not empty and existing_data_behavior is to error"); |
| 515 | } |
| 516 | } |
| 517 | return Status::OK(); |
| 518 | } |
| 519 | |
| 520 | // Rule of thumb for the max rows to stage. It will grow with max_rows_queued until |
| 521 | // max_rows_queued starts to get too large and then it caps out at 8 million rows. |