Verifies the file layout and data produced by a table writer.
| 990 | |
| 991 | // Verifies the file layout and data produced by a table writer. |
| 992 | void verifyTableWriterOutput( |
| 993 | const std::string& targetDir, |
| 994 | const RowTypePtr& bucketCheckFileType, |
| 995 | bool verifyPartitionedData = true, |
| 996 | bool verifyBucketedData = true) { |
| 997 | SCOPED_TRACE(testParam_.toString()); |
| 998 | std::vector<std::filesystem::path> filePaths; |
| 999 | std::vector<std::filesystem::path> dirPaths; |
| 1000 | for (auto& path : fs::recursive_directory_iterator(targetDir)) { |
| 1001 | if (path.is_regular_file()) { |
| 1002 | filePaths.push_back(path.path()); |
| 1003 | } else { |
| 1004 | dirPaths.push_back(path.path()); |
| 1005 | } |
| 1006 | } |
| 1007 | if (testMode_ == TestMode::kUnpartitioned) { |
| 1008 | ASSERT_EQ(dirPaths.size(), 0); |
| 1009 | ASSERT_LE(filePaths.size(), numTableWriterCount_); |
| 1010 | verifyUnbucketedFilePath(filePaths[0], targetDir); |
| 1011 | return; |
| 1012 | } else if (testMode_ == TestMode::kOnlyBucketed) { |
| 1013 | ASSERT_EQ(dirPaths.size(), 0); |
| 1014 | for (const auto& filePath : filePaths) { |
| 1015 | ASSERT_EQ(filePath.parent_path().string(), targetDir); |
| 1016 | verifyBucketedFileName(filePath); |
| 1017 | if (verifyBucketedData) { |
| 1018 | verifyBucketedFileData(filePath, bucketCheckFileType); |
| 1019 | } |
| 1020 | } |
| 1021 | return; |
| 1022 | } |
| 1023 | // Validation for both partitioned with and without buckets. |
| 1024 | ASSERT_EQ(numPartitionKeyValues_.size(), 2); |
| 1025 | const auto totalPartitions = |
| 1026 | numPartitionKeyValues_[0] * numPartitionKeyValues_[1]; |
| 1027 | ASSERT_LE(dirPaths.size(), totalPartitions + numPartitionKeyValues_[0]); |
| 1028 | int32_t numLeafDir{0}; |
| 1029 | for (const auto& dirPath : dirPaths) { |
| 1030 | verifyPartitionedDirPath(dirPath, targetDir); |
| 1031 | if (dirPath.parent_path().string() != targetDir) { |
| 1032 | ++numLeafDir; |
| 1033 | } |
| 1034 | } |
| 1035 | if (testMode_ == TestMode::kPartitioned) { |
| 1036 | // We expect only one file under each directory without dynamic writer |
| 1037 | // support. |
| 1038 | ASSERT_GE(numLeafDir * numTableWriterCount_, filePaths.size()); |
| 1039 | for (const auto& filePath : filePaths) { |
| 1040 | verifyPartitionedFilePath(filePath, targetDir); |
| 1041 | if (verifyPartitionedData) { |
| 1042 | verifyPartitionedFilesData({filePath}, filePath.parent_path()); |
| 1043 | } |
| 1044 | } |
| 1045 | return; |
| 1046 | } |
| 1047 | ASSERT_GE(numLeafDir * bucketProperty_->bucketCount(), filePaths.size()); |
| 1048 | std::unordered_map<std::string, std::vector<std::filesystem::path>> |
| 1049 | bucketFilesPerPartition; |