Verifies the bucketed file data by checking if the bucket id of each read row is the same as the one encoded in the corresponding bucketed file name.
| 934 | // Verifies the bucketed file data by checking if the bucket id of each read |
| 935 | // row is the same as the one encoded in the corresponding bucketed file name. |
| 936 | void verifyBucketedFileData( |
| 937 | const std::filesystem::path& filePath, |
| 938 | const RowTypePtr& outputFileType) { |
| 939 | const std::vector<std::filesystem::path> filePaths = {filePath}; |
| 940 | |
| 941 | // Read data from bucketed file on disk into 'rowVector'. |
| 942 | core::PlanNodeId scanNodeId; |
| 943 | auto plan = PlanBuilder() |
| 944 | .tableScan(outputFileType, {}, "", outputFileType) |
| 945 | .capturePlanNodeId(scanNodeId) |
| 946 | .planNode(); |
| 947 | const auto resultVector = |
| 948 | AssertQueryBuilder(plan) |
| 949 | .splits(scanNodeId, makeHiveConnectorSplits(filePaths)) |
| 950 | .copyResults(pool_.get()); |
| 951 | |
| 952 | // Parse the bucket id encoded in bucketed file name. |
| 953 | const uint32_t expectedBucketId = |
| 954 | parseBucketId(filePath.filename().string()); |
| 955 | |
| 956 | // Compute the bucket id from read result by applying hash partition on |
| 957 | // bucketed columns in read result, and we expect they all match the one |
| 958 | // encoded in file name. |
| 959 | auto bucketFunction = getBucketFunction(outputFileType); |
| 960 | std::vector<uint32_t> bucketIds; |
| 961 | bucketIds.reserve(resultVector->size()); |
| 962 | bucketFunction->partition(*resultVector, bucketIds); |
| 963 | for (const auto bucketId : bucketIds) { |
| 964 | ASSERT_EQ(expectedBucketId, bucketId); |
| 965 | } |
| 966 | |
| 967 | if (!testParam_.bucketSort()) { |
| 968 | return; |
| 969 | } |
| 970 | // Verifies the sorting behavior |
| 971 | for (int i = 0; i < resultVector->size() - 1; ++i) { |
| 972 | for (int j = 0; j < sortColumnIndices_.size(); ++j) { |
| 973 | auto compareResult = |
| 974 | resultVector->childAt(sortColumnIndices_.at(j)) |
| 975 | ->compare( |
| 976 | resultVector->childAt(sortColumnIndices_.at(j)) |
| 977 | ->wrappedVector(), |
| 978 | i, |
| 979 | i + 1, |
| 980 | sortedFlags_[j]); |
| 981 | if (compareResult.has_value()) { |
| 982 | if (compareResult.value() < 0) { |
| 983 | break; |
| 984 | } |
| 985 | ASSERT_EQ(compareResult.value(), 0); |
| 986 | } |
| 987 | } |
| 988 | } |
| 989 | } |
| 990 | |
| 991 | // Verifies the file layout and data produced by a table writer. |
| 992 | void verifyTableWriterOutput( |
nothing calls this directly
no test coverage detected