Merges multiple ORC files that all have the same schema to produce a single ORC file. The merge will reject files that aren't compatible with the merged file so the output list may be shorter than the input list. The stripes are copied as serialized byte buffers. The user metadata are merged and fil
(Path outputPath,
WriterOptions options,
List<Path> inputFiles)
| 1294 | * @throws IOException |
| 1295 | */ |
| 1296 | public static List<Path> mergeFiles(Path outputPath, |
| 1297 | WriterOptions options, |
| 1298 | List<Path> inputFiles) throws IOException { |
| 1299 | Writer output = null; |
| 1300 | final Configuration conf = options.getConfiguration(); |
| 1301 | KeyProvider keyProvider = options.getKeyProvider(); |
| 1302 | try { |
| 1303 | byte[] buffer = new byte[0]; |
| 1304 | Reader firstFile = null; |
| 1305 | List<Path> result = new ArrayList<>(inputFiles.size()); |
| 1306 | Map<String, ByteBuffer> userMetadata = new HashMap<>(); |
| 1307 | int bufferSize = 0; |
| 1308 | |
| 1309 | for (Path input : inputFiles) { |
| 1310 | FileSystem fs = input.getFileSystem(conf); |
| 1311 | Reader reader = createReader(input, |
| 1312 | readerOptions(options.getConfiguration()) |
| 1313 | .filesystem(fs) |
| 1314 | .setKeyProvider(keyProvider)); |
| 1315 | |
| 1316 | if (!understandFormat(input, reader)) { |
| 1317 | continue; |
| 1318 | } else if (firstFile == null) { |
| 1319 | // if this is the first file that we are including, grab the values |
| 1320 | firstFile = reader; |
| 1321 | bufferSize = reader.getCompressionSize(); |
| 1322 | CompressionKind compression = reader.getCompressionKind(); |
| 1323 | options.bufferSize(bufferSize) |
| 1324 | .version(reader.getFileVersion()) |
| 1325 | .writerVersion(reader.getWriterVersion()) |
| 1326 | .compress(compression) |
| 1327 | .rowIndexStride(reader.getRowIndexStride()) |
| 1328 | .setSchema(reader.getSchema()); |
| 1329 | if (compression != CompressionKind.NONE) { |
| 1330 | options.enforceBufferSize().bufferSize(bufferSize); |
| 1331 | } |
| 1332 | mergeMetadata(userMetadata, reader); |
| 1333 | // ensure that the merged file uses the same key versions |
| 1334 | for(EncryptionKey key: reader.getColumnEncryptionKeys()) { |
| 1335 | options.setKeyVersion(key.getKeyName(), key.getKeyVersion(), |
| 1336 | key.getAlgorithm()); |
| 1337 | } |
| 1338 | output = createWriter(outputPath, options); |
| 1339 | } else if (!readerIsCompatible(firstFile, userMetadata, input, reader)) { |
| 1340 | continue; |
| 1341 | } else { |
| 1342 | mergeMetadata(userMetadata, reader); |
| 1343 | if (bufferSize < reader.getCompressionSize()) { |
| 1344 | bufferSize = reader.getCompressionSize(); |
| 1345 | ((WriterInternal) output).increaseCompressionSize(bufferSize); |
| 1346 | } |
| 1347 | } |
| 1348 | EncryptionVariant[] variants = reader.getEncryptionVariants(); |
| 1349 | List<StripeStatistics>[] completeList = new List[variants.length + 1]; |
| 1350 | for(int v=0; v < variants.length; ++v) { |
| 1351 | completeList[v] = reader.getVariantStripeStatistics(variants[v]); |
| 1352 | } |
| 1353 | completeList[completeList.length - 1] = reader.getVariantStripeStatistics(null); |