Class ConcatTool

lang/java/tools/src/main/java/org/apache/avro/tool/ConcatTool.java:44–171

Tool to concatenate avro files with the same schema and non-reserved metatdata.

 * metatdata.
 */
public class ConcatTool implements Tool {
  /**
   * @return 0 for success, 1 if the schemas of the input files differ, 2 if the
   *         non-reserved input metadata differs, 3 if the input files are encoded
   *         with more than one codec.
   */
  @Override
  public int run(InputStream in, PrintStream out, PrintStream err, List<String> args) throws Exception {

    if (args.isEmpty()) {
      printHelp(out);
      return 0;
    }

    OutputStream output = out;
    if (args.size() > 1) {
      output = Util.fileOrStdout(args.get(args.size() - 1), out);
      args = args.subList(0, args.size() - 1);
    }

    DataFileWriter<GenericRecord> writer = new DataFileWriter<>(new GenericDatumWriter<>());
    Schema schema = null;
    Map<String, byte[]> metadata = new TreeMap<>();
    String inputCodec = null;

    for (String inFile : expandsInputFiles(args)) {
      InputStream input = Util.fileOrStdin(inFile, in);
      DataFileStream<GenericRecord> reader = new DataFileStream<>(input, new GenericDatumReader<>());

      if (schema == null) {
        // this is the first file - set up the writer, and store the
        // Schema & metadata we'll use.
        schema = reader.getSchema();
        for (String key : reader.getMetaKeys()) {
          if (!DataFileWriter.isReservedMeta(key)) {
            byte[] metadatum = reader.getMeta(key);
            metadata.put(key, metadatum);
            writer.setMeta(key, metadatum);
          }
        }
        inputCodec = reader.getMetaString(DataFileConstants.CODEC);
        if (inputCodec == null) {
          inputCodec = DataFileConstants.NULL_CODEC;
        }
        writer.setCodec(CodecFactory.fromString(inputCodec));
        writer.create(schema, output);
      } else {
        // check that we're appending to the same schema & metadata.
        if (!schema.equals(reader.getSchema())) {
          err.println("input files have different schemas");
          reader.close();
          return 1;
        }
        for (String key : reader.getMetaKeys()) {
          if (!DataFileWriter.isReservedMeta(key)) {
            byte[] metadatum = reader.getMeta(key);
            byte[] writersMetadatum = metadata.get(key);
            if (!Arrays.equals(metadatum, writersMetadatum)) {