Method GenerateMultiStreamData

be/src/util/decompress-test.cc:340–375

Try to simulate pbzip2 behavior. pbzip2 splits large input into smaller chunks and compresses them separately, then concatenate the compressed streams together. We generate ~16MB compressed data to make sure it's bigger than the decompressor's output buffer size(STREAM_OUT_BUF_SIZE). With the generated raw input data, we expect ~2:1 compression ratio so we need 4xSTREAM_OUT_BUF_SIZE input data in

  // output buffer size(STREAM_OUT_BUF_SIZE). With the generated raw input data, we
  // expect ~2:1 compression ratio so we need 4xSTREAM_OUT_BUF_SIZE input data in total.
  void GenerateMultiStreamData(THdfsCompression::type format, int64_t* uncompressed_len,
      uint8_t** uncompressed_data, int64_t* compressed_len, uint8_t** compressed_data) {
    uint8_t raw_input[RAW_INPUT_SIZE + 1];
    for (int i = 0; i < RAW_INPUT_SIZE; ++i) {
      raw_input[i] = 'a' + rand() % 26;
    }
    raw_input[RAW_INPUT_SIZE] = 0;

    // Repeatedly pick random-size input data(~1MB), compress it, then concatenate
    // those small compressed streams into one big buffer. Also save random input
    // into a single buffer to verify decompressor output.
    *compressed_data = mem_pool_.Allocate(COMPRESSED_BUFFER_SIZE);
    *uncompressed_data = mem_pool_.Allocate(UNCOMPRESSED_BUFFER_SIZE);
    *uncompressed_len = 0;
    *compressed_len = 0;

    scoped_ptr<Codec> compressor;
    Codec::CodecInfo codec_info(format);
    EXPECT_OK(Codec::CreateCompressor(&mem_pool_, true, codec_info, &compressor));

    // Make sure we don't completely fill the buffer, leave at least RAW_INPUT_SIZE
    // bytes free in compressed buffer for junk data testing (Test case 3).
    while (*compressed_len < (COMPRESSED_BUFFER_SIZE - RAW_INPUT_SIZE)
        && *uncompressed_len < (UNCOMPRESSED_BUFFER_SIZE - RAW_INPUT_SIZE)) {
      int len = RAW_INPUT_SIZE - (rand() % 1024);
      uint8_t* compressed_stream = NULL;
      int64_t compressed_length = 0;
      EXPECT_OK(compressor->ProcessBlock(false, len, raw_input, &compressed_length,
          &compressed_stream));
      memcpy(*compressed_data + *compressed_len, compressed_stream, compressed_length);
      memcpy(*uncompressed_data + *uncompressed_len, raw_input, len);
      *uncompressed_len += len;
      *compressed_len += compressed_length;
    }
    compressor->Close();
  }

  // Buffer to hold generated random data. Size doesn't matter, use 1MB for easy
  // calculation.