Function testDictionaryMultipleStripes

c++/test/TestDictionaryEncoding.cc:353–433

  }

  void testDictionaryMultipleStripes(double threshold, bool enableIndex) {
    MemoryOutputStream memStream(DEFAULT_MEM_STREAM_SIZE);
    MemoryPool* pool = getDefaultPool();
    std::unique_ptr<Type> type(Type::buildTypeFromString("struct<col1:string>"));

    WriterOptions options;
    options.setStripeSize(1);
    options.setMemoryBlockSize(1024);
    options.setCompressionBlockSize(2 * 1024);
    options.setCompression(CompressionKind_ZLIB);
    options.setMemoryPool(pool);
    options.setDictionaryKeySizeThreshold(threshold);
    options.setRowIndexStride(enableIndex ? 10000 : 0);
    std::unique_ptr<Writer> writer = createWriter(*type, &memStream, options);

    char dataBuffer[800000];
    uint64_t offset = 0, rowCount = 65535, stripeCount = 3;
    std::unique_ptr<ColumnVectorBatch> batch = writer->createRowBatch(rowCount);
    StructVectorBatch* structBatch = dynamic_cast<StructVectorBatch*>(batch.get());
    StringVectorBatch* strBatch = dynamic_cast<StringVectorBatch*>(structBatch->fields[0]);

    uint64_t dictionarySize = DICTIONARY_SIZE_1K;

    for (uint64_t stripe = 0; stripe != stripeCount; ++stripe) {
      for (uint64_t i = 0; i < rowCount; ++i) {
        std::ostringstream os;
        os << (i % dictionarySize);
        memcpy(dataBuffer + offset, os.str().c_str(), os.str().size());

        strBatch->data[i] = dataBuffer + offset;
        strBatch->length[i] = static_cast<int64_t>(os.str().size());

        offset += os.str().size();
      }

      structBatch->numElements = rowCount;
      strBatch->numElements = rowCount;

      writer->add(*batch);
    }

    writer->close();

    std::unique_ptr<InputStream> inStream(
        new MemoryInputStream(memStream.getData(), memStream.getLength()));
    std::unique_ptr<Reader> reader = createReader(pool, std::move(inStream));
    std::unique_ptr<RowReader> rowReader = createRowReader(reader.get());

    // make sure there are multiple stripes
    EXPECT_EQ(rowCount * stripeCount, reader->getNumberOfRows());
    EXPECT_EQ(stripeCount, reader->getNumberOfStripes());

    // test reading sequentially for data correctness
    batch = rowReader->createRowBatch(rowCount);
    for (uint64_t stripe = 0; stripe != stripeCount; ++stripe) {
      EXPECT_EQ(true, rowReader->next(*batch));

      structBatch = dynamic_cast<StructVectorBatch*>(batch.get());