Function testStringDictionary

c++/test/TestDictionaryEncoding.cc:82–142

  }

  void testStringDictionary(bool enableIndex, double threshold, bool enableEncodedBlock = false) {
    MemoryOutputStream memStream(DEFAULT_MEM_STREAM_SIZE);
    MemoryPool* pool = getDefaultPool();
    std::unique_ptr<Type> type(Type::buildTypeFromString("struct<col1:string>"));

    WriterOptions options;
    options.setStripeSize(1024);
    options.setMemoryBlockSize(64);
    options.setCompressionBlockSize(1024);
    options.setCompression(CompressionKind_ZLIB);
    options.setMemoryPool(pool);
    options.setDictionaryKeySizeThreshold(threshold);
    options.setRowIndexStride(enableIndex ? 10000 : 0);
    std::unique_ptr<Writer> writer = createWriter(*type, &memStream, options);

    char dataBuffer[327675];
    uint64_t offset = 0, rowCount = 65535;
    std::unique_ptr<ColumnVectorBatch> batch = writer->createRowBatch(rowCount);
    StructVectorBatch* structBatch = dynamic_cast<StructVectorBatch*>(batch.get());
    StringVectorBatch* strBatch = dynamic_cast<StringVectorBatch*>(structBatch->fields[0]);

    uint64_t dictionarySize = DICTIONARY_SIZE_1K;
    for (uint64_t i = 0; i < rowCount; ++i) {
      std::ostringstream os;
      os << (i % dictionarySize);
      memcpy(dataBuffer + offset, os.str().c_str(), os.str().size());

      strBatch->data[i] = dataBuffer + offset;
      strBatch->length[i] = static_cast<int64_t>(os.str().size());

      offset += os.str().size();
    }
    structBatch->numElements = rowCount;
    strBatch->numElements = rowCount;
    writer->add(*batch);
    writer->close();

    std::unique_ptr<InputStream> inStream(
        new MemoryInputStream(memStream.getData(), memStream.getLength()));
    std::unique_ptr<Reader> reader = createReader(pool, std::move(inStream));
    std::unique_ptr<RowReader> rowReader = createRowReader(reader.get(), enableEncodedBlock);
    EXPECT_EQ(rowCount, reader->getNumberOfRows());

    batch = rowReader->createRowBatch(rowCount);
    EXPECT_EQ(true, rowReader->next(*batch));
    EXPECT_EQ(rowCount, batch->numElements);

    structBatch = dynamic_cast<StructVectorBatch*>(batch.get());
    strBatch = dynamic_cast<StringVectorBatch*>(structBatch->fields[0]);
    if (doubleEquals(threshold, DICT_THRESHOLD) && enableEncodedBlock) {
      checkDictionaryEncoding(strBatch);
      strBatch->decodeDictionary();
    }

    for (uint64_t i = 0; i < rowCount; ++i) {
      std::string str(strBatch->data[i], static_cast<size_t>(strBatch->length[i]));
      EXPECT_EQ(i % dictionarySize, static_cast<uint64_t>(atoi(str.c_str())));
    }