Function loadStringDictionary

c++/src/DictionaryLoader.cc:45–98

  }  // namespace

  std::shared_ptr<StringDictionary> loadStringDictionary(uint64_t columnId, StripeStreams& stripe,
                                                         MemoryPool& pool) {
    // Get encoding information
    proto::ColumnEncoding encoding = stripe.getEncoding(columnId);
    RleVersion rleVersion = convertRleVersion(encoding.kind());
    uint32_t dictSize = encoding.dictionary_size();

    // Create the dictionary object
    auto dictionary = std::make_shared<StringDictionary>(pool);

    // Read LENGTH stream to get dictionary entry lengths
    std::unique_ptr<SeekableInputStream> stream =
        stripe.getStream(columnId, proto::Stream_Kind_LENGTH, false);
    if (dictSize > 0 && stream == nullptr) {
      std::stringstream ss;
      ss << "LENGTH stream not found in StringDictionaryColumn for column " << columnId;
      throw ParseError(ss.str());
    }
    std::unique_ptr<RleDecoder> lengthDecoder =
        createRleDecoder(std::move(stream), false, rleVersion, pool, stripe.getReaderMetrics());

    // Decode dictionary entry lengths
    dictionary->dictionaryOffset.resize(dictSize + 1);
    int64_t* lengthArray = dictionary->dictionaryOffset.data();
    lengthDecoder->next(lengthArray + 1, dictSize, nullptr);
    lengthArray[0] = 0;

    // Convert lengths to cumulative offsets
    for (uint32_t i = 1; i < dictSize + 1; ++i) {
      if (lengthArray[i] < 0) {
        std::stringstream ss;
        ss << "Negative dictionary entry length for column " << columnId;
        throw ParseError(ss.str());
      }
      lengthArray[i] += lengthArray[i - 1];
    }

    int64_t blobSize = lengthArray[dictSize];

    // Read DICTIONARY_DATA stream to get dictionary content
    dictionary->dictionaryBlob.resize(static_cast<uint64_t>(blobSize));
    std::unique_ptr<SeekableInputStream> blobStream =
        stripe.getStream(columnId, proto::Stream_Kind_DICTIONARY_DATA, false);
    if (blobSize > 0 && blobStream == nullptr) {
      std::stringstream ss;
      ss << "DICTIONARY_DATA stream not found in StringDictionaryColumn for column " << columnId;
      throw ParseError(ss.str());
    }

    // Read the dictionary blob
    readFully(dictionary->dictionaryBlob.data(), blobSize, blobStream.get());

    return dictionary;
  }

}  // namespace orc