Function dictionaryEncodeStrings

pj_datastore/src/encoding.cpp:291–337

// ---------------------------------------------------------------------------

DictionaryEncoded dictionaryEncodeStrings(
    Span<const uint8_t> offsets_data, Span<const uint8_t> values_data, std::size_t row_count) {
  DictionaryEncoded result;
  result.count = row_count;

  if (row_count == 0) {
    result.index_bytes = 1;
    return result;
  }

  // First pass: build dictionary to determine size
  tsl::robin_map<std::string, uint32_t> lookup;
  std::vector<uint32_t> temp_indices;
  temp_indices.reserve(row_count);

  for (std::size_t row = 0; row < row_count; ++row) {
    uint32_t start_offset = 0;
    uint32_t end_offset = 0;
    std::memcpy(&start_offset, offsets_data.data() + row * sizeof(uint32_t), sizeof(uint32_t));
    std::memcpy(&end_offset, offsets_data.data() + (row + 1) * sizeof(uint32_t), sizeof(uint32_t));

    std::string_view str_view(
        reinterpret_cast<const char*>(values_data.data() + start_offset), end_offset - start_offset);

    std::string key_str(str_view);
    auto it = lookup.find(key_str);
    uint32_t index = 0;
    if (it != lookup.end()) {
      index = it->second;
    } else {
      index = static_cast<uint32_t>(result.dictionary.size());
      result.dictionary.push_back(key_str);
      lookup[std::move(key_str)] = index;
    }
    temp_indices.push_back(index);
  }

  // Determine narrowed index width
  result.index_bytes = indexBytesFor(result.dictionary.size());
  result.indices.reserve(row_count * result.index_bytes);

  for (uint32_t idx : temp_indices) {
    writeIndex(result.indices, idx, result.index_bytes);
  }

  return result;
}

std::string_view dictionaryLookup(const DictionaryEncoded& encoded, std::size_t row) {
  uint32_t index = readIndex(encoded.indices.data(), row, encoded.index_bytes);