MCPcopy Create free account
hub / github.com/apache/arrow / ReadDictionary

Function ReadDictionary

cpp/src/arrow/ipc/reader.cc:908–965  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

906}
907
908Status ReadDictionary(const Buffer& metadata, IpcReadContext context,
909 DictionaryKind* kind, io::RandomAccessFile* file) {
910 const flatbuf::Message* message = nullptr;
911 RETURN_NOT_OK(internal::VerifyMessage(metadata.data(), metadata.size(), &message));
912 const auto dictionary_batch = message->header_as_DictionaryBatch();
913 if (dictionary_batch == nullptr) {
914 return Status::IOError(
915 "Header-type of flatbuffer-encoded Message is not DictionaryBatch.");
916 }
917
918 // The dictionary is embedded in a record batch with a single column
919 const auto batch_meta = dictionary_batch->data();
920
921 CHECK_FLATBUFFERS_NOT_NULL(batch_meta, "DictionaryBatch.data");
922
923 RETURN_NOT_OK(GetCompression(batch_meta, &context.compression));
924 if (context.compression == Compression::UNCOMPRESSED &&
925 message->version() == flatbuf::MetadataVersion::MetadataVersion_V4) {
926 // Possibly obtain codec information from experimental serialization format
927 // in 0.17.x
928 RETURN_NOT_OK(GetCompressionExperimental(message, &context.compression));
929 }
930
931 const int64_t id = dictionary_batch->id();
932
933 // Look up the dictionary value type, which must have been added to the
934 // DictionaryMemo already prior to invoking this function
935 ARROW_ASSIGN_OR_RAISE(auto value_type, context.dictionary_memo->GetDictionaryType(id));
936
937 // Load the dictionary data from the dictionary batch
938 ArrayLoader loader(batch_meta, internal::GetMetadataVersion(message->version()),
939 context.options, file);
940 auto dict_data = std::make_shared<ArrayData>();
941 const Field dummy_field("", value_type);
942 RETURN_NOT_OK(loader.Load(&dummy_field, dict_data.get()));
943
944 // Run post-load steps: buffer decompression, etc.
945 RecordBatchLoader batch_loader{context, /*schema=*/nullptr, batch_meta->length(),
946 /*inclusion_mask=*/std::vector<bool>{}};
947 ARROW_ASSIGN_OR_RAISE(
948 auto dict_columns,
949 batch_loader.CreateColumns({dict_data}, /*resolve_dictionaries=*/false));
950 DCHECK_EQ(dict_columns.size(), 1);
951 dict_data = dict_columns[0];
952
953 if (dictionary_batch->isDelta()) {
954 if (kind != nullptr) {
955 *kind = DictionaryKind::Delta;
956 }
957 return context.dictionary_memo->AddDictionaryDelta(id, dict_data);
958 }
959 ARROW_ASSIGN_OR_RAISE(bool inserted,
960 context.dictionary_memo->AddOrReplaceDictionary(id, dict_data));
961 if (kind != nullptr) {
962 *kind = inserted ? DictionaryKind::New : DictionaryKind::Replacement;
963 }
964 return Status::OK();
965}

Callers 4

ReadDictionaryMethod · 0.70
ReadOneDictionaryMethod · 0.70

Calls 15

VerifyMessageFunction · 0.85
IOErrorFunction · 0.85
GetCompressionFunction · 0.85
GetMetadataVersionFunction · 0.85
AddDictionaryDeltaMethod · 0.80
OKFunction · 0.50
dataMethod · 0.45
sizeMethod · 0.45
versionMethod · 0.45
idMethod · 0.45
LoadMethod · 0.45

Tested by

no test coverage detected