Method Load

catboost/private/libs/text_features/text_processing_collection.cpp:289–337

    }

    void TTextProcessingCollection::Load(IInputStream* stream) {
        DefaultInit(stream);
        auto guidId = CreateComponentGuidsMapping();

        ui64 headerSize;
        while (TryLoad(stream, headerSize)) {
            TArrayHolder<ui8> buffer(new ui8[headerSize]);
            const ui32 loadedBytes = stream->Load(buffer.Get(), headerSize);
            CB_ENSURE(
                loadedBytes == headerSize,
                "Failed to deserialize: Failed to load collection part"
            );

            auto collectionPart = flatbuffers::GetRoot<NCatBoostFbs::TCollectionPart>(buffer.Get());
            const auto partId = GuidFromFbs(collectionPart->Id());

            if (collectionPart->PartType() == NCatBoostFbs::EPartType_Tokenizer) {
                auto tokenizer = MakeIntrusive<TTokenizer>();
                tokenizer->Load(stream);
                CB_ENSURE(
                    partId == tokenizer->Id(),
                    "Failed to deserialize: TokenizerId not equal to PartId"
                );

                Digitizers[guidId[partId]].Tokenizer = std::move(tokenizer);
            } else if (collectionPart->PartType() == NCatBoostFbs::EPartType_Dictionary) {
                auto dictionary = MakeIntrusive<TDictionaryProxy>();
                dictionary->Load(stream);
                CB_ENSURE(
                    partId == dictionary->Id(),
                    "Failed to deserialize: DictionaryId not equal to PartId"
                );

                Digitizers[guidId[partId]].Dictionary = std::move(dictionary);
            } else if (collectionPart->PartType() == NCatBoostFbs::EPartType_FeatureCalcer) {
                TTextFeatureCalcerPtr calcer = TTextCalcerSerializer::Load(stream);
                FeatureCalcers[guidId[partId]] = calcer;
                CB_ENSURE(partId == calcer->Id(), "Failed to deserialize: CalcerId not equal to PartId");
            } else if (collectionPart->PartType() == NCatBoostFbs::EPartType_Terminate){
                break;
            } else {
                CB_ENSURE(false, "Failed to deserialize: Unknown part type");
            }
        }

        CheckForMissingParts();
        CalcRuntimeData();
        CheckPerFeatureIdx();
    }

    void TTextProcessingCollection::LoadNonOwning(TMemoryInput* in) {
        DefaultInit(in);