Method LoadQuantizedPool

catboost/private/libs/quantized_pool/serialization.cpp:522–621

}

void TFileQuantizedPoolLoader::LoadQuantizedPool(NCB::TLoadQuantizedPoolParameters params) {
    CB_ENSURE_INTERNAL(
        params.DatasetSubset.Range == NCB::TDatasetSubset().Range,
        "Scheme quantized supports only default load subset range"
    );

    Pool.Blobs.push_back(params.LockMemory
        ? TBlob::LockedFromFile(TString(PathWithScheme.Path))
        : TBlob::FromFile(TString(PathWithScheme.Path)));

    // TODO(yazevnul): optionally precharge pool

    const TConstArrayRef<ui8> blob{
        Pool.Blobs.back().AsUnsignedCharPtr(),
        Pool.Blobs.back().Size()};

    ValidatePoolPart(blob);

    TPoolMetainfo poolMetainfo;
    auto parseMetainfo = [&] (TConstArrayRef<ui8> bytes) {
        const auto poolMetainfoParsed = poolMetainfo.ParseFromArray(bytes.data(), bytes.size());
        CB_ENSURE(poolMetainfoParsed);
    };
    auto parseSchema = [&] (TConstArrayRef<ui8> bytes) {
        const auto quantizationSchemaParsed = Pool.QuantizationSchema.ParseFromArray(bytes.data(), bytes.size());
        CB_ENSURE(quantizationSchemaParsed);
    };

    TVector<TVector<NCB::TQuantizedPool::TChunkDescription>> stringColumnChunks;
    THashMap<ui32, EColumn> stringColumnIndexToColumnType;

    TVector<NCB::TQuantizedPool::TChunkDescription>* currentChunksPointer = nullptr;
    auto parseColumn = [&] (ui32 columnIndex) -> bool {
        CB_ENSURE(!Pool.ColumnIndexToLocalIndex.contains(columnIndex),
            "Quantized pool should have unique column indices, but " <<
            LabeledOutput(columnIndex) << " is repeated.");

        const bool isFakeColumn = NCB::NQuantizationSchemaDetail::IsFakeIndex(columnIndex, poolMetainfo);
        if (!isFakeColumn) {
            if (!params.DatasetSubset.HasFeatures) {
                auto pbColumnType = poolMetainfo.columnindextotype().at(columnIndex);
                if (IsFactorColumn(IdlColumnTypeToEColumn(pbColumnType))) {
                    return false;
                }
            }

            const auto localFeatureIndex = Pool.Chunks.size();
            Pool.ColumnIndexToLocalIndex.emplace(columnIndex, localFeatureIndex);
            Pool.Chunks.push_back({});
            currentChunksPointer = &Pool.Chunks.back();
        } else {
            EColumn columnType;
            if (columnIndex == poolMetainfo.GetStringDocIdFakeColumnIndex()) {
                columnType = EColumn::SampleId;
            } else if (columnIndex == poolMetainfo.GetStringGroupIdFakeColumnIndex()) {
                columnType = EColumn::GroupId;
            } else if (columnIndex == poolMetainfo.GetStringSubgroupIdFakeColumnIndex()){
                columnType = EColumn::SubgroupId;