Method Do

catboost/private/libs/quantized_pool/loader.cpp:396–471

}

void NCB::TCBQuantizedDataLoader::Do(IQuantizedFeaturesDataVisitor* visitor) {
    visitor->Start(
        DataMetaInfo,
        ObjectCount,
        ObjectsOrder,
        {},
        QuantizationSchemaFromProto(QuantizedPool.QuantizationSchema),
        /*wholeColumns*/ false);

    const auto columnIdxToTargetIdx = GetColumnIndexToTargetIndexMap(QuantizedPool);
    const auto columnIdxToFlatIdx = GetColumnIndexToFlatIndexMap(QuantizedPool);
    const auto columnIdxToBaselineIdx = GetColumnIndexToBaselineIndexMap(QuantizedPool);
    const auto chunkRefs = GatherAndSortChunks(QuantizedPool);

    TSequentialChunkEvictor evictor(1ULL << 24);
    CATBOOST_DEBUG_LOG << "Number of chunks to process " << chunkRefs.size() << Endl;
    for (const auto chunkRef : chunkRefs) {
        if (QuantizedPool.ChunkStorage.empty()) { // reading from mapped file
            evictor.Push(chunkRef);
        }
        Y_DEFER { evictor.MaybeEvict(); };

        const auto columnIdx = chunkRef.ColumnIndex;
        const auto localIdx = chunkRef.LocalIndex;
        const auto isStringColumn = QuantizedPool.HasStringColumns &&
            (localIdx == QuantizedPool.StringDocIdLocalIndex ||
             localIdx == QuantizedPool.StringGroupIdLocalIndex ||
             localIdx == QuantizedPool.StringSubgroupIdLocalIndex);
        if (isStringColumn) {
            // Ignore string columns, they are only needed for fancy output for evaluation.
            continue;
        }

        const auto columnType = QuantizedPool.ColumnTypes[localIdx];
        if (columnType == EColumn::SampleId) {
            // Skip DocId columns presented in old pools.
            continue;
        }

        CB_ENSURE(
            EqualToOneOf(columnType, EColumn::Num, EColumn::Baseline,
            EColumn::Label, EColumn::Categ, EColumn::Weight,
            EColumn::GroupWeight, EColumn::GroupId, EColumn::SubgroupId,
            EColumn::Timestamp),
            "Expected Num, Baseline, Label, Categ, Weight, GroupWeight, GroupId, Subgroupid, or Timestamp; got "
            LabeledOutput(columnType, columnIdx));
        if (!DatasetSubset.HasFeatures) {
            CB_ENSURE(
                columnType != EColumn::Num && columnType != EColumn::Categ,
                "CollectChunks collected a feature chunk despite HasFeatures = false");
        }

        const auto* const flatFeatureIdx = columnIdxToFlatIdx.FindPtr(columnIdx);
        if (flatFeatureIdx && IsFeatureIgnored[*flatFeatureIdx]) {
            continue;
        }

        const auto* const baselineIdx = columnIdxToBaselineIdx.FindPtr(columnIdx);