| 758 | } // anonymous namespace |
| 759 | |
| 760 | TDataProviderPtr NCB::ReadAndQuantizeDataset( |
| 761 | const TPathWithScheme& poolPath, |
| 762 | const TPathWithScheme& pairsFilePath, // can be uninited |
| 763 | const TPathWithScheme& graphFilePath, // can be uninited |
| 764 | const TPathWithScheme& groupWeightsFilePath, // can be uninited |
| 765 | const TPathWithScheme& timestampsFilePath, // can be uninited |
| 766 | const TPathWithScheme& baselineFilePath, // can be uninited |
| 767 | const TPathWithScheme& featureNamesPath, // can be uninited |
| 768 | const TPathWithScheme& poolMetaInfoPath, // can be uninited |
| 769 | const TPathWithScheme& inputBordersPath, // can be uninited |
| 770 | const NCatboostOptions::TColumnarPoolFormatParams& columnarPoolFormatParams, |
| 771 | const TVector<ui32>& ignoredFeatures, |
| 772 | EObjectsOrder objectsOrder, |
| 773 | NJson::TJsonValue plainJsonParams, |
| 774 | TMaybe<ui32> blockSize, |
| 775 | TQuantizedFeaturesInfoPtr quantizedFeaturesInfo, |
| 776 | TDatasetSubset loadSubset, |
| 777 | TMaybe<TVector<NJson::TJsonValue>*> classLabels, |
| 778 | NPar::ILocalExecutor* localExecutor) { |
| 779 | |
| 780 | if (!blockSize) { |
| 781 | blockSize = 10000; |
| 782 | } |
| 783 | |
| 784 | TVector<NJson::TJsonValue> emptyClassLabels; |
| 785 | CB_ENSURE_INTERNAL( |
| 786 | !baselineFilePath.Inited() || classLabels, |
| 787 | "ClassLabels must be specified if baseline file is specified"); |
| 788 | if (classLabels) { |
| 789 | UpdateClassLabelsFromBaselineFile(baselineFilePath, *classLabels); |
| 790 | } else { |
| 791 | classLabels = &emptyClassLabels; |
| 792 | } |
| 793 | |
| 794 | NJson::TJsonValue jsonParams; |
| 795 | NJson::TJsonValue outputJsonParams; |
| 796 | NCatboostOptions::PlainJsonToOptions(plainJsonParams, &jsonParams, &outputJsonParams); |
| 797 | NCatboostOptions::TCatBoostOptions catBoostOptions(NCatboostOptions::LoadOptions(jsonParams)); |
| 798 | |
| 799 | auto datasetLoader = GetProcessor<IDatasetLoader>( |
| 800 | poolPath, // for choosing processor |
| 801 | // processor args |
| 802 | TDatasetLoaderPullArgs{ |
| 803 | poolPath, |
| 804 | TDatasetLoaderCommonArgs { |
| 805 | pairsFilePath, |
| 806 | graphFilePath, |
| 807 | groupWeightsFilePath, |
| 808 | baselineFilePath, |
| 809 | timestampsFilePath, |
| 810 | featureNamesPath, |
| 811 | poolMetaInfoPath, |
| 812 | **classLabels, |
| 813 | columnarPoolFormatParams.DsvFormat, |
| 814 | MakeCdProviderFromFile(columnarPoolFormatParams.CdFilePath), |
| 815 | ignoredFeatures, |
| 816 | objectsOrder, |
| 817 | *blockSize, |
nothing calls this directly
no test coverage detected