| 16 | namespace NCB { |
| 17 | |
| 18 | TDataProviderPtr ReadDataset( |
| 19 | TMaybe<ETaskType> taskType, |
| 20 | const TPathWithScheme& poolPath, |
| 21 | const TPathWithScheme& pairsFilePath, // can be uninited |
| 22 | const TPathWithScheme& graphFilePath, // can be uninited |
| 23 | const TPathWithScheme& groupWeightsFilePath, // can be uninited |
| 24 | const TPathWithScheme& timestampsFilePath, // can be uninited |
| 25 | const TPathWithScheme& baselineFilePath, // can be uninited |
| 26 | const TPathWithScheme& featureNamesPath, // can be uninited |
| 27 | const TPathWithScheme& poolMetaInfoPath, // can be uninited |
| 28 | const NCatboostOptions::TColumnarPoolFormatParams& columnarPoolFormatParams, |
| 29 | const TVector<ui32>& ignoredFeatures, |
| 30 | EObjectsOrder objectsOrder, |
| 31 | TDatasetSubset loadSubset, |
| 32 | bool loadSampleIds, |
| 33 | bool forceUnitAutoPairWeights, |
| 34 | TMaybe<TVector<NJson::TJsonValue>*> classLabels, |
| 35 | NPar::ILocalExecutor* localExecutor |
| 36 | ) { |
| 37 | CB_ENSURE_INTERNAL(!baselineFilePath.Inited() || classLabels, "ClassLabels must be specified if baseline file is specified"); |
| 38 | if (classLabels) { |
| 39 | UpdateClassLabelsFromBaselineFile(baselineFilePath, *classLabels); |
| 40 | } |
| 41 | auto datasetLoader = GetProcessor<IDatasetLoader>( |
| 42 | poolPath, // for choosing processor |
| 43 | |
| 44 | // processor args |
| 45 | TDatasetLoaderPullArgs { |
| 46 | poolPath, |
| 47 | |
| 48 | TDatasetLoaderCommonArgs { |
| 49 | pairsFilePath, |
| 50 | graphFilePath, |
| 51 | groupWeightsFilePath, |
| 52 | baselineFilePath, |
| 53 | timestampsFilePath, |
| 54 | featureNamesPath, |
| 55 | poolMetaInfoPath, |
| 56 | classLabels ? **classLabels : TVector<NJson::TJsonValue>(), |
| 57 | columnarPoolFormatParams.DsvFormat, |
| 58 | MakeCdProviderFromFile(columnarPoolFormatParams.CdFilePath), |
| 59 | ignoredFeatures, |
| 60 | objectsOrder, |
| 61 | 10000, // TODO: make it a named constant |
| 62 | loadSubset, |
| 63 | /*LoadColumnsAsString*/ loadSampleIds, |
| 64 | /*LoadSampleIds*/ loadSampleIds, |
| 65 | forceUnitAutoPairWeights, |
| 66 | localExecutor |
| 67 | } |
| 68 | } |
| 69 | ); |
| 70 | |
| 71 | TDataProviderBuilderOptions builderOptions; |
| 72 | builderOptions.GpuDistributedFormat = !loadSubset.HasFeatures && taskType && *taskType == ETaskType::GPU |
| 73 | && EDatasetVisitorType::QuantizedFeatures == datasetLoader->GetVisitorType() |
| 74 | && poolPath.Inited() && IsSharedFs(poolPath); |
| 75 | builderOptions.PoolPath = poolPath; |