| 1879 | |
| 1880 | |
| 1881 | static void ProcessCatFeature( |
| 1882 | TCatFeatureIdx catFeatureIdx, |
| 1883 | const THashedCatValuesHolder& srcFeature, |
| 1884 | const TQuantizationOptions& options, |
| 1885 | bool bundleExclusiveFeatures, |
| 1886 | bool storeFeaturesDataAsExternalValuesHolder, |
| 1887 | const TIncrementalDenseIndexing& incrementalDenseIndexing, |
| 1888 | const TFeaturesArraySubsetIndexing* dstSubsetIndexing, |
| 1889 | NPar::ILocalExecutor* localExecutor, |
| 1890 | TQuantizedFeaturesInfoPtr quantizedFeaturesInfo, |
| 1891 | THolder<IQuantizedCatValuesHolder>* dstQuantizedFeature |
| 1892 | ) { |
| 1893 | const bool updatePerfectHashOnly = bundleExclusiveFeatures; |
| 1894 | |
| 1895 | // GPU-only external columns |
| 1896 | const bool quantizeData = !updatePerfectHashOnly && !storeFeaturesDataAsExternalValuesHolder; |
| 1897 | |
| 1898 | |
| 1899 | TCompressedArray quantizedDataStorage; |
| 1900 | |
| 1901 | auto onNonDefaultValues = [&] ( |
| 1902 | const ITypedArraySubset<ui32>& srcNonDefaultValues, |
| 1903 | TMaybe<TDefaultValue<ui32>> srcDefaultValue) { |
| 1904 | |
| 1905 | // can quantize data at first pass only if data is dense and default bin value won't be determined |
| 1906 | const bool quantizeDataAtFirstPass |
| 1907 | = quantizeData && !srcDefaultValue.Defined() && |
| 1908 | !options.DefaultValueFractionToEnableSparseStorage.Defined(); |
| 1909 | |
| 1910 | TArrayRef<ui32> quantizedDataValue; |
| 1911 | |
| 1912 | if (quantizeDataAtFirstPass) { |
| 1913 | // TODO(akhropov): support other bitsPerKey. MLTOOLS-2425 |
| 1914 | const ui32 bitsPerKey = 32; |
| 1915 | |
| 1916 | quantizedDataStorage |
| 1917 | = TCompressedArray::CreateWithUninitializedData(srcNonDefaultValues.GetSize(), bitsPerKey); |
| 1918 | quantizedDataValue = quantizedDataStorage.GetRawArray<ui32>(); |
| 1919 | } |
| 1920 | |
| 1921 | TCatFeaturesPerfectHashHelper catFeaturesPerfectHashHelper(quantizedFeaturesInfo); |
| 1922 | |
| 1923 | catFeaturesPerfectHashHelper.UpdatePerfectHashAndMaybeQuantize( |
| 1924 | catFeatureIdx, |
| 1925 | srcNonDefaultValues, |
| 1926 | /*mapMostFrequentValueTo0*/ bundleExclusiveFeatures, |
| 1927 | srcDefaultValue, |
| 1928 | options.DefaultValueFractionToEnableSparseStorage, |
| 1929 | quantizeDataAtFirstPass ? TMaybe<TArrayRef<ui32>*>(&quantizedDataValue) : Nothing() |
| 1930 | ); |
| 1931 | }; |
| 1932 | |
| 1933 | if (const auto* denseSrcFeature = dynamic_cast<const THashedCatArrayValuesHolder*>(&srcFeature)) { |
| 1934 | onNonDefaultValues(*denseSrcFeature->GetData(), Nothing()); |
| 1935 | } else if (const auto* sparseSrcFeature |
| 1936 | = dynamic_cast<const THashedCatSparseValuesHolder*>(&srcFeature)) |
| 1937 | { |
| 1938 | const TConstPolymorphicValuesSparseArray<ui32, ui32>& sparseArray = sparseSrcFeature->GetData(); |
no test coverage detected