| 203 | |
| 204 | |
| 205 | void TLibSvmDataLoader::ProcessBlock(IRawObjectsOrderDataVisitor* visitor) { |
| 206 | visitor->StartNextBlock(AsyncRowProcessor.GetParseBufferSize()); |
| 207 | |
| 208 | auto parseBlock = [&](TString& line, int lineIdx) { |
| 209 | const auto& featuresLayout = *DataMetaInfo.FeaturesLayout; |
| 210 | |
| 211 | TConstArrayRef<TFeatureMetaInfo> featuresMetaInfo = featuresLayout.GetExternalFeaturesMetaInfo(); |
| 212 | |
| 213 | TVector<ui32> floatFeatureIndices; |
| 214 | floatFeatureIndices.reserve(featuresLayout.GetFloatFeatureCount()); |
| 215 | TVector<float> floatFeatureValues; |
| 216 | floatFeatureValues.reserve(featuresLayout.GetFloatFeatureCount()); |
| 217 | |
| 218 | TVector<ui32> catFeatureIndices; |
| 219 | catFeatureIndices.reserve(featuresLayout.GetCatFeatureCount()); |
| 220 | TVector<ui32> catFeatureValues; |
| 221 | catFeatureValues.reserve(featuresLayout.GetCatFeatureCount()); |
| 222 | |
| 223 | try { |
| 224 | auto lineSplitter = StringSplitter(line).Split(' '); |
| 225 | auto lineIterator = lineSplitter.begin(); |
| 226 | auto lineEndIterator = lineSplitter.end(); |
| 227 | |
| 228 | size_t tokenCount = 0; |
| 229 | TStringBuf token; |
| 230 | ui32 lastFeatureIdxPlus1 = 0; // +1 to allow to compare first featureIdx |
| 231 | try { |
| 232 | CB_ENSURE(lineIterator != lineEndIterator, "line is empty"); |
| 233 | token = (*lineIterator).Token(); |
| 234 | |
| 235 | CB_ENSURE(token.length() != 0, "empty values not supported for Label"); |
| 236 | float label; |
| 237 | CB_ENSURE(TryFromString(token, label), "Target value must be float"); |
| 238 | visitor->AddTarget(lineIdx, label); |
| 239 | |
| 240 | ++tokenCount; |
| 241 | ++lineIterator; |
| 242 | |
| 243 | if (DataMetaInfo.HasGroupId) { |
| 244 | CB_ENSURE(lineIterator != lineEndIterator, "line does not contain 'qid' field"); |
| 245 | token = (*lineIterator).Token(); |
| 246 | |
| 247 | TStringBuf left; |
| 248 | TStringBuf right; |
| 249 | token.Split(':', left, right); |
| 250 | |
| 251 | CB_ENSURE(left == "qid"sv, "line does not contain 'qid' field"); |
| 252 | TGroupId groupId; |
| 253 | CB_ENSURE(TryFromString(right, groupId), "'qid' value must be integer"); |
| 254 | visitor->AddGroupId(lineIdx, groupId); |
| 255 | |
| 256 | ++tokenCount; |
| 257 | ++lineIterator; |
| 258 | } |
| 259 | |
| 260 | for (; lineIterator != lineEndIterator; ++lineIterator, ++tokenCount) { |
| 261 | token = (*lineIterator).Token(); |
| 262 | // allow extra space at the end of line |
nothing calls this directly
no test coverage detected