| 126 | } |
| 127 | |
| 128 | static TVector<ui64> ReadGroupTimestamps( |
| 129 | const TPathWithScheme& filePath, |
| 130 | TConstArrayRef<TGroupId> groupIds, |
| 131 | ui64 docCount, |
| 132 | TDatasetSubset loadSubset |
| 133 | ) { |
| 134 | Y_UNUSED(loadSubset); |
| 135 | THashSet<TGroupId> knownGroups(groupIds.begin(), groupIds.end()); |
| 136 | |
| 137 | THashMap<TGroupId, ui64> groupTimestamps; |
| 138 | ui32 skippedGroupsCount = 0; |
| 139 | |
| 140 | auto reader = GetLineDataReader(filePath); |
| 141 | TString line; |
| 142 | for (size_t lineNumber = 0; reader->ReadLine(&line); ++lineNumber) { |
| 143 | TVector<TString> tokens = StringSplitter(line).Split('\t'); |
| 144 | if (tokens.empty()) { |
| 145 | continue; |
| 146 | } |
| 147 | CB_ENSURE( |
| 148 | tokens.size() == 2, |
| 149 | "Timestamps file " << filePath << ", line number " << lineNumber << ": expect two items, got " << tokens.size()); |
| 150 | auto group = CalcGroupIdFor(tokens[0]); |
| 151 | if (knownGroups.count(group) == 0) { |
| 152 | ++skippedGroupsCount; |
| 153 | continue; |
| 154 | } |
| 155 | CB_ENSURE( |
| 156 | groupTimestamps.insert( |
| 157 | std::make_pair( |
| 158 | group, |
| 159 | FromString<ui64>(tokens[1]) |
| 160 | ) |
| 161 | ).second, |
| 162 | "Timestamps file " << filePath << ", line number " << lineNumber << ": multiple timestamps for GroupId " << tokens[0] |
| 163 | ); |
| 164 | } |
| 165 | CATBOOST_INFO_LOG << "Number of groups from file not in dataset: " << skippedGroupsCount << Endl; |
| 166 | |
| 167 | TVector<ui64> timestamps; |
| 168 | timestamps.reserve(docCount); |
| 169 | CB_ENSURE_INTERNAL(groupIds.size() == docCount, "Each object should have GroupId"); |
| 170 | for (auto group : groupIds) { |
| 171 | CB_ENSURE( |
| 172 | groupTimestamps.count(group) != 0, |
| 173 | "Timestamps file " << filePath << ": no timestamp for GroupId with hash " << group); |
| 174 | timestamps.push_back(groupTimestamps.at(group)); |
| 175 | } |
| 176 | return timestamps; |
| 177 | } |
| 178 | |
| 179 | void SetPairs(const TPathWithScheme& pairsPath, TDatasetSubset loadSubset, IDatasetVisitor* visitor) { |
| 180 | DumpMemUsage("After data read"); |
no test coverage detected