MCPcopy Create free account
hub / github.com/catboost/catboost / AddPair

Function AddPair

library/cpp/text_processing/dictionary/bpe_dictionary.cpp:26–43  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

24
25template <typename TUnitToTokenId>
26static void AddPair(
27 int firstPosition,
28 const TUnitToTokenId& unitToTokenId,
29 TEraseList<TTokenId>* tokenIdsList,
30 TPositionsMap* positions,
31 TUnitQueue* minIds
32) {
33 if (tokenIdsList->IsLastElement(firstPosition)) {
34 return;
35 }
36 auto unit = tokenIdsList->GetPair(firstPosition);
37 auto unitId = unitToTokenId(unit);
38 if (!unitId) {
39 return;
40 }
41 minIds->push({*unitId, unit});
42 (*positions)[*unitId].push_back(firstPosition);
43}
44
45// TODO(annaveronika): more efficient apply.
46template <typename TStringVector, typename TUnitToTokenId>

Callers 1

ApplyImplFunction · 0.85

Calls 4

IsLastElementMethod · 0.80
GetPairMethod · 0.80
pushMethod · 0.45
push_backMethod · 0.45

Tested by

no test coverage detected