Method Load

library/cpp/text_processing/dictionary/bpe_dictionary.cpp:217–226

}

void TBpeDictionary::Load(IInputStream* stream) {
    TString line;
    while (stream->ReadLine(line)) {
        TBpeUnit unit;
        TString _;
        StringSplitter(line).Split('\t').Limit(4).CollectInto(&unit.Left, &unit.Right, &unit.Count, &_);
        BpeUnits.push_back(unit);
    }
    InitBpeTokens();
}

void TBpeDictionary::Load(const TString& dictionaryPath, const TString& bpePath) {
    TFileInput dictInput(dictionaryPath);