Function BuildBpeLetter

library/cpp/text_processing/app_helpers/app_helpers.cpp:149–194

}

static TIntrusivePtr<TBpeDictionary> BuildBpeLetter(
    const TDictionaryBuilderOptions& dictionaryBuilderOptions,
    const TDictionaryOptions& dictionaryOptions,
    const TBpeDictionaryOptions& bpeOptions,
    const TString& inputFilePath,
    const TTokenizerOptions& tokenizerOptions,
    bool useTokenizer,
    bool verbose
) {
    if (verbose) {
        Cerr << "Stage [1/2]: Dictionary building\n";
    }

    NFH::TFlatHashMap<TString, ui64> tokenCounts;
    TDictionaryBuilder dictionaryBuilder(dictionaryBuilderOptions, dictionaryOptions);

    ApplyFuncTotokenizedText(
        inputFilePath,
        tokenizerOptions,
        useTokenizer,
        verbose,
        [&](const TVector<TString>& tokens) {
            dictionaryBuilder.Add(tokens, /*weight*/1);
            for (const auto& token : tokens) {
                ++tokenCounts[token];
            }
        }
    );

    auto dictionary = dictionaryBuilder.FinishBuilding();

    if (verbose) {
        Cerr << "Stage [2/2]: Bpe building\n";
    }

    TBpeDictionaryBuilder bpeBuilder(
        bpeOptions.NumUnits,
        bpeOptions.SkipUnknown,
        dictionary);

    for (const auto& [token, count] : tokenCounts) {
        bpeBuilder.Add(TVector<TStringBuf>({token}), count);
    }

    return bpeBuilder.FinishBuilding();
}

TIntrusivePtr<TBpeDictionary> NTextProcessing::NDictionary::BuildBpe(
    const TString& inputFilePath,