Function BuildBpeWord

library/cpp/text_processing/app_helpers/app_helpers.cpp:108–147

}

static TIntrusivePtr<TBpeDictionary> BuildBpeWord(
    const TDictionaryBuilderOptions& dictionaryBuilderOptions,
    const TDictionaryOptions& dictionaryOptions,
    const TBpeDictionaryOptions& bpeOptions,
    const TString& inputFilePath,
    const TTokenizerOptions& tokenizerOptions,
    bool useTokenizer,
    bool verbose
) {
    if (verbose) {
        Cerr << "Stage [1/2]: Dictionary building\n";
    }

    auto dictionary = BuildDictionary(
        inputFilePath,
        dictionaryBuilderOptions,
        dictionaryOptions,
        tokenizerOptions,
        useTokenizer,
        verbose
    );

    TBpeDictionaryBuilder bpeBuilder(bpeOptions.NumUnits, bpeOptions.SkipUnknown, dictionary);

    if (verbose) {
        Cerr << "Stage [2/2]: Bpe building\n";
    }

    ApplyFuncTotokenizedText(
        inputFilePath,
        tokenizerOptions,
        useTokenizer,
        verbose,
        [&](const TVector<TString>& tokens) {
            bpeBuilder.Add(tokens, /*weight*/1);
        }
    );

    return bpeBuilder.FinishBuilding();
}

static TIntrusivePtr<TBpeDictionary> BuildBpeLetter(
    const TDictionaryBuilderOptions& dictionaryBuilderOptions,