| 85 | } |
| 86 | |
| 87 | TIntrusivePtr<TDictionary> NTextProcessing::NDictionary::BuildDictionary( |
| 88 | const TString& inputFilePath, |
| 89 | const TDictionaryBuilderOptions& dictionaryBuilderOptions, |
| 90 | const TDictionaryOptions& dictionaryOptions, |
| 91 | const TTokenizerOptions& tokenizerOptions, |
| 92 | bool useTokenizer, |
| 93 | bool verbose |
| 94 | ) { |
| 95 | TDictionaryBuilder dictionaryBuilder(dictionaryBuilderOptions, dictionaryOptions); |
| 96 | ApplyFuncTotokenizedText( |
| 97 | inputFilePath, |
| 98 | tokenizerOptions, |
| 99 | useTokenizer, |
| 100 | verbose, |
| 101 | [&](const TVector<TString>& tokens) { |
| 102 | dictionaryBuilder.Add(tokens, /*weight*/1); |
| 103 | } |
| 104 | ); |
| 105 | return dictionaryBuilder.FinishBuilding(); |
| 106 | } |
| 107 | |
| 108 | static TIntrusivePtr<TBpeDictionary> BuildBpeWord( |
| 109 | const TDictionaryBuilderOptions& dictionaryBuilderOptions, |
nothing calls this directly
no test coverage detected