| 194 | } |
| 195 | |
| 196 | TIntrusivePtr<TBpeDictionary> NTextProcessing::NDictionary::BuildBpe( |
| 197 | const TString& inputFilePath, |
| 198 | const TDictionaryBuilderOptions& dictionaryBuilderOptions, |
| 199 | const TDictionaryOptions& dictionaryOptions, |
| 200 | const TBpeDictionaryOptions& bpeOptions, |
| 201 | const TTokenizerOptions& tokenizerOptions, |
| 202 | bool useTokenizer, |
| 203 | bool verbose |
| 204 | ) { |
| 205 | if (dictionaryOptions.TokenLevelType == ETokenLevelType::Word || bpeOptions.ContextLevel == EContextLevel::Sentence) { |
| 206 | return BuildBpeWord( |
| 207 | dictionaryBuilderOptions, |
| 208 | dictionaryOptions, |
| 209 | bpeOptions, |
| 210 | inputFilePath, |
| 211 | tokenizerOptions, |
| 212 | useTokenizer, |
| 213 | verbose |
| 214 | ); |
| 215 | } else { |
| 216 | Y_ASSERT(dictionaryOptions.TokenLevelType == ETokenLevelType::Letter); |
| 217 | return BuildBpeLetter( |
| 218 | dictionaryBuilderOptions, |
| 219 | dictionaryOptions, |
| 220 | bpeOptions, |
| 221 | inputFilePath, |
| 222 | tokenizerOptions, |
| 223 | useTokenizer, |
| 224 | verbose |
| 225 | ); |
| 226 | } |
| 227 | } |
| 228 | |
| 229 | void NTextProcessing::NDictionary::ApplyDictionaryToFile( |
| 230 | const TString& inputPath, |
nothing calls this directly
no test coverage detected