MCPcopy Create free account
hub / github.com/catboost/catboost / NormalizeEncodingPrefixes

Function NormalizeEncodingPrefixes

library/cpp/charset/codepage.cpp:265–309  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

263};
264
265static inline void NormalizeEncodingPrefixes(TString& enc) {
266 size_t preflen = enc.find_first_of("0123456789");
267 if (preflen == TString::npos)
268 return;
269
270 TString prefix = enc.substr(0, preflen);
271 for (size_t i = 0; i < prefix.length(); ++i) {
272 if (prefix[i] == '-') {
273 prefix.remove(i--);
274 }
275 }
276
277 if (Singleton<TWindowsPrefixesHashSet>()->Has(prefix)) {
278 enc.remove(0, preflen);
279 enc.prepend("windows-");
280 return;
281 }
282
283 if (Singleton<TCpPrefixesHashSet>()->Has(prefix)) {
284 if (enc.length() > preflen + 3 && !strncmp(enc.c_str() + preflen, "125", 3) && isdigit(enc[preflen + 3])) {
285 enc.remove(0, preflen);
286 enc.prepend("windows-");
287 return;
288 }
289 enc.remove(0, preflen);
290 enc.prepend("cp");
291 return;
292 }
293
294 if (Singleton<TIsoPrefixesHashSet>()->Has(prefix)) {
295 if (enc.length() == preflen + 1 || enc.length() == preflen + 2) {
296 TString enccopy = enc.substr(preflen);
297 enccopy.prepend("latin");
298 const TLatinToIsoHash* latinhash = Singleton<TLatinToIsoHash>();
299 TLatinToIsoHash::const_iterator it = latinhash->find(enccopy.data());
300 if (it != latinhash->end())
301 enc.assign(it->second);
302 return;
303 } else if (enc.length() > preflen + 5 && enc[preflen] == '8') {
304 enc.remove(0, preflen);
305 enc.prepend("iso-");
306 return;
307 }
308 }
309}
310
311class TEncodingNamesHashSet: public THashSetType {
312public:

Callers 1

EncodingHintByNameFunction · 0.85

Calls 11

find_first_ofMethod · 0.45
substrMethod · 0.45
lengthMethod · 0.45
removeMethod · 0.45
HasMethod · 0.45
prependMethod · 0.45
c_strMethod · 0.45
findMethod · 0.45
dataMethod · 0.45
endMethod · 0.45
assignMethod · 0.45

Tested by

no test coverage detected