| 31 | |
| 32 | |
| 33 | class SingleByteCharSetProber(CharSetProber): |
| 34 | SAMPLE_SIZE = 64 |
| 35 | SB_ENOUGH_REL_THRESHOLD = 1024 # 0.25 * SAMPLE_SIZE^2 |
| 36 | POSITIVE_SHORTCUT_THRESHOLD = 0.95 |
| 37 | NEGATIVE_SHORTCUT_THRESHOLD = 0.05 |
| 38 | |
| 39 | def __init__(self, model, reversed=False, name_prober=None): |
| 40 | super(SingleByteCharSetProber, self).__init__() |
| 41 | self._model = model |
| 42 | # TRUE if we need to reverse every pair in the model lookup |
| 43 | self._reversed = reversed |
| 44 | # Optional auxiliary prober for name decision |
| 45 | self._name_prober = name_prober |
| 46 | self._last_order = None |
| 47 | self._seq_counters = None |
| 48 | self._total_seqs = None |
| 49 | self._total_char = None |
| 50 | self._freq_char = None |
| 51 | self.reset() |
| 52 | |
| 53 | def reset(self): |
| 54 | super(SingleByteCharSetProber, self).reset() |
| 55 | # char order of last character |
| 56 | self._last_order = 255 |
| 57 | self._seq_counters = [0] * SequenceLikelihood.get_num_categories() |
| 58 | self._total_seqs = 0 |
| 59 | self._total_char = 0 |
| 60 | # characters that fall in our sampling range |
| 61 | self._freq_char = 0 |
| 62 | |
| 63 | @property |
| 64 | def charset_name(self): |
| 65 | if self._name_prober: |
| 66 | return self._name_prober.charset_name |
| 67 | else: |
| 68 | return self._model['charset_name'] |
| 69 | |
| 70 | @property |
| 71 | def language(self): |
| 72 | if self._name_prober: |
| 73 | return self._name_prober.language |
| 74 | else: |
| 75 | return self._model.get('language') |
| 76 | |
| 77 | def feed(self, byte_str): |
| 78 | if not self._model['keep_english_letter']: |
| 79 | byte_str = self.filter_international_words(byte_str) |
| 80 | if not byte_str: |
| 81 | return self.state |
| 82 | char_to_order_map = self._model['char_to_order_map'] |
| 83 | for i, c in enumerate(byte_str): |
| 84 | # XXX: Order is in range 1-64, so one would think we want 0-63 here, |
| 85 | # but that leads to 27 more test failures than before. |
| 86 | order = char_to_order_map[c] |
| 87 | # XXX: This was SYMBOL_CAT_ORDER before, with a value of 250, but |
| 88 | # CharacterCategory.SYMBOL is actually 253, so we use CONTROL |
| 89 | # to make it closer to the original intent. The only difference |
| 90 | # is whether or not we count digits and control characters for |
no outgoing calls
no test coverage detected
searching dependent graphs…