| 33 | |
| 34 | |
| 35 | class UTF8Prober(CharSetProber): |
| 36 | ONE_CHAR_PROB = 0.5 |
| 37 | |
| 38 | def __init__(self): |
| 39 | super(UTF8Prober, self).__init__() |
| 40 | self.coding_sm = CodingStateMachine(UTF8_SM_MODEL) |
| 41 | self._num_mb_chars = None |
| 42 | self.reset() |
| 43 | |
| 44 | def reset(self): |
| 45 | super(UTF8Prober, self).reset() |
| 46 | self.coding_sm.reset() |
| 47 | self._num_mb_chars = 0 |
| 48 | |
| 49 | @property |
| 50 | def charset_name(self): |
| 51 | return "utf-8" |
| 52 | |
| 53 | @property |
| 54 | def language(self): |
| 55 | return "" |
| 56 | |
| 57 | def feed(self, byte_str): |
| 58 | for c in byte_str: |
| 59 | coding_state = self.coding_sm.next_state(c) |
| 60 | if coding_state == MachineState.ERROR: |
| 61 | self._state = ProbingState.NOT_ME |
| 62 | break |
| 63 | elif coding_state == MachineState.ITS_ME: |
| 64 | self._state = ProbingState.FOUND_IT |
| 65 | break |
| 66 | elif coding_state == MachineState.START: |
| 67 | if self.coding_sm.get_current_charlen() >= 2: |
| 68 | self._num_mb_chars += 1 |
| 69 | |
| 70 | if self.state == ProbingState.DETECTING: |
| 71 | if self.get_confidence() > self.SHORTCUT_THRESHOLD: |
| 72 | self._state = ProbingState.FOUND_IT |
| 73 | |
| 74 | return self.state |
| 75 | |
| 76 | def get_confidence(self): |
| 77 | unlike = 0.99 |
| 78 | if self._num_mb_chars < 6: |
| 79 | unlike *= self.ONE_CHAR_PROB ** self._num_mb_chars |
| 80 | return 1.0 - unlike |
| 81 | else: |
| 82 | return unlike |
no outgoing calls
no test coverage detected
searching dependent graphs…