| 34 | |
| 35 | |
| 36 | class SJISProber(MultiByteCharSetProber): |
| 37 | def __init__(self): |
| 38 | super(SJISProber, self).__init__() |
| 39 | self.coding_sm = CodingStateMachine(SJIS_SM_MODEL) |
| 40 | self.distribution_analyzer = SJISDistributionAnalysis() |
| 41 | self.context_analyzer = SJISContextAnalysis() |
| 42 | self.reset() |
| 43 | |
| 44 | def reset(self): |
| 45 | super(SJISProber, self).reset() |
| 46 | self.context_analyzer.reset() |
| 47 | |
| 48 | @property |
| 49 | def charset_name(self): |
| 50 | return self.context_analyzer.charset_name |
| 51 | |
| 52 | @property |
| 53 | def language(self): |
| 54 | return "Japanese" |
| 55 | |
| 56 | def feed(self, byte_str): |
| 57 | for i in range(len(byte_str)): |
| 58 | coding_state = self.coding_sm.next_state(byte_str[i]) |
| 59 | if coding_state == MachineState.ERROR: |
| 60 | self.logger.debug('%s %s prober hit error at byte %s', |
| 61 | self.charset_name, self.language, i) |
| 62 | self._state = ProbingState.NOT_ME |
| 63 | break |
| 64 | elif coding_state == MachineState.ITS_ME: |
| 65 | self._state = ProbingState.FOUND_IT |
| 66 | break |
| 67 | elif coding_state == MachineState.START: |
| 68 | char_len = self.coding_sm.get_current_charlen() |
| 69 | if i == 0: |
| 70 | self._last_char[1] = byte_str[0] |
| 71 | self.context_analyzer.feed(self._last_char[2 - char_len:], |
| 72 | char_len) |
| 73 | self.distribution_analyzer.feed(self._last_char, char_len) |
| 74 | else: |
| 75 | self.context_analyzer.feed(byte_str[i + 1 - char_len:i + 3 |
| 76 | - char_len], char_len) |
| 77 | self.distribution_analyzer.feed(byte_str[i - 1:i + 1], |
| 78 | char_len) |
| 79 | |
| 80 | self._last_char[0] = byte_str[-1] |
| 81 | |
| 82 | if self.state == ProbingState.DETECTING: |
| 83 | if (self.context_analyzer.got_enough_data() and |
| 84 | (self.get_confidence() > self.SHORTCUT_THRESHOLD)): |
| 85 | self._state = ProbingState.FOUND_IT |
| 86 | |
| 87 | return self.state |
| 88 | |
| 89 | def get_confidence(self): |
| 90 | context_conf = self.context_analyzer.get_confidence() |
| 91 | distrib_conf = self.distribution_analyzer.get_confidence() |
| 92 | return max(context_conf, distrib_conf) |
no outgoing calls
no test coverage detected
searching dependent graphs…