This CharSetProber uses a "code scheme" approach for detecting encodings, whereby easily recognizable escape or shift sequences are relied on to identify these encodings.
| 33 | |
| 34 | |
| 35 | class EscCharSetProber(CharSetProber): |
| 36 | """ |
| 37 | This CharSetProber uses a "code scheme" approach for detecting encodings, |
| 38 | whereby easily recognizable escape or shift sequences are relied on to |
| 39 | identify these encodings. |
| 40 | """ |
| 41 | |
| 42 | def __init__(self, lang_filter=None): |
| 43 | super(EscCharSetProber, self).__init__(lang_filter=lang_filter) |
| 44 | self.coding_sm = [] |
| 45 | if self.lang_filter & LanguageFilter.CHINESE_SIMPLIFIED: |
| 46 | self.coding_sm.append(CodingStateMachine(HZ_SM_MODEL)) |
| 47 | self.coding_sm.append(CodingStateMachine(ISO2022CN_SM_MODEL)) |
| 48 | if self.lang_filter & LanguageFilter.JAPANESE: |
| 49 | self.coding_sm.append(CodingStateMachine(ISO2022JP_SM_MODEL)) |
| 50 | if self.lang_filter & LanguageFilter.KOREAN: |
| 51 | self.coding_sm.append(CodingStateMachine(ISO2022KR_SM_MODEL)) |
| 52 | self.active_sm_count = None |
| 53 | self._detected_charset = None |
| 54 | self._detected_language = None |
| 55 | self._state = None |
| 56 | self.reset() |
| 57 | |
| 58 | def reset(self): |
| 59 | super(EscCharSetProber, self).reset() |
| 60 | for coding_sm in self.coding_sm: |
| 61 | if not coding_sm: |
| 62 | continue |
| 63 | coding_sm.active = True |
| 64 | coding_sm.reset() |
| 65 | self.active_sm_count = len(self.coding_sm) |
| 66 | self._detected_charset = None |
| 67 | self._detected_language = None |
| 68 | |
| 69 | @property |
| 70 | def charset_name(self): |
| 71 | return self._detected_charset |
| 72 | |
| 73 | @property |
| 74 | def language(self): |
| 75 | return self._detected_language |
| 76 | |
| 77 | def get_confidence(self): |
| 78 | if self._detected_charset: |
| 79 | return 0.99 |
| 80 | else: |
| 81 | return 0.00 |
| 82 | |
| 83 | def feed(self, byte_str): |
| 84 | for c in byte_str: |
| 85 | for coding_sm in self.coding_sm: |
| 86 | if not coding_sm or not coding_sm.active: |
| 87 | continue |
| 88 | coding_state = coding_sm.next_state(c) |
| 89 | if coding_state == MachineState.ERROR: |
| 90 | coding_sm.active = False |
| 91 | self.active_sm_count -= 1 |
| 92 | if self.active_sm_count <= 0: |
no outgoing calls
no test coverage detected
searching dependent graphs…