| 126 | # charset identified, either "windows-1255" or "ISO-8859-8". |
| 127 | |
| 128 | class HebrewProber(CharSetProber): |
| 129 | # windows-1255 / ISO-8859-8 code points of interest |
| 130 | FINAL_KAF = 0xea |
| 131 | NORMAL_KAF = 0xeb |
| 132 | FINAL_MEM = 0xed |
| 133 | NORMAL_MEM = 0xee |
| 134 | FINAL_NUN = 0xef |
| 135 | NORMAL_NUN = 0xf0 |
| 136 | FINAL_PE = 0xf3 |
| 137 | NORMAL_PE = 0xf4 |
| 138 | FINAL_TSADI = 0xf5 |
| 139 | NORMAL_TSADI = 0xf6 |
| 140 | |
| 141 | # Minimum Visual vs Logical final letter score difference. |
| 142 | # If the difference is below this, don't rely solely on the final letter score |
| 143 | # distance. |
| 144 | MIN_FINAL_CHAR_DISTANCE = 5 |
| 145 | |
| 146 | # Minimum Visual vs Logical model score difference. |
| 147 | # If the difference is below this, don't rely at all on the model score |
| 148 | # distance. |
| 149 | MIN_MODEL_DISTANCE = 0.01 |
| 150 | |
| 151 | VISUAL_HEBREW_NAME = "ISO-8859-8" |
| 152 | LOGICAL_HEBREW_NAME = "windows-1255" |
| 153 | |
| 154 | def __init__(self): |
| 155 | super(HebrewProber, self).__init__() |
| 156 | self._final_char_logical_score = None |
| 157 | self._final_char_visual_score = None |
| 158 | self._prev = None |
| 159 | self._before_prev = None |
| 160 | self._logical_prober = None |
| 161 | self._visual_prober = None |
| 162 | self.reset() |
| 163 | |
| 164 | def reset(self): |
| 165 | self._final_char_logical_score = 0 |
| 166 | self._final_char_visual_score = 0 |
| 167 | # The two last characters seen in the previous buffer, |
| 168 | # mPrev and mBeforePrev are initialized to space in order to simulate |
| 169 | # a word delimiter at the beginning of the data |
| 170 | self._prev = ' ' |
| 171 | self._before_prev = ' ' |
| 172 | # These probers are owned by the group prober. |
| 173 | |
| 174 | def set_model_probers(self, logicalProber, visualProber): |
| 175 | self._logical_prober = logicalProber |
| 176 | self._visual_prober = visualProber |
| 177 | |
| 178 | def is_final(self, c): |
| 179 | return c in [self.FINAL_KAF, self.FINAL_MEM, self.FINAL_NUN, |
| 180 | self.FINAL_PE, self.FINAL_TSADI] |
| 181 | |
| 182 | def is_non_final(self, c): |
| 183 | # The normal Tsadi is not a good Non-Final letter due to words like |
| 184 | # 'lechotet' (to chat) containing an apostrophe after the tsadi. This |
| 185 | # apostrophe is converted to a space in FilterWithoutEnglishLetters |
no outgoing calls
no test coverage detected
searching dependent graphs…