| 150 | self.other_words.update(set(all_words)) |
| 151 | |
| 152 | def post_process(self, sent, check_seperated): |
| 153 | for m in reversed(range(2, 8)): |
| 154 | end = len(sent)-m |
| 155 | if end < 0: |
| 156 | continue |
| 157 | i = 0 |
| 158 | while (i < end + 1): |
| 159 | merged_words = ''.join(sent[i:i+m]) |
| 160 | if merged_words in self.common_words: |
| 161 | do_seg = True |
| 162 | elif merged_words in self.other_words: |
| 163 | if check_seperated: |
| 164 | seperated = all(((w in self.common_words) |
| 165 | or (w in self.other_words)) for w in sent[i:i+m]) |
| 166 | else: |
| 167 | seperated = False |
| 168 | if seperated: |
| 169 | do_seg = False |
| 170 | else: |
| 171 | do_seg = True |
| 172 | else: |
| 173 | do_seg = False |
| 174 | if do_seg: |
| 175 | for k in range(m): |
| 176 | del sent[i] |
| 177 | sent.insert(i, merged_words) |
| 178 | i += 1 |
| 179 | end = len(sent) - m |
| 180 | else: |
| 181 | i += 1 |
| 182 | return sent |
| 183 | |
| 184 | def __call__(self, sent): |
| 185 | if not self.do_process: |