| 191 | return tokens |
| 192 | |
| 193 | class Parser(_Parser): |
| 194 | |
| 195 | def find_tokens(self, tokens, **kwargs): |
| 196 | kwargs.setdefault("abbreviations", ABBREVIATIONS) |
| 197 | kwargs.setdefault("replace", {}) |
| 198 | return _Parser.find_tokens(self, tokens, **kwargs) |
| 199 | |
| 200 | def find_lemmata(self, tokens, **kwargs): |
| 201 | return find_lemmata(tokens) |
| 202 | |
| 203 | def find_tags(self, tokens, **kwargs): |
| 204 | if kwargs.get("tagset") in (PENN, None): |
| 205 | kwargs.setdefault("map", lambda token, tag: stts2penntreebank(token, tag)) |
| 206 | if kwargs.get("tagset") == UNIVERSAL: |
| 207 | kwargs.setdefault("map", lambda token, tag: stts2universal(token, tag)) |
| 208 | if kwargs.get("tagset") is STTS: |
| 209 | kwargs.setdefault("map", lambda token,tag: (token, tag)) |
| 210 | # The lexicon uses Swiss spelling: "ss" instead of "ß". |
| 211 | # We restore the "ß" after parsing. |
| 212 | tokens_ss = [t.replace(u"ß", "ss") for t in tokens] |
| 213 | tokens_ss = _Parser.find_tags(self, tokens_ss, **kwargs) |
| 214 | return [[w] + tokens_ss[i][1:] for i, w in enumerate(tokens)] |
| 215 | |
| 216 | lexicon = Lexicon( |
| 217 | path = os.path.join(MODULE, "de-lexicon.txt"), |
no outgoing calls
no test coverage detected
searching dependent graphs…