| 128 | return tokens |
| 129 | |
| 130 | class Parser(_Parser): |
| 131 | |
| 132 | def find_tokens(self, tokens, **kwargs): |
| 133 | kwargs.setdefault("abbreviations", ABBREVIATIONS) |
| 134 | kwargs.setdefault("replace", replacements) |
| 135 | s = _Parser.find_tokens(self, tokens, **kwargs) |
| 136 | s = [s.replace("&rsquo ;", u"’") if isinstance(s, unicode) else s for s in s] |
| 137 | return s |
| 138 | |
| 139 | def find_lemmata(self, tokens, **kwargs): |
| 140 | return find_lemmata(tokens) |
| 141 | |
| 142 | def find_tags(self, tokens, **kwargs): |
| 143 | if kwargs.get("tagset") in (PENN, None): |
| 144 | kwargs.setdefault("map", lambda token, tag: (token, tag)) |
| 145 | if kwargs.get("tagset") == UNIVERSAL: |
| 146 | kwargs.setdefault("map", lambda token, tag: penntreebank2universal(token, tag)) |
| 147 | return _Parser.find_tags(self, tokens, **kwargs) |
| 148 | |
| 149 | lexicon = Lexicon( |
| 150 | path = os.path.join(MODULE, "fr-lexicon.txt"), |
no outgoing calls
no test coverage detected
searching dependent graphs…