Takes a string (sentences) and returns a tagged Unicode string (TaggedString). Sentences in the output are separated by newlines. With tokenize=True, punctuation is split from words and sentences are separated by \n. With tags=True, part-of-speech tags are parse
(self, s, tokenize=True, tags=True, chunks=True, relations=False, lemmata=False, encoding="utf-8", **kwargs)
| 556 | return [token + [token[0].lower()] for token in tokens] |
| 557 | |
| 558 | def parse(self, s, tokenize=True, tags=True, chunks=True, relations=False, lemmata=False, encoding="utf-8", **kwargs): |
| 559 | """ Takes a string (sentences) and returns a tagged Unicode string (TaggedString). |
| 560 | Sentences in the output are separated by newlines. |
| 561 | With tokenize=True, punctuation is split from words and sentences are separated by \n. |
| 562 | With tags=True, part-of-speech tags are parsed (NN, VB, IN, ...). |
| 563 | With chunks=True, phrase chunk tags are parsed (NP, VP, PP, PNP, ...). |
| 564 | With relations=True, semantic role labels are parsed (SBJ, OBJ). |
| 565 | With lemmata=True, word lemmata are parsed. |
| 566 | Optional parameters are passed to |
| 567 | the tokenizer, tagger, chunker, labeler and lemmatizer. |
| 568 | """ |
| 569 | # Tokenizer. |
| 570 | if tokenize is True: |
| 571 | s = self.find_tokens(s) |
| 572 | if isinstance(s, (list, tuple)): |
| 573 | s = [isinstance(s, basestring) and s.split(" ") or s for s in s] |
| 574 | if isinstance(s, basestring): |
| 575 | s = [s.split(" ") for s in s.split("\n")] |
| 576 | # Unicode. |
| 577 | for i in range(len(s)): |
| 578 | for j in range(len(s[i])): |
| 579 | if isinstance(s[i][j], str): |
| 580 | s[i][j] = decode_string(s[i][j], encoding) |
| 581 | # Tagger (required by chunker, labeler & lemmatizer). |
| 582 | if tags or chunks or relations or lemmata: |
| 583 | s[i] = self.find_tags(s[i], **kwargs) |
| 584 | else: |
| 585 | s[i] = [[w] for w in s[i]] |
| 586 | # Chunker. |
| 587 | if chunks or relations: |
| 588 | s[i] = self.find_chunks(s[i], **kwargs) |
| 589 | # Labeler. |
| 590 | if relations: |
| 591 | s[i] = self.find_labels(s[i], **kwargs) |
| 592 | # Lemmatizer. |
| 593 | if lemmata: |
| 594 | s[i] = self.find_lemmata(s[i], **kwargs) |
| 595 | # Slash-formatted tagged string. |
| 596 | # With collapse=False (or split=True), returns raw list |
| 597 | # (this output is not usable by tree.Text). |
| 598 | if not kwargs.get("collapse", True) \ |
| 599 | or kwargs.get("split", False): |
| 600 | return s |
| 601 | # Construct TaggedString.format. |
| 602 | # (this output is usable by tree.Text). |
| 603 | format = ["word"] |
| 604 | if tags: |
| 605 | format.append("part-of-speech") |
| 606 | if chunks: |
| 607 | format.extend(("chunk", "preposition")) |
| 608 | if relations: |
| 609 | format.append("relation") |
| 610 | if lemmata: |
| 611 | format.append("lemma") |
| 612 | # Collapse raw list. |
| 613 | # Sentences are separated by newlines, tokens by spaces, tags by slashes. |
| 614 | # Slashes in words are encoded with &slash; |
| 615 | for i in range(len(s)): |
no test coverage detected