Returns the arguments for Sentence.append() from a tagged token representation. The order in which token tags appear can be specified. The default order is (separated by slashes): - word, - part-of-speech, - (IOB-)chunk, -
(self, token, tags=[WORD, POS, CHUNK, PNP, REL, ANCHOR, LEMMA])
| 666 | self._do_conjunction() |
| 667 | |
| 668 | def parse_token(self, token, tags=[WORD, POS, CHUNK, PNP, REL, ANCHOR, LEMMA]): |
| 669 | """ Returns the arguments for Sentence.append() from a tagged token representation. |
| 670 | The order in which token tags appear can be specified. |
| 671 | The default order is (separated by slashes): |
| 672 | - word, |
| 673 | - part-of-speech, |
| 674 | - (IOB-)chunk, |
| 675 | - (IOB-)preposition, |
| 676 | - chunk(-relation)(-role), |
| 677 | - anchor, |
| 678 | - lemma. |
| 679 | Examples: |
| 680 | The/DT/B-NP/O/NP-SBJ-1/O/the |
| 681 | cats/NNS/I-NP/O/NP-SBJ-1/O/cat |
| 682 | clawed/VBD/B-VP/O/VP-1/A1/claw |
| 683 | at/IN/B-PP/B-PNP/PP/P1/at |
| 684 | the/DT/B-NP/I-PNP/NP/P1/the |
| 685 | sofa/NN/I-NP/I-PNP/NP/P1/sofa |
| 686 | ././O/O/O/O/. |
| 687 | Returns a (word, lemma, type, chunk, role, relation, preposition, anchor, iob, custom)-tuple, |
| 688 | which can be passed to Sentence.append(): Sentence.append(*Sentence.parse_token("cats/NNS/NP")) |
| 689 | The custom value is a dictionary of (tag, value)-items of unrecognized tags in the token. |
| 690 | """ |
| 691 | p = { WORD: "", |
| 692 | POS: None, |
| 693 | IOB: None, |
| 694 | CHUNK: None, |
| 695 | PNP: None, |
| 696 | REL: None, |
| 697 | ROLE: None, |
| 698 | ANCHOR: None, |
| 699 | LEMMA: None } |
| 700 | custom = [tag for tag in tags if tag not in p] # Custom tags. |
| 701 | p.update(dict.fromkeys(custom, None)) |
| 702 | # Split the slash-formatted token into separate tags in the given order. |
| 703 | # Decode &slash; characters (usually in words and lemmata). |
| 704 | # Assume None for missing tags (except the word itself, which defaults to an empty string). |
| 705 | token = token.split("/") |
| 706 | for i in range(min(len(token), len(tags))): |
| 707 | if token[i] != OUTSIDE \ |
| 708 | or tags[i] in (WORD, LEMMA): # "O is part of the alphabet" => "O" != OUTSIDE. |
| 709 | p[tags[i]] = decode_entities(token[i]) |
| 710 | # Split IOB-prefix from the chunk tag: |
| 711 | # B- marks the start of a new chunk, |
| 712 | # I- marks inside of a chunk. |
| 713 | if p[CHUNK] is not None: |
| 714 | x = p[CHUNK].split("-") |
| 715 | if len(x) == 2: p[CHUNK] = x[1]; p[IOB] = x[0] # B-NP |
| 716 | if len(x) == 1: p[CHUNK] = x[0] # NP |
| 717 | # Split the role from the relation: |
| 718 | # NP-SBJ-1 => relation id is 1 and role is SBJ, |
| 719 | # VP-1 => relation id is 1 with no role. |
| 720 | # Tokens may be tagged with multiple relations (e.g., NP-OBJ-1*NP-OBJ-3). |
| 721 | if p[REL] is not None: |
| 722 | ch, p[REL], p[ROLE] = self._parse_relation(p[REL]) |
| 723 | # Infer a missing chunk tag from the relation tag (e.g., NP-SBJ-1 => NP). |
| 724 | # For PP relation tags (e.g., PP-CLR-1), the first chunk is PP, the following chunks NP. |
| 725 | if ch == "PP" \ |
no test coverage detected