The input is a list of [token, tag]-items. The output is a list of [token, tag, chunk]-items: The/DT nice/JJ fish/NN is/VBZ dead/JJ ./. => The/DT/B-NP nice/JJ/I-NP fish/NN/I-NP is/VBZ/B-VP dead/JJ/B-ADJP ././O
(tagged, language="en")
| 911 | CHUNKS[1].insert(1, CHUNKS[1].pop(3)) |
| 912 | |
| 913 | def find_chunks(tagged, language="en"): |
| 914 | """ The input is a list of [token, tag]-items. |
| 915 | The output is a list of [token, tag, chunk]-items: |
| 916 | The/DT nice/JJ fish/NN is/VBZ dead/JJ ./. => |
| 917 | The/DT/B-NP nice/JJ/I-NP fish/NN/I-NP is/VBZ/B-VP dead/JJ/B-ADJP ././O |
| 918 | """ |
| 919 | chunked = [x for x in tagged] |
| 920 | tags = "".join("%s%s" % (tag, SEPARATOR) for token, tag in tagged) |
| 921 | # Use Germanic or Romance chunking rules according to given language. |
| 922 | for tag, rule in CHUNKS[int(language in ("ca", "es", "pt", "fr", "it", "pt", "ro"))]: |
| 923 | for m in rule.finditer(tags): |
| 924 | # Find the start of chunks inside the tags-string. |
| 925 | # Number of preceding separators = number of preceding tokens. |
| 926 | i = m.start() |
| 927 | j = tags[:i].count(SEPARATOR) |
| 928 | n = m.group(0).count(SEPARATOR) |
| 929 | for k in range(j, j+n): |
| 930 | if len(chunked[k]) == 3: |
| 931 | continue |
| 932 | if len(chunked[k]) < 3: |
| 933 | # A conjunction can not be start of a chunk. |
| 934 | if k == j and chunked[k][1] in ("CC", "CJ", "KON", "Conj(neven)"): |
| 935 | j += 1 |
| 936 | # Mark first token in chunk with B-. |
| 937 | elif k == j: |
| 938 | chunked[k].append("B-"+tag) |
| 939 | # Mark other tokens in chunk with I-. |
| 940 | else: |
| 941 | chunked[k].append("I-"+tag) |
| 942 | # Mark chinks (tokens outside of a chunk) with O-. |
| 943 | for chink in filter(lambda x: len(x) < 3, chunked): |
| 944 | chink.append("O") |
| 945 | # Post-processing corrections. |
| 946 | for i, (word, tag, chunk) in enumerate(chunked): |
| 947 | if tag.startswith("RB") and chunk == "B-NP": |
| 948 | # "Very nice work" (NP) <=> "Perhaps" (ADVP) + "you" (NP). |
| 949 | if i < len(chunked)-1 and not chunked[i+1][1].startswith("JJ"): |
| 950 | chunked[i+0][2] = "B-ADVP" |
| 951 | chunked[i+1][2] = "B-NP" |
| 952 | return chunked |
| 953 | |
| 954 | def find_prepositions(chunked): |
| 955 | """ The input is a list of [token, tag, chunk]-items. |