(text, dataset)
| 29 | |
| 30 | |
| 31 | def fix_tokenization(text, dataset): |
| 32 | if dataset == 'cnn_dm_org': |
| 33 | return text |
| 34 | if dataset == 'gigaword': |
| 35 | text = text.replace('[UNK]', 'UNK') |
| 36 | return text |
| 37 | input_tokens = text.split() |
| 38 | output_tokens = [] |
| 39 | has_left_quote = False |
| 40 | has_left_single_quote = False |
| 41 | |
| 42 | i = 0 |
| 43 | prev_dash = False |
| 44 | while i < len(input_tokens): |
| 45 | tok = input_tokens[i] |
| 46 | flag_prev_dash = False |
| 47 | if tok == "\"": |
| 48 | if has_left_quote: |
| 49 | output_tokens.append("''") |
| 50 | else: |
| 51 | output_tokens.append("``") |
| 52 | has_left_quote = not has_left_quote |
| 53 | i += 1 |
| 54 | elif tok == "'" and len(output_tokens) > 0 and output_tokens[-1].endswith("n") and i < len(input_tokens) - 1 and \ |
| 55 | input_tokens[i + 1] == "t": |
| 56 | output_tokens[-1] = output_tokens[-1][:-1] |
| 57 | output_tokens.append("n't") |
| 58 | i += 2 |
| 59 | elif tok == "'" and i < len(input_tokens) - 1 and input_tokens[i + 1] in ("s", "d", "ll"): |
| 60 | output_tokens.append("'" + input_tokens[i + 1]) |
| 61 | i += 2 |
| 62 | elif tok == "'": |
| 63 | if has_left_single_quote: |
| 64 | output_tokens.append("'") |
| 65 | else: |
| 66 | output_tokens.append("`") |
| 67 | has_left_single_quote = not has_left_single_quote |
| 68 | i += 1 |
| 69 | elif tok == "." and i < len(input_tokens) - 2 and input_tokens[i + 1] == "." and input_tokens[i + 2] == ".": |
| 70 | output_tokens.append("...") |
| 71 | i += 3 |
| 72 | elif tok == "," and len(output_tokens) > 0 and _is_digit(output_tokens[-1]) and i < len( |
| 73 | input_tokens) - 1 and _is_digit(input_tokens[i + 1]): |
| 74 | # $ 3 , 000 -> $ 3,000 |
| 75 | output_tokens[-1] += ',' + input_tokens[i + 1] |
| 76 | i += 2 |
| 77 | elif tok == "." and len(output_tokens) > 0 and output_tokens[-1].isdigit() and i < len(input_tokens) - 1 and \ |
| 78 | input_tokens[i + 1].isdigit(): |
| 79 | # 3 . 03 -> $ 3.03 |
| 80 | output_tokens[-1] += '.' + input_tokens[i + 1] |
| 81 | i += 2 |
| 82 | elif tok == "." and len(output_tokens) > 0 and len(output_tokens[-1]) == 1 and output_tokens[ |
| 83 | -1].isalpha() and i < len(input_tokens) - 2 and len(input_tokens[i + 1]) == 1 and input_tokens[ |
| 84 | i + 1].isalpha() and input_tokens[i + 2] == '.': |
| 85 | # U . N . -> U.N. |
| 86 | k = i + 3 |
| 87 | while k + 2 < len(input_tokens): |
| 88 | if len(input_tokens[k + 1]) == 1 and input_tokens[k + 1].isalpha() and input_tokens[k + 2] == '.': |
no test coverage detected