cleaning wikitext dataset
(string)
| 51 | |
| 52 | |
| 53 | def clean_wikitext(string): |
| 54 | """cleaning wikitext dataset""" |
| 55 | # contractions |
| 56 | string = string.replace("s '", "s'") |
| 57 | string = re.sub(r"/' [0-9]/", r"/'[0-9]/", string) |
| 58 | # number separators |
| 59 | string = string.replace(" @-@ ", "-") |
| 60 | string = string.replace(" @,@ ", ",") |
| 61 | string = string.replace(" @.@ ", ".") |
| 62 | # punctuation |
| 63 | string = string.replace(" : ", ": ") |
| 64 | string = string.replace(" ; ", "; ") |
| 65 | string = string.replace(" . ", ". ") |
| 66 | string = string.replace(" ! ", "! ") |
| 67 | string = string.replace(" ? ", "? ") |
| 68 | string = string.replace(" , ", ", ") |
| 69 | # double brackets |
| 70 | string = re.sub(r"\(\s*([^\)]*?)\s*\)", r"(\1)", string) |
| 71 | string = re.sub(r"\[\s*([^\]]*?)\s*\]", r"[\1]", string) |
| 72 | string = re.sub(r"{\s*([^}]*?)\s*}", r"{\1}", string) |
| 73 | string = re.sub(r"\"\s*([^\"]*?)\s*\"", r'"\1"', string) |
| 74 | string = re.sub(r"'\s*([^']*?)\s*'", r"'\1'", string) |
| 75 | # miscellaneous |
| 76 | string = string.replace("= = = =", "====") |
| 77 | string = string.replace("= = =", "===") |
| 78 | string = string.replace("= =", "==") |
| 79 | string = string.replace(" " + chr(176) + " ", chr(176)) |
| 80 | string = string.replace(" \n", "\n") |
| 81 | string = string.replace("\n ", "\n") |
| 82 | string = string.replace(" N ", " 1 ") |
| 83 | string = string.replace(" 's", "'s") |
| 84 | return string |
| 85 | |
| 86 | |
| 87 | def tokenize_openwebtext(tokenizer, iterator, seq_length, eot): |