| 56 | |
| 57 | |
| 58 | def basic_process(title, text): |
| 59 | title = html.unescape(title) |
| 60 | text = html.unescape(text) |
| 61 | text = text.strip() |
| 62 | |
| 63 | if "(disambiguation)" in title.lower(): |
| 64 | return None, None |
| 65 | if "(disambiguation page)" in title.lower(): |
| 66 | return None, None |
| 67 | # Take out List/Index/Outline pages (mostly links) |
| 68 | if re.match(r"(List of .+)|(Index of .+)|(Outline of .+)", title): |
| 69 | return None, None |
| 70 | if text.startswith("REDIRECT") or text.startswith("redirect"): |
| 71 | return None, None |
| 72 | if text.endswith(". References."): |
| 73 | text = text[: -len(" References.")].strip() |
| 74 | |
| 75 | text = re.sub("\{\{cite .*?\}\}", " ", text, flags=re.DOTALL) |
| 76 | text = text.replace(r"TABLETOREPLACE", " ") |
| 77 | text = text.replace(r"'''", " ") |
| 78 | text = text.replace(r"[[", " ") |
| 79 | text = text.replace(r"]]", " ") |
| 80 | text = text.replace(r"{{", " ") |
| 81 | text = text.replace(r"}}", " ") |
| 82 | text = text.replace("<br>", " ") |
| 83 | text = text.replace(""", '"') |
| 84 | text = text.replace("&", "&") |
| 85 | text = text.replace("& amp;", "&") |
| 86 | text = text.replace("nbsp;", " ") |
| 87 | text = text.replace("formatnum:", "") |
| 88 | |
| 89 | # text = re.sub('<poem.*?</poem>', ' ', text, flags=re.DOTALL) # might have useful information? |
| 90 | text = re.sub("<math.*?</math>", "", text, flags=re.DOTALL) |
| 91 | text = re.sub("<chem.*?</chem>", "", text, flags=re.DOTALL) |
| 92 | text = re.sub("<score.*?</score>", "", text, flags=re.DOTALL) |
| 93 | |
| 94 | # clean residual mess from xml dump that shouldn't have made its way here |
| 95 | text = re.sub("\| ?item[0-9]?_?style= ?.*? ", " ", text) |
| 96 | text = re.sub("\| ?col[0-9]?_?style= ?.*? ", " ", text) |
| 97 | text = re.sub("\| ?row[0-9]?_?style= ?.*? ", " ", text) |
| 98 | text = re.sub("\| ?style= ?.*? ", " ", text) |
| 99 | text = re.sub("\| ?bodystyle= ?.*? ", " ", text) |
| 100 | text = re.sub("\| ?frame_?style= ?.*? ", " ", text) |
| 101 | text = re.sub("\| ?data_?style= ?.*? ", " ", text) |
| 102 | text = re.sub("\| ?label_?style= ?.*? ", " ", text) |
| 103 | text = re.sub("\| ?headerstyle= ?.*? ", " ", text) |
| 104 | text = re.sub("\| ?list_?style= ?.*? ", " ", text) |
| 105 | text = re.sub("\| ?title_?style= ?.*? ", " ", text) |
| 106 | text = re.sub("\| ?ul_?style= ?.*? ", " ", text) |
| 107 | text = re.sub("\| ?li_?style= ?.*? ", " ", text) |
| 108 | text = re.sub("\| ?border-style= ?.*? ", " ", text) |
| 109 | text = re.sub('\|? ?style=".*?"', "", text) |
| 110 | text = re.sub('\|? ?rowspan=".*?"', "", text) |
| 111 | text = re.sub('\|? ?colspan=".*?"', "", text) |
| 112 | text = re.sub('\|? ?scope=".*?"', "", text) |
| 113 | text = re.sub('\|? ?align=".*?"', "", text) |
| 114 | text = re.sub('\|? ?valign=".*?"', "", text) |
| 115 | text = re.sub('\|? ?lang=".*?"', "", text) |