| 42 | |
| 43 | |
| 44 | def basic_process(title, text): |
| 45 | title = html.unescape(title) |
| 46 | text = html.unescape(text) |
| 47 | text = text.strip() |
| 48 | |
| 49 | if "(disambiguation)" in title.lower(): |
| 50 | return None, None |
| 51 | if "(disambiguation page)" in title.lower(): |
| 52 | return None, None |
| 53 | # Take out List/Index/Outline pages (mostly links) |
| 54 | if re.match(r"(List of .+)|(Index of .+)|(Outline of .+)", title): |
| 55 | return None, None |
| 56 | if text.startswith("REDIRECT") or text.startswith("redirect"): |
| 57 | return None, None |
| 58 | if text.endswith(". References."): |
| 59 | text = text[: -len(" References.")].strip() |
| 60 | |
| 61 | text = re.sub("\{\{cite .*?\}\}", " ", text, flags=re.DOTALL) |
| 62 | text = text.replace(r"TABLETOREPLACE", " ") |
| 63 | text = text.replace(r"'''", " ") |
| 64 | text = text.replace(r"[[", " ") |
| 65 | text = text.replace(r"]]", " ") |
| 66 | text = text.replace(r"{{", " ") |
| 67 | text = text.replace(r"}}", " ") |
| 68 | text = text.replace("<br>", " ") |
| 69 | text = text.replace(""", '"') |
| 70 | text = text.replace("&", "&") |
| 71 | text = text.replace("& amp;", "&") |
| 72 | text = text.replace("nbsp;", " ") |
| 73 | text = text.replace("formatnum:", "") |
| 74 | |
| 75 | # text = re.sub('<poem.*?</poem>', ' ', text, flags=re.DOTALL) # might have useful information? |
| 76 | text = re.sub("<math.*?</math>", "", text, flags=re.DOTALL) |
| 77 | text = re.sub("<chem.*?</chem>", "", text, flags=re.DOTALL) |
| 78 | text = re.sub("<score.*?</score>", "", text, flags=re.DOTALL) |
| 79 | |
| 80 | # clean residual mess from xml dump that shouldn't have made its way here |
| 81 | text = re.sub("\| ?item[0-9]?_?style= ?.*? ", " ", text) |
| 82 | text = re.sub("\| ?col[0-9]?_?style= ?.*? ", " ", text) |
| 83 | text = re.sub("\| ?row[0-9]?_?style= ?.*? ", " ", text) |
| 84 | text = re.sub("\| ?style= ?.*? ", " ", text) |
| 85 | text = re.sub("\| ?bodystyle= ?.*? ", " ", text) |
| 86 | text = re.sub("\| ?frame_?style= ?.*? ", " ", text) |
| 87 | text = re.sub("\| ?data_?style= ?.*? ", " ", text) |
| 88 | text = re.sub("\| ?label_?style= ?.*? ", " ", text) |
| 89 | text = re.sub("\| ?headerstyle= ?.*? ", " ", text) |
| 90 | text = re.sub("\| ?list_?style= ?.*? ", " ", text) |
| 91 | text = re.sub("\| ?title_?style= ?.*? ", " ", text) |
| 92 | text = re.sub("\| ?ul_?style= ?.*? ", " ", text) |
| 93 | text = re.sub("\| ?li_?style= ?.*? ", " ", text) |
| 94 | text = re.sub("\| ?border-style= ?.*? ", " ", text) |
| 95 | text = re.sub('\|? ?style=".*?"', "", text) |
| 96 | text = re.sub('\|? ?rowspan=".*?"', "", text) |
| 97 | text = re.sub('\|? ?colspan=".*?"', "", text) |
| 98 | text = re.sub('\|? ?scope=".*?"', "", text) |
| 99 | text = re.sub('\|? ?align=".*?"', "", text) |
| 100 | text = re.sub('\|? ?valign=".*?"', "", text) |
| 101 | text = re.sub('\|? ?lang=".*?"', "", text) |