Convert a webpage into clean one-line text blocks.
(html_text: str)
| 1042 | |
| 1043 | |
| 1044 | def _html_to_text_lines(html_text: str) -> List[str]: |
| 1045 | """Convert a webpage into clean one-line text blocks.""" |
| 1046 | without_scripts = re.sub(r"(?is)<(script|style|noscript)[^>]*>.*?</\1>", " ", html_text or "") |
| 1047 | block_marked = re.sub( |
| 1048 | r"(?i)</?(?:br|p|div|li|ul|ol|h[1-6]|section|article|header|footer|tr|td|th|table)>", |
| 1049 | "\n", |
| 1050 | without_scripts, |
| 1051 | ) |
| 1052 | plain_text = html.unescape(re.sub(r"(?s)<[^>]+>", " ", block_marked)) |
| 1053 | return [ |
| 1054 | line |
| 1055 | for line in (_collapse_whitespace(item) for item in plain_text.splitlines()) |
| 1056 | if line and len(line) <= 240 |
| 1057 | ] |
| 1058 | |
| 1059 | |
| 1060 | def _classify_homepage_section(line: str) -> str: |
no test coverage detected