| 59 | return '' |
| 60 | |
| 61 | def extract_full_text(soup: BeautifulSoup) -> str: |
| 62 | # Remove unwanted tags: |
| 63 | for tag in soup.find_all(['script', 'style', 'meta', 'noscript']): |
| 64 | tag.extract() |
| 65 | |
| 66 | # Attempt to find the main document element based on common HTML structures. |
| 67 | # You may need to adjust the tag name and class name based on the specific HTML structure of the pages you're working with. |
| 68 | document_element = soup.find('div', {'class': 'document-content'}) |
| 69 | if document_element: |
| 70 | return document_element.get_text(' ', strip=True) # Use a space as the separator for text in different elements, and strip leading/trailing whitespace. |
| 71 | |
| 72 | # If the main document element wasn't found, fall back to extracting all text. |
| 73 | return soup.get_text(' ', strip=True) |
| 74 | |
| 75 | def normalize_whitespace(text: str) -> str: |
| 76 | return ' '.join(text.split()) |