Asynchronously fetch a webpage's content.
(url: str, context)
| 21 | logger = logging.getLogger(__name__) |
| 22 | |
| 23 | async def fetch_page(url: str, context) -> Optional[str]: |
| 24 | """Asynchronously fetch a webpage's content.""" |
| 25 | page = await context.new_page() |
| 26 | try: |
| 27 | logger.info(f"Fetching {url}") |
| 28 | await page.goto(url) |
| 29 | await page.wait_for_load_state('networkidle') |
| 30 | content = await page.content() |
| 31 | logger.info(f"Successfully fetched {url}") |
| 32 | return content |
| 33 | except Exception as e: |
| 34 | logger.error(f"Error fetching {url}: {str(e)}") |
| 35 | return None |
| 36 | finally: |
| 37 | await page.close() |
| 38 | |
| 39 | def parse_html(html_content: Optional[str]) -> str: |
| 40 | """Parse HTML content and extract text with hyperlinks in markdown format.""" |
no outgoing calls