(url: str)
| 27 | |
| 28 | |
| 29 | def read_html(url: str) -> str: |
| 30 | normalized = url.strip().strip("\"'") |
| 31 | if not normalized: |
| 32 | return "ReadHTML error: empty URL." |
| 33 | |
| 34 | parsed = urlparse(normalized) |
| 35 | if parsed.scheme not in {"http", "https"}: |
| 36 | return f"ReadHTML error: unsupported URL scheme `{parsed.scheme or 'unknown'}`." |
| 37 | |
| 38 | request = Request(normalized, headers={"User-Agent": "AutoPT/1.0"}) |
| 39 | try: |
| 40 | with urlopen(request, timeout=DEFAULT_READ_TIMEOUT_SECONDS) as response: # nosec B310 - URL comes from explicit tool usage |
| 41 | raw_html = response.read(DEFAULT_MAX_READ_BYTES + 1) |
| 42 | except HTTPError as exc: |
| 43 | return f"ReadHTML error: HTTP {exc.code} for {normalized}" |
| 44 | except (URLError, TimeoutError, SocketTimeout, ValueError) as exc: |
| 45 | return f"ReadHTML error: {exc}" |
| 46 | except Exception as exc: # pragma: no cover - defensive fallback |
| 47 | return f"ReadHTML error: unexpected failure: {exc}" |
| 48 | |
| 49 | truncated = len(raw_html) > DEFAULT_MAX_READ_BYTES |
| 50 | html = raw_html[:DEFAULT_MAX_READ_BYTES].decode("utf-8", "ignore") |
| 51 | parser = _BodyTextParser() |
| 52 | parser.feed(html) |
| 53 | text = parser.get_text() or html |
| 54 | if len(text) > DEFAULT_MAX_TEXT_CHARS: |
| 55 | text = text[:DEFAULT_MAX_TEXT_CHARS].rstrip() + "\n...[truncated]" |
| 56 | elif truncated: |
| 57 | text = text.rstrip() + "\n...[truncated]" |
| 58 | return text |
nothing calls this directly
no test coverage detected