Extract metadata from PDF or HTML page
(self, url: str)
| 886 | return None |
| 887 | |
| 888 | def _extract_metadata_from_url(self, url: str) -> Optional[Dict]: |
| 889 | """Extract metadata from PDF or HTML page""" |
| 890 | try: |
| 891 | response = requests.get(url, timeout=15, headers={'User-Agent': 'OneCite/1.0'}, stream=True) |
| 892 | content_len = response.headers.get('content-length') |
| 893 | if content_len and int(content_len) > 5 * 1024 * 1024: |
| 894 | self.logger.warning(f"Skipping URL {url}: response too large ({content_len} bytes)") |
| 895 | return None |
| 896 | response._content = response.raw.read(5 * 1024 * 1024) |
| 897 | response.raise_for_status() |
| 898 | |
| 899 | # Check if it's a PDF |
| 900 | content_type = response.headers.get('content-type', '').lower() |
| 901 | if 'pdf' in content_type or url.lower().endswith('.pdf'): |
| 902 | return self._extract_from_pdf_content(response.content) |
| 903 | else: |
| 904 | return self._extract_from_html_content(response.content) |
| 905 | |
| 906 | except Exception as e: |
| 907 | self.logger.warning(f"Failed to extract metadata from URL {url}: {str(e)}") |
| 908 | return None |
| 909 | |
| 910 | def _extract_from_html_content(self, content: bytes) -> Optional[Dict]: |
| 911 | """Extract metadata from HTML content""" |