Extract DOI from URL page. Prioritize meta tags and avoid extracting DOIs from reference sections.
(self, url: str)
| 811 | return None |
| 812 | |
| 813 | def _extract_doi_from_url(self, url: str) -> Optional[str]: |
| 814 | """Extract DOI from URL page. Prioritize meta tags and avoid extracting DOIs from reference sections.""" |
| 815 | try: |
| 816 | response = requests.get(url, timeout=10, headers={'User-Agent': 'OneCite/1.0'}, stream=True) |
| 817 | content_len = response.headers.get('content-length') |
| 818 | if content_len and int(content_len) > 5 * 1024 * 1024: |
| 819 | self.logger.warning(f"Skipping URL {url}: response too large ({content_len} bytes)") |
| 820 | return None |
| 821 | content = response.raw.read(5 * 1024 * 1024) |
| 822 | soup = BeautifulSoup(content, 'html.parser') |
| 823 | |
| 824 | # 1. Look for DOI in meta tags (most reliable) |
| 825 | doi_meta = soup.find('meta', attrs={'name': 'citation_doi'}) or \ |
| 826 | soup.find('meta', attrs={'name': 'dc.identifier'}) or \ |
| 827 | soup.find('meta', attrs={'property': 'citation_doi'}) |
| 828 | |
| 829 | if doi_meta and 'content' in doi_meta.attrs: |
| 830 | doi = doi_meta['content'] |
| 831 | if self._validate_doi(doi): |
| 832 | self.logger.info(f"Found DOI in meta tags: {doi}") |
| 833 | return doi |
| 834 | |
| 835 | # 2. Check schema.org structured data |
| 836 | script_tags = soup.find_all('script', type='application/ld+json') |
| 837 | for script in script_tags: |
| 838 | try: |
| 839 | import json |
| 840 | data = json.loads(script.string) |
| 841 | if isinstance(data, dict) and 'identifier' in data: |
| 842 | identifier = data['identifier'] |
| 843 | if isinstance(identifier, str) and self._validate_doi(identifier): |
| 844 | self.logger.info(f"Found DOI in structured data: {identifier}") |
| 845 | return identifier |
| 846 | except Exception: |
| 847 | pass |
| 848 | |
| 849 | # 3. Limited search in main content only (exclude reference sections) |
| 850 | # Remove known reference/citation sections to avoid false matches |
| 851 | for ref_section in soup.find_all(['div', 'section', 'article'], |
| 852 | attrs={'class': re.compile(r'(reference|citation|bibliography)', re.IGNORECASE)}): |
| 853 | ref_section.decompose() |
| 854 | for ref_section in soup.find_all(['div', 'section', 'article'], |
| 855 | id=re.compile(r'(reference|citation|bibliography)', re.IGNORECASE)): |
| 856 | ref_section.decompose() |
| 857 | |
| 858 | # Also remove common reference list elements |
| 859 | for ref_list in soup.find_all(['ul', 'ol'], |
| 860 | attrs={'class': re.compile(r'(reference|citation)', re.IGNORECASE)}): |
| 861 | ref_list.decompose() |
| 862 | |
| 863 | # Search in remaining main content area |
| 864 | main_content = soup.find('main') or soup.find('article') or soup.find('body') |
| 865 | if main_content: |
| 866 | # Look for DOI patterns, but be cautious |
| 867 | content_text = main_content.get_text() |
| 868 | doi_match = re.search(r'(?:doi:?\s*|https?://doi\.org/)?(10\.\d{4,}/[^\s"<>,}]+)', content_text, re.IGNORECASE) |
| 869 | if doi_match: |
| 870 | doi = doi_match.group(1) if doi_match.lastindex >= 1 else doi_match.group(0) |