Extract metadata from HTML content
(self, content: bytes)
| 908 | return None |
| 909 | |
| 910 | def _extract_from_html_content(self, content: bytes) -> Optional[Dict]: |
| 911 | """Extract metadata from HTML content""" |
| 912 | try: |
| 913 | soup = BeautifulSoup(content, 'html.parser') |
| 914 | metadata = {} |
| 915 | |
| 916 | # Look for academic metadata in meta tags |
| 917 | meta_mappings = { |
| 918 | 'title': ['citation_title', 'dc.title', 'og:title'], |
| 919 | 'author': ['citation_author', 'dc.creator', 'author'], |
| 920 | 'journal': ['citation_journal_title', 'dc.source', 'citation_conference_title'], |
| 921 | 'year': ['citation_publication_date', 'citation_date', 'dc.date'], |
| 922 | 'abstract': ['citation_abstract', 'dc.description', 'description'], |
| 923 | 'volume': ['citation_volume'], |
| 924 | 'pages': ['citation_firstpage', 'citation_lastpage'] |
| 925 | } |
| 926 | |
| 927 | authors = [] |
| 928 | for field, tag_names in meta_mappings.items(): |
| 929 | for tag_name in tag_names: |
| 930 | metas = soup.find_all('meta', attrs={'name': tag_name}) + \ |
| 931 | soup.find_all('meta', attrs={'property': tag_name}) |
| 932 | |
| 933 | for meta in metas: |
| 934 | if meta.get('content'): |
| 935 | content_value = meta['content'].strip() |
| 936 | if not content_value: |
| 937 | continue |
| 938 | |
| 939 | if field == 'author': |
| 940 | authors.append(content_value) |
| 941 | elif field == 'year': |
| 942 | year_match = re.search(r'\b(19|20)\d{2}\b', content_value) |
| 943 | if year_match: |
| 944 | metadata[field] = int(year_match.group()) |
| 945 | elif field == 'journal': |
| 946 | # Don't overwrite if already found |
| 947 | if field not in metadata: |
| 948 | metadata[field] = content_value |
| 949 | else: |
| 950 | metadata[field] = content_value |
| 951 | |
| 952 | # For non-author fields, break after finding first valid value |
| 953 | if field != 'author': |
| 954 | break |
| 955 | |
| 956 | # For non-author fields, break after finding value from any tag |
| 957 | if field != 'author' and field in metadata: |
| 958 | break |
| 959 | |
| 960 | # Process authors |
| 961 | if authors: |
| 962 | # Clean up author names and join them |
| 963 | cleaned_authors = [] |
| 964 | for author in authors: |
| 965 | # Remove extra whitespace and common prefixes |
| 966 | author = re.sub(r'^\s*(by\s+)?', '', author, flags=re.IGNORECASE).strip() |
| 967 | if author and len(author) > 2: |