Method _extract_year_from_content

onecite/pipeline/identifier.py:1044–1083

Extract publication year from content

(self, soup: BeautifulSoup, title: str = '')

        return None
    
    def _extract_year_from_content(self, soup: BeautifulSoup, title: str = '') -> Optional[int]:
        """Extract publication year from content"""
        try:
            # First try to find year in title
            if title:
                year_match = re.search(r'\b(19|20)\d{2}\b', title)
                if year_match:
                    return int(year_match.group())
            
            # Look for year in specific elements
            year_selectors = [
                '[class*="year"]',
                '[class*="date"]',
                '.publication-date',
                '.pub-date'
            ]
            
            for selector in year_selectors:
                elements = soup.select(selector)
                for elem in elements:
                    text = elem.get_text()
                    year_match = re.search(r'\b(19|20)\d{2}\b', text)
                    if year_match:
                        return int(year_match.group())
            
            # Try to find year in the first few paragraphs
            paragraphs = soup.find_all('p')[:5]
            for p in paragraphs:
                text = p.get_text()
                year_match = re.search(r'\b(19|20)\d{2}\b', text)
                if year_match:
                    year = int(year_match.group())
                    # Only accept reasonable years for academic papers
                    if 1950 <= year <= 2030:
                        return year
                        
        except Exception as e:
            self.logger.warning(f"Failed to extract year from content: {str(e)}")
        
        return None
    
    def _extract_from_pdf_content(self, content: bytes) -> Optional[Dict]:
        """Extract metadata from PDF content"""