Model for learned scraping patterns.
| 16 | |
| 17 | |
| 18 | class ScrapingPattern(Base): |
| 19 | """Model for learned scraping patterns.""" |
| 20 | __tablename__ = "scraping_patterns" |
| 21 | |
| 22 | id = Column(String, primary_key=True) |
| 23 | domain = Column(String, nullable=False, index=True) |
| 24 | data_type = Column(String, nullable=False, index=True) |
| 25 | |
| 26 | # Pattern details |
| 27 | selectors_used = Column(JSON) # CSS selectors or extraction patterns |
| 28 | schema_pattern = Column(JSON) # Common schema fields |
| 29 | url_patterns = Column(JSON) # URL structure patterns |
| 30 | |
| 31 | # Performance metrics |
| 32 | success_rate = Column(Float, default=0.0) |
| 33 | avg_extraction_time = Column(Float, default=0.0) |
| 34 | total_executions = Column(Integer, default=0) |
| 35 | |
| 36 | # Optimization data |
| 37 | optimizations = Column(JSON) # Successful optimizations applied |
| 38 | common_errors = Column(JSON) # Common errors and solutions |
| 39 | |
| 40 | # Timestamps (renamed from Metadata to avoid conflicts) |
| 41 | created_at = Column(DateTime, default=datetime.utcnow) |
| 42 | updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) |
| 43 | last_seen = Column(DateTime, default=datetime.utcnow) |
| 44 | |
| 45 | |
| 46 | class DomainKnowledge(Base): |
nothing calls this directly
no outgoing calls
no test coverage detected