Domain-specific knowledge base.
| 44 | |
| 45 | |
| 46 | class DomainKnowledge(Base): |
| 47 | """Domain-specific knowledge base.""" |
| 48 | __tablename__ = "domain_knowledge" |
| 49 | |
| 50 | id = Column(String, primary_key=True) |
| 51 | domain = Column(String, nullable=False, unique=True, index=True) |
| 52 | |
| 53 | # Knowledge data |
| 54 | common_fields = Column(JSON) # Commonly extracted fields |
| 55 | field_mappings = Column(JSON) # Field name variations |
| 56 | extraction_rules = Column(JSON) # Domain-specific rules |
| 57 | |
| 58 | # Structure information |
| 59 | has_pagination = Column(Integer, default=0) # Boolean as int |
| 60 | has_infinite_scroll = Column(Integer, default=0) |
| 61 | requires_javascript = Column(Integer, default=0) |
| 62 | has_anti_scraping = Column(Integer, default=0) |
| 63 | |
| 64 | # Best practices |
| 65 | optimal_delay = Column(Float, default=1.0) # Seconds between requests |
| 66 | max_concurrent = Column(Integer, default=3) |
| 67 | retry_strategy = Column(JSON) |
| 68 | |
| 69 | # Statistics |
| 70 | total_scraped = Column(Integer, default=0) |
| 71 | avg_success_rate = Column(Float, default=0.0) |
| 72 | |
| 73 | # Timestamps |
| 74 | created_at = Column(DateTime, default=datetime.utcnow) |
| 75 | updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) |
| 76 | |
| 77 | |
| 78 | class PatternLearner: |
nothing calls this directly
no outgoing calls
no test coverage detected