LangGraph-based agent for web scraping pipelines.
| 92 | |
| 93 | |
| 94 | class ScrapeCraftAgent: |
| 95 | """LangGraph-based agent for web scraping pipelines.""" |
| 96 | |
| 97 | def __init__(self): |
| 98 | self.llm = get_llm() |
| 99 | self.scraping_service = ScrapingService(settings.SCRAPEGRAPH_API_KEY) |
| 100 | self.memory = MemorySaver() |
| 101 | self.graph = self._build_graph() |
| 102 | |
| 103 | def _build_graph(self) -> StateGraph: |
| 104 | """Build the LangGraph state machine.""" |
| 105 | workflow = StateGraph(AgentState) |
| 106 | |
| 107 | # Add nodes |
| 108 | workflow.add_node("analyze_request", self.analyze_request) |
| 109 | workflow.add_node("collect_urls", self.collect_urls) |
| 110 | workflow.add_node("validate_urls", self.validate_urls) |
| 111 | workflow.add_node("define_schema", self.define_schema) |
| 112 | workflow.add_node("validate_schema", self.validate_schema) |
| 113 | workflow.add_node("generate_code", self.generate_code) |
| 114 | workflow.add_node("await_approval", self.await_approval) |
| 115 | workflow.add_node("execute_pipeline", self.execute_pipeline) |
| 116 | workflow.add_node("handle_error", self.handle_error) |
| 117 | |
| 118 | # Set entry point |
| 119 | workflow.set_entry_point("analyze_request") |
| 120 | |
| 121 | # Add conditional edges |
| 122 | workflow.add_conditional_edges( |
| 123 | "analyze_request", |
| 124 | self.route_after_analysis, |
| 125 | { |
| 126 | "collect_urls": "collect_urls", |
| 127 | "validate_urls": "validate_urls", |
| 128 | "define_schema": "define_schema", |
| 129 | "generate_code": "generate_code", |
| 130 | "error": "handle_error" |
| 131 | } |
| 132 | ) |
| 133 | |
| 134 | workflow.add_conditional_edges( |
| 135 | "collect_urls", |
| 136 | lambda state: "validate_urls" if state["urls"] else "handle_error" |
| 137 | ) |
| 138 | |
| 139 | workflow.add_conditional_edges( |
| 140 | "validate_urls", |
| 141 | lambda state: "await_approval" if state["requires_approval"] else "define_schema" |
| 142 | ) |
| 143 | |
| 144 | workflow.add_conditional_edges( |
| 145 | "await_approval", |
| 146 | self.route_after_approval, |
| 147 | { |
| 148 | "continue": "define_schema", |
| 149 | "reject": "collect_urls", |
| 150 | "timeout": "handle_error" |
| 151 | } |