Class KimiScrapingAgent

backend/app/agents/kimi_agent.py:12–168

from app.services.scrapegraph import ScrapingService

class KimiScrapingAgent:
    def __init__(self):
        self.llm = get_llm()
        self.scraping_service = ScrapingService(settings.SCRAPEGRAPH_API_KEY)
        self.memory = ConversationBufferMemory(return_messages=True)
        self.agent_executor = self._create_agent()
    
    def _create_agent(self):
        """Create the LangChain agent with tools."""
        
        # Define tools
        tools = [
            Tool(
                name="add_url",
                description="Add a URL to the scraping pipeline. Use this when the user wants to add a website to scrape.",
                func=self._add_url_tool
            ),
            Tool(
                name="list_urls",
                description="List all URLs in the current pipeline.",
                func=self._list_urls_tool
            ),
            Tool(
                name="define_schema",
                description="Define or update the data extraction schema. Use when user wants to specify what data to extract.",
                func=self._define_schema_tool
            ),
            Tool(
                name="generate_code",
                description="Generate Python code for the scraping pipeline.",
                func=self._generate_code_tool
            ),
            Tool(
                name="validate_url",
                description="Check if a URL is valid and accessible.",
                func=self._validate_url_tool
            )
        ]
        
        # Create prompt
        prompt = ChatPromptTemplate.from_messages([
            ("system", """You are ScrapeCraft AI, an expert assistant for building web scraping pipelines using ScrapeGraphAI.

Your role is to help users:
1. Add and manage URLs they want to scrape
2. Define data schemas (what information to extract)
3. Generate Python code using the scrapegraph_py library
4. Execute scraping pipelines

Current pipeline state:
URLs: {urls}
Schema: {schema}

Be helpful, concise, and guide users through the scraping process. When users want to add URLs or define schemas, use the appropriate tools.
Always explain what you're doing and provide clear next steps."""),
            MessagesPlaceholder(variable_name="chat_history"),
            ("human", "{input}"),
            MessagesPlaceholder(variable_name="agent_scratchpad")