Function scrape_urls

backend/app/api/workflow_v2.py:83–119

Scrape multiple URLs with a given prompt. Request body: { "pipeline_id": "string", "urls": ["url1", "url2"], "extraction_prompt": "What data to extract" }

(
    request: Dict[str, Any],
    connection_manager: ConnectionManager = Depends(lambda: ConnectionManager()),
    current_user: User = Depends(get_current_user)
)

@router.post("/scrape")
async def scrape_urls(
    request: Dict[str, Any],
    connection_manager: ConnectionManager = Depends(lambda: ConnectionManager()),
    current_user: User = Depends(get_current_user)
) -> Dict[str, Any]:
    """
    Scrape multiple URLs with a given prompt.
    
    Request body:
    {
        "pipeline_id": "string",
        "urls": ["url1", "url2"],
        "extraction_prompt": "What data to extract"
    }
    """
    pipeline_id = request.get("pipeline_id")
    urls = request.get("urls", [])
    extraction_prompt = request.get("extraction_prompt", "Extract all relevant data")
    
    if not pipeline_id or not urls:
        raise HTTPException(status_code=400, detail="pipeline_id and urls are required")
    
    use_tools = os.getenv("USE_TOOLS_AGENT", "false").lower() == "true"
    manager = get_enhanced_workflow_manager(connection_manager, use_tools)
    
    try:
        results = await manager.scrape_urls(pipeline_id, urls, extraction_prompt)
        
        return {
            "success": True,
            "urls_scraped": len(urls),
            "results": results,
            "agent_mode": "tools-based" if use_tools else "state-based"
        }
        
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))


@router.get("/tools")