Execute the generated scraping code.
(
request: ExecuteRequest,
connection_manager: ConnectionManager = Depends(lambda: connection_manager)
)
| 33 | |
| 34 | @router.post("/execute", response_model=ExecuteResponse) |
| 35 | async def execute_pipeline( |
| 36 | request: ExecuteRequest, |
| 37 | connection_manager: ConnectionManager = Depends(lambda: connection_manager) |
| 38 | ): |
| 39 | """Execute the generated scraping code.""" |
| 40 | start_time = asyncio.get_event_loop().time() |
| 41 | results = [] |
| 42 | errors = [] |
| 43 | |
| 44 | try: |
| 45 | # Use provided API key or fallback to settings |
| 46 | api_key = request.api_key or settings.SCRAPEGRAPH_API_KEY |
| 47 | if not api_key: |
| 48 | raise HTTPException(status_code=400, detail="SCRAPEGRAPH_API_KEY not configured") |
| 49 | |
| 50 | # Stream start of execution |
| 51 | await connection_manager.stream_execution_updates( |
| 52 | pipeline_id=request.pipeline_id, |
| 53 | url="all", |
| 54 | status="starting", |
| 55 | data={"message": "Starting pipeline execution..."} |
| 56 | ) |
| 57 | |
| 58 | # Create temporary Python file with the code |
| 59 | with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f: |
| 60 | # Inject the API key and URLs into the code |
| 61 | # Replace API key placeholder and ensure URLS is set |
| 62 | modified_code = request.code.replace( |
| 63 | '"your-api-key-here"', |
| 64 | f'"{api_key}"' |
| 65 | ).replace( |
| 66 | 'API_KEY = "your-api-key-here"', |
| 67 | f'API_KEY = "{api_key}"' |
| 68 | ) |
| 69 | |
| 70 | # If URLS is not defined in the code, add it |
| 71 | if 'URLS = ' not in modified_code: |
| 72 | # Add URLS definition before the main block |
| 73 | modified_code = f'URLS = {json.dumps(request.urls)}\n' + modified_code |
| 74 | |
| 75 | # Ensure API_KEY is defined if not present |
| 76 | if 'API_KEY = ' not in modified_code: |
| 77 | modified_code = f'API_KEY = "{api_key}"\n' + modified_code |
| 78 | |
| 79 | # Extract function name from code |
| 80 | func_match = re.search(r'async def (\w+)\(', modified_code) |
| 81 | func_name = func_match.group(1) if func_match else 'scrape_pipeline' |
| 82 | |
| 83 | # Add result capture logic |
| 84 | capture_code = f""" |
| 85 | import json |
| 86 | import sys |
| 87 | |
| 88 | # Original code |
| 89 | {modified_code} |
| 90 | |
| 91 | # Capture and output results |
| 92 | if __name__ == "__main__": |
nothing calls this directly
no test coverage detected