Execute a scraping job asynchronously.
(
request: ScrapingRequest,
background_tasks: BackgroundTasks
)
| 30 | |
| 31 | @router.post("/execute", response_model=ScrapingResponse) |
| 32 | async def execute_scraping( |
| 33 | request: ScrapingRequest, |
| 34 | background_tasks: BackgroundTasks |
| 35 | ): |
| 36 | """Execute a scraping job asynchronously.""" |
| 37 | task_id = str(uuid.uuid4()) |
| 38 | |
| 39 | # Add to background tasks |
| 40 | background_tasks.add_task( |
| 41 | run_scraping_task, |
| 42 | task_id, |
| 43 | request.urls, |
| 44 | request.prompt, |
| 45 | request.schema |
| 46 | ) |
| 47 | |
| 48 | scraping_tasks[task_id] = { |
| 49 | "status": "pending", |
| 50 | "results": None, |
| 51 | "error": None |
| 52 | } |
| 53 | |
| 54 | return ScrapingResponse( |
| 55 | task_id=task_id, |
| 56 | status="pending", |
| 57 | message="Scraping task started" |
| 58 | ) |
| 59 | |
| 60 | async def run_scraping_task( |
| 61 | task_id: str, |
nothing calls this directly
no test coverage detected