(self, shared)
| 8 | """Batch processes multiple URLs simultaneously to extract clean text content AND discover all links from those pages""" |
| 9 | |
| 10 | def prep(self, shared): |
| 11 | # The calling application is responsible for populating `urls_to_process`. |
| 12 | # This node just consumes the list. |
| 13 | urls_to_crawl = [] |
| 14 | for url_idx in shared.get("urls_to_process", []): |
| 15 | if url_idx < len(shared.get("all_discovered_urls", [])): |
| 16 | urls_to_crawl.append((url_idx, shared["all_discovered_urls"][url_idx])) |
| 17 | |
| 18 | return urls_to_crawl |
| 19 | |
| 20 | def exec(self, url_data): |
| 21 | """Process a single URL to extract content and links""" |
nothing calls this directly
no outgoing calls
no test coverage detected