(crawler)
| 263 | print_result(result= result) |
| 264 | |
| 265 | def using_crawler_hooks_dleay_example(crawler): |
| 266 | def delay(driver): |
| 267 | print("Delaying for 5 seconds...") |
| 268 | time.sleep(5) |
| 269 | print("Resuming...") |
| 270 | |
| 271 | def create_crawler(): |
| 272 | crawler_strategy = LocalSeleniumCrawlerStrategy(verbose=True) |
| 273 | crawler_strategy.set_hook('after_get_url', delay) |
| 274 | crawler = WebCrawler(verbose=True, crawler_strategy=crawler_strategy) |
| 275 | crawler.warmup() |
| 276 | return crawler |
| 277 | |
| 278 | cprint("\n🔗 [bold cyan]Using Crawler Hooks: Let's add a delay after fetching the url to make sure entire page is fetched.[/bold cyan]") |
| 279 | crawler = create_crawler() |
| 280 | result = crawler.run(url="https://google.com", bypass_cache=True) |
| 281 | |
| 282 | cprint("[LOG] 📦 [bold yellow]Crawler Hooks result:[/bold yellow]") |
| 283 | print_result(result) |
| 284 | |
| 285 | |
| 286 |
nothing calls this directly
no test coverage detected
searching dependent graphs…