(
self,
# db_path: str = None,
crawler_strategy: CrawlerStrategy = None,
always_by_pass_cache: bool = False,
verbose: bool = False,
)
| 15 | |
| 16 | class WebCrawler: |
| 17 | def __init__( |
| 18 | self, |
| 19 | # db_path: str = None, |
| 20 | crawler_strategy: CrawlerStrategy = None, |
| 21 | always_by_pass_cache: bool = False, |
| 22 | verbose: bool = False, |
| 23 | ): |
| 24 | # self.db_path = db_path |
| 25 | self.crawler_strategy = crawler_strategy or LocalSeleniumCrawlerStrategy(verbose=verbose) |
| 26 | self.always_by_pass_cache = always_by_pass_cache |
| 27 | |
| 28 | # Create the .crawl4ai folder in the user's home directory if it doesn't exist |
| 29 | self.crawl4ai_folder = os.path.join(Path.home(), ".crawl4ai") |
| 30 | os.makedirs(self.crawl4ai_folder, exist_ok=True) |
| 31 | os.makedirs(f"{self.crawl4ai_folder}/cache", exist_ok=True) |
| 32 | |
| 33 | # If db_path is not provided, use the default path |
| 34 | # if not db_path: |
| 35 | # self.db_path = f"{self.crawl4ai_folder}/crawl4ai.db" |
| 36 | |
| 37 | # flush_db() |
| 38 | init_db() |
| 39 | |
| 40 | self.ready = False |
| 41 | |
| 42 | def warmup(self): |
| 43 | print("[LOG] 🌤️ Warming up the WebCrawler") |
nothing calls this directly
no test coverage detected