(
self,
crawler_strategy: Optional[AsyncCrawlerStrategy] = None,
always_by_pass_cache: bool = False,
base_directory: str = str(Path.home()),
**kwargs,
)
| 20 | |
| 21 | class AsyncWebCrawler: |
| 22 | def __init__( |
| 23 | self, |
| 24 | crawler_strategy: Optional[AsyncCrawlerStrategy] = None, |
| 25 | always_by_pass_cache: bool = False, |
| 26 | base_directory: str = str(Path.home()), |
| 27 | **kwargs, |
| 28 | ): |
| 29 | self.crawler_strategy = crawler_strategy or AsyncPlaywrightCrawlerStrategy( |
| 30 | **kwargs |
| 31 | ) |
| 32 | self.always_by_pass_cache = always_by_pass_cache |
| 33 | # self.crawl4ai_folder = os.path.join(Path.home(), ".crawl4ai") |
| 34 | self.crawl4ai_folder = os.path.join(base_directory, ".crawl4ai") |
| 35 | os.makedirs(self.crawl4ai_folder, exist_ok=True) |
| 36 | os.makedirs(f"{self.crawl4ai_folder}/cache", exist_ok=True) |
| 37 | self.ready = False |
| 38 | self.verbose = kwargs.get("verbose", False) |
| 39 | |
| 40 | async def __aenter__(self): |
| 41 | await self.crawler_strategy.__aenter__() |
nothing calls this directly
no test coverage detected