| 15 | |
| 16 | |
| 17 | def get_web_documents(config: WebLoaderConfig): |
| 18 | from llama_index.readers.web import WholeSiteReader |
| 19 | from selenium import webdriver |
| 20 | from selenium.webdriver.chrome.options import Options |
| 21 | |
| 22 | options = Options() |
| 23 | driver_arguments = config.driver_arguments or [] |
| 24 | for arg in driver_arguments: |
| 25 | options.add_argument(arg) |
| 26 | |
| 27 | docs = [] |
| 28 | for url in config.urls: |
| 29 | scraper = WholeSiteReader( |
| 30 | prefix=url.prefix, |
| 31 | max_depth=url.max_depth, |
| 32 | driver=webdriver.Chrome(options=options), |
| 33 | ) |
| 34 | docs.extend(scraper.load_data(url.base_url)) |
| 35 | |
| 36 | return docs |