Scrape a webpage, convert to markdown and send in a message
(self, args, return_content=False)
| 217 | self.io.tool_output("Please provide a partial model name to search for.") |
| 218 | |
| 219 | def cmd_web(self, args, return_content=False): |
| 220 | "Scrape a webpage, convert to markdown and send in a message" |
| 221 | |
| 222 | url = args.strip() |
| 223 | if not url: |
| 224 | self.io.tool_error("Please provide a URL to scrape.") |
| 225 | return |
| 226 | |
| 227 | self.io.tool_output(f"Scraping {url}...") |
| 228 | if not self.scraper: |
| 229 | disable_playwright = getattr(self.args, "disable_playwright", False) |
| 230 | if disable_playwright: |
| 231 | res = False |
| 232 | else: |
| 233 | res = install_playwright(self.io) |
| 234 | if not res: |
| 235 | self.io.tool_warning("Unable to initialize playwright.") |
| 236 | |
| 237 | self.scraper = Scraper( |
| 238 | print_error=self.io.tool_error, |
| 239 | playwright_available=res, |
| 240 | verify_ssl=self.verify_ssl, |
| 241 | ) |
| 242 | |
| 243 | content = self.scraper.scrape(url) or "" |
| 244 | content = f"Here is the content of {url}:\n\n" + content |
| 245 | if return_content: |
| 246 | return content |
| 247 | |
| 248 | self.io.tool_output("... added to chat.") |
| 249 | |
| 250 | self.coder.cur_messages += [ |
| 251 | dict(role="user", content=content), |
| 252 | dict(role="assistant", content="Ok."), |
| 253 | ] |
| 254 | |
| 255 | def is_command(self, inp): |
| 256 | return inp[0] in "/!" |