| 45 | self.scraper_country_code = "us" |
| 46 | self.scraper_time_out = 40 |
| 47 | def readpage(self, url: str) -> str: |
| 48 | payload = {'api_key': self.scraper_key, |
| 49 | 'url': url, |
| 50 | 'output_format': 'markdown', |
| 51 | 'country_code': self.scraper_country_code } |
| 52 | max_retries = 2 |
| 53 | for attempt in range(max_retries): |
| 54 | try: |
| 55 | r = requests.get('https://api.scraperapi.com/', params=payload, timeout=self.scraper_time_out) |
| 56 | content = r.text |
| 57 | return content |
| 58 | except requests.exceptions.Timeout: |
| 59 | # 超时情况下返回默认内容 |
| 60 | content = "[visit] Request timed out after {} seconds. Using default content.".format(self.scraper_time_out) |
| 61 | except Exception as e: |
| 62 | content ="[visit] Failed to read page." |
| 63 | return content |
| 64 | |
| 65 | |
| 66 | @register_tool('retrieve', allow_overwrite=True) |