| 62 | raise RuntimeError("Failed to access Bing Search API.") |
| 63 | |
| 64 | def load_page(self, url : str, max_retry : int = 3) -> Tuple[bool, str]: |
| 65 | for _ in range(max_retry): |
| 66 | try: |
| 67 | res = requests.get(url, timeout=15) |
| 68 | if res.status_code == 200: |
| 69 | res.raise_for_status() |
| 70 | else: |
| 71 | raise RuntimeError("Failed to load page, code {}".format(res.status_code)) |
| 72 | except Exception: |
| 73 | # failed, retry |
| 74 | res = None |
| 75 | continue |
| 76 | res.encoding = res.apparent_encoding |
| 77 | content = res.text |
| 78 | break |
| 79 | if res is None: |
| 80 | return False, "Timeout for loading this page, Please try to load another one or search again." |
| 81 | try: |
| 82 | soup = BeautifulSoup(content, 'html.parser') |
| 83 | paragraphs = soup.find_all('p') |
| 84 | page_detail = "" |
| 85 | for p in paragraphs: |
| 86 | text = p.get_text().strip() |
| 87 | page_detail += text |
| 88 | return True, page_detail |
| 89 | except Exception: |
| 90 | return False, "Timeout for loading this page, Please try to load another one or search again." |
| 91 | |
| 92 | class CONTENT_TYPE(Enum): |
| 93 | SEARCH_RESULT = 0 |