| 5 | |
| 6 | |
| 7 | class ProbeHTTP(BaseThread): |
| 8 | def __init__(self, domains, concurrency=6): |
| 9 | super().__init__(self._build_targets(domains), concurrency = concurrency) |
| 10 | |
| 11 | self.sites = [] |
| 12 | self.domains = domains |
| 13 | |
| 14 | def _build_targets(self, domains): |
| 15 | _targets = [] |
| 16 | for item in domains: |
| 17 | domain = item |
| 18 | if hasattr(item, 'domain'): |
| 19 | domain = item.domain |
| 20 | |
| 21 | _targets.append("https://{}".format(domain)) |
| 22 | _targets.append("http://{}".format(domain)) |
| 23 | |
| 24 | return _targets |
| 25 | |
| 26 | def work(self, target): |
| 27 | conn = utils.http_req(target, 'get', timeout=(3, 2), stream=True) |
| 28 | conn.close() |
| 29 | |
| 30 | if conn.status_code in [502, 504, 501, 422, 410]: |
| 31 | logger.debug(f"{target} 状态码为 {conn.status_code} 跳过") |
| 32 | return |
| 33 | |
| 34 | self.sites.append(target) |
| 35 | |
| 36 | def run(self): |
| 37 | t1 = time.time() |
| 38 | logger.info("start ProbeHTTP {}".format(len(self.targets))) |
| 39 | self._run() |
| 40 | # 去除https和http相同的 |
| 41 | alive_site = [] |
| 42 | for x in self.sites: |
| 43 | if x.startswith("https://"): |
| 44 | alive_site.append(x) |
| 45 | |
| 46 | elif x.startswith("http://"): |
| 47 | x_temp = "https://" + x[7:] |
| 48 | if x_temp not in self.sites: |
| 49 | alive_site.append(x) |
| 50 | |
| 51 | elapse = time.time() - t1 |
| 52 | logger.info("end ProbeHTTP {} elapse {}".format(len(alive_site), elapse)) |
| 53 | |
| 54 | return alive_site |
| 55 | |
| 56 | |
| 57 | def probe_http(domain, concurrency=10): |