| 133 | _exit(1) |
| 134 | |
| 135 | def retrieve(url, data=None): |
| 136 | global proxies_index |
| 137 | |
| 138 | retval = {} |
| 139 | |
| 140 | if proxies: |
| 141 | while True: |
| 142 | try: |
| 143 | opener = build_opener(ProxyHandler({"http": proxies[proxies_index], "https": proxies[proxies_index]})) |
| 144 | install_opener(opener) |
| 145 | proxies_index = (proxies_index + 1) % len(proxies) |
| 146 | urlopen(PROXY_TESTING_PAGE).read() |
| 147 | except KeyboardInterrupt: |
| 148 | raise |
| 149 | except: |
| 150 | pass |
| 151 | else: |
| 152 | break |
| 153 | |
| 154 | try: |
| 155 | req = Request("".join(url[_].replace(' ', "%20") if _ > url.find('?') else url[_] for _ in xrange(len(url))), data, HEADERS) |
| 156 | resp = urlopen(req, timeout=options.timeout) |
| 157 | retval[URL] = resp.url |
| 158 | retval[HTML] = resp.read() |
| 159 | retval[HTTPCODE] = resp.code |
| 160 | retval[RAW] = "%s %d %s\n%s\n%s" % (httplib.HTTPConnection._http_vsn_str, retval[HTTPCODE], resp.msg, str(resp.headers), retval[HTML]) |
| 161 | except Exception as ex: |
| 162 | retval[URL] = getattr(ex, "url", url) |
| 163 | retval[HTTPCODE] = getattr(ex, "code", None) |
| 164 | try: |
| 165 | retval[HTML] = ex.read() if hasattr(ex, "read") else getattr(ex, "msg", str(ex)) |
| 166 | except: |
| 167 | retval[HTML] = "" |
| 168 | retval[RAW] = "%s %s %s\n%s\n%s" % (httplib.HTTPConnection._http_vsn_str, retval[HTTPCODE] or "", getattr(ex, "msg", ""), str(ex.headers) if hasattr(ex, "headers") else "", retval[HTML]) |
| 169 | |
| 170 | for encoding in re.findall(r"charset=[\s\"']?([\w-]+)", retval[RAW])[::-1] + ["utf8"]: |
| 171 | encoding = ENCODING_TRANSLATIONS.get(encoding, encoding) |
| 172 | try: |
| 173 | retval[HTML] = retval[HTML].decode(encoding, errors="replace") |
| 174 | break |
| 175 | except: |
| 176 | pass |
| 177 | |
| 178 | match = re.search(r"<title>\s*(?P<result>[^<]+?)\s*</title>", retval[HTML], re.I) |
| 179 | retval[TITLE] = match.group("result") if match and "result" in match.groupdict() else None |
| 180 | retval[TEXT] = re.sub(r"(?si)<script.+?</script>|<!--.+?-->|<style.+?</style>|<[^>]+>|\s+", " ", retval[HTML]) |
| 181 | match = re.search(r"(?im)^Server: (.+)", retval[RAW]) |
| 182 | retval[SERVER] = match.group(1).strip() if match else "" |
| 183 | return retval |
| 184 | |
| 185 | def calc_hash(value, binary=True): |
| 186 | value = value.encode("utf8") if not isinstance(value, bytes) else value |