Returns a connection to the url from which data can be retrieved with connection.read(). When the timeout amount of seconds is exceeded, raises a URLTimeout. When an error occurs, raises a URLError (e.g. HTTP404NotFound).
(self, timeout=10, proxy=None, user_agent=USER_AGENT, referrer=REFERRER, authentication=None)
| 349 | raise AttributeError, "'URL' object has no attribute '%s'" % k |
| 350 | |
| 351 | def open(self, timeout=10, proxy=None, user_agent=USER_AGENT, referrer=REFERRER, authentication=None): |
| 352 | """ Returns a connection to the url from which data can be retrieved with connection.read(). |
| 353 | When the timeout amount of seconds is exceeded, raises a URLTimeout. |
| 354 | When an error occurs, raises a URLError (e.g. HTTP404NotFound). |
| 355 | """ |
| 356 | url = self.string |
| 357 | # Use basic urllib.urlopen() instead of urllib2.urlopen() for local files. |
| 358 | if os.path.exists(url): |
| 359 | return urllib.urlopen(url) |
| 360 | # Get the query string as a separate parameter if method=POST. |
| 361 | post = self.method == POST and self.querystring or None |
| 362 | socket.setdefaulttimeout(timeout) |
| 363 | if proxy: |
| 364 | proxy = urllib2.ProxyHandler({proxy[1]: proxy[0]}) |
| 365 | proxy = urllib2.build_opener(proxy, urllib2.HTTPHandler) |
| 366 | urllib2.install_opener(proxy) |
| 367 | try: |
| 368 | request = urllib2.Request(bytestring(url), post, { |
| 369 | "User-Agent": user_agent, |
| 370 | "Referer": referrer |
| 371 | }) |
| 372 | # Basic authentication is established with authentication=(username, password). |
| 373 | if authentication is not None: |
| 374 | request.add_header("Authorization", "Basic %s" % |
| 375 | base64.encodestring('%s:%s' % authentication)) |
| 376 | return urllib2.urlopen(request) |
| 377 | except urllib2.HTTPError, e: |
| 378 | if e.code == 301: raise HTTP301Redirect(src=e) |
| 379 | if e.code == 400: raise HTTP400BadRequest(src=e) |
| 380 | if e.code == 401: raise HTTP401Authentication(src=e) |
| 381 | if e.code == 403: raise HTTP403Forbidden(src=e) |
| 382 | if e.code == 404: raise HTTP404NotFound(src=e) |
| 383 | if e.code == 420: raise HTTP420Error(src=e) |
| 384 | if e.code == 429: raise HTTP429TooMayRequests(src=e) |
| 385 | if e.code == 500: raise HTTP500InternalServerError(src=e) |
| 386 | raise HTTPError(src=e) |
| 387 | except socket.timeout, e: |
| 388 | raise URLTimeout(src=e) |
| 389 | except urllib2.URLError, e: |
| 390 | if e.reason == "timed out" \ |
| 391 | or e.reason[0] in (36, "timed out"): |
| 392 | raise URLTimeout(src=e) |
| 393 | raise URLError(e.reason, src=e) |
| 394 | except ValueError, e: |
| 395 | raise URLError(src=e) |
| 396 | |
| 397 | def download(self, timeout=10, cached=True, throttle=0, proxy=None, user_agent=USER_AGENT, referrer=REFERRER, authentication=None, unicode=False, **kwargs): |
| 398 | """ Downloads the content at the given URL (by default it will be cached locally). |