Downloads the content at the given URL (by default it will be cached locally). Unless unicode=False, the content is returned as a unicode string.
(self, timeout=10, cached=True, throttle=0, proxy=None, user_agent=USER_AGENT, referrer=REFERRER, authentication=None, unicode=False, **kwargs)
| 395 | raise URLError(src=e) |
| 396 | |
| 397 | def download(self, timeout=10, cached=True, throttle=0, proxy=None, user_agent=USER_AGENT, referrer=REFERRER, authentication=None, unicode=False, **kwargs): |
| 398 | """ Downloads the content at the given URL (by default it will be cached locally). |
| 399 | Unless unicode=False, the content is returned as a unicode string. |
| 400 | """ |
| 401 | # Filter OAuth parameters from cache id (they will be unique for each request). |
| 402 | if self._parts is None and self.method == GET and "oauth_" not in self._string: |
| 403 | id = self._string |
| 404 | else: |
| 405 | id = repr(self.parts) |
| 406 | id = re.sub("u{0,1}'oauth_.*?': u{0,1}'.*?', ", "", id) |
| 407 | # Keep a separate cache of unicode and raw download for same URL. |
| 408 | if unicode is True: |
| 409 | id = "u" + id |
| 410 | if cached and id in cache: |
| 411 | if isinstance(cache, dict): # Not a Cache object. |
| 412 | return cache[id] |
| 413 | if unicode is True: |
| 414 | return cache[id] |
| 415 | if unicode is False: |
| 416 | return cache.get(id, unicode=False) |
| 417 | t = time.time() |
| 418 | # Open a connection with the given settings, read it and (by default) cache the data. |
| 419 | try: |
| 420 | data = self.open(timeout, proxy, user_agent, referrer, authentication).read() |
| 421 | except socket.timeout, e: |
| 422 | raise URLTimeout(src=e) |
| 423 | if unicode is True: |
| 424 | data = u(data) |
| 425 | if cached: |
| 426 | cache[id] = data |
| 427 | if throttle: |
| 428 | time.sleep(max(throttle-(time.time()-t), 0)) |
| 429 | return data |
| 430 | |
| 431 | def read(self, *args): |
| 432 | return self.open().read(*args) |