Overcome HTTPRequest's load() size limit to allow loading very big web pages by overrding HTTPRequest's write() function
| 45 | |
| 46 | |
| 47 | class BIGHTTPRequest(HTTPRequest): |
| 48 | """ |
| 49 | Overcome HTTPRequest's load() size limit to allow |
| 50 | loading very big web pages by overrding HTTPRequest's write() function |
| 51 | """ |
| 52 | |
| 53 | # @TODO: Add 'limit' parameter to HTTPRequest in v0.4.10 |
| 54 | def __init__(self, cookies=None, options=None, limit=2000000): |
| 55 | self.limit = limit |
| 56 | HTTPRequest.__init__(self, cookies=cookies, options=options) |
| 57 | |
| 58 | def write(self, buf): |
| 59 | """ writes response """ |
| 60 | if self.limit and self.rep.tell() > self.limit or self.abort: |
| 61 | rep = self.getResponse() |
| 62 | if self.abort: |
| 63 | raise Abort() |
| 64 | f = open("response.dump", "wb") |
| 65 | f.write(rep) |
| 66 | f.close() |
| 67 | raise Exception("Loaded Url exceeded limit") |
| 68 | |
| 69 | self.rep.write(buf) |
| 70 | |
| 71 | |
| 72 | class Ffmpeg(object): |