Overcome HTTPRequest's load() size limit to allow loading very big web pages by overrding HTTPRequest's write() function
| 20 | |
| 21 | |
| 22 | class BIGHTTPRequest(HTTPRequest): |
| 23 | """ |
| 24 | Overcome HTTPRequest's load() size limit to allow |
| 25 | loading very big web pages by overrding HTTPRequest's write() function |
| 26 | """ |
| 27 | |
| 28 | # @TODO: Add 'limit' parameter to HTTPRequest in v0.4.10 |
| 29 | def __init__(self, cookies=None, options=None, limit=1000000): |
| 30 | self.limit = limit |
| 31 | HTTPRequest.__init__(self, cookies=cookies, options=options) |
| 32 | |
| 33 | def write(self, buf): |
| 34 | """ writes response """ |
| 35 | if self.limit and self.rep.tell() > self.limit or self.abort: |
| 36 | rep = self.getResponse() |
| 37 | if self.abort: |
| 38 | raise Abort() |
| 39 | f = open("response.dump", "wb") |
| 40 | f.write(rep) |
| 41 | f.close() |
| 42 | raise Exception("Loaded Url exceeded limit") |
| 43 | |
| 44 | self.rep.write(buf) |
| 45 | |
| 46 | |
| 47 | class FilecryptCc(Crypter): |