Overcome HTTPRequest's load() size limit to allow loading very big web pages by overrding HTTPRequest's write() function
| 9 | |
| 10 | |
| 11 | class BIGHTTPRequest(HTTPRequest): |
| 12 | """ |
| 13 | Overcome HTTPRequest's load() size limit to allow |
| 14 | loading very big web pages by overrding HTTPRequest's write() function |
| 15 | """ |
| 16 | |
| 17 | # @TODO: Add 'limit' parameter to HTTPRequest in v0.4.10 |
| 18 | def __init__(self, cookies=None, options=None, limit=1000000): |
| 19 | self.limit = limit |
| 20 | HTTPRequest.__init__(self, cookies=cookies, options=options) |
| 21 | |
| 22 | def write(self, buf): |
| 23 | """ writes response """ |
| 24 | if self.limit and self.rep.tell() > self.limit or self.abort: |
| 25 | rep = self.getResponse() |
| 26 | if self.abort: |
| 27 | raise Abort() |
| 28 | f = open("response.dump", "wb") |
| 29 | f.write(rep) |
| 30 | f.close() |
| 31 | raise Exception("Loaded Url exceeded limit") |
| 32 | |
| 33 | self.rep.write(buf) |
| 34 | |
| 35 | |
| 36 | class UserscloudCom(SimpleHoster): |