| 2 | import urlparse |
| 3 | |
| 4 | def pipeline(domain,pages,max_out_bound=4,debuglevel=0): |
| 5 | pagecount = len(pages) |
| 6 | conn = HTTPConnection(domain) |
| 7 | conn.set_debuglevel(debuglevel) |
| 8 | respobjs = [None]*pagecount |
| 9 | finished = [False]*pagecount |
| 10 | data = [None]*pagecount |
| 11 | headers = {'Host':domain,'Content-Length':0,'Connection':'Keep-Alive'} |
| 12 | |
| 13 | while not all(finished): |
| 14 | # Send |
| 15 | out_bound = 0 |
| 16 | for i,page in enumerate(pages): |
| 17 | if out_bound >= max_out_bound: |
| 18 | break |
| 19 | elif page and not finished[i] and respobjs[i] is None: |
| 20 | if debuglevel > 0: |
| 21 | print 'Sending request for %r...' % (page,) |
| 22 | conn._HTTPConnection__state = _CS_IDLE # FU private variable! |
| 23 | conn.request("GET", page, None, headers) |
| 24 | respobjs[i] = conn.response_class(conn.sock, strict=conn.strict, method=conn._method) |
| 25 | out_bound += 1 |
| 26 | # Try to read a response |
| 27 | for i,resp in enumerate(respobjs): |
| 28 | if resp is None: |
| 29 | continue |
| 30 | if debuglevel > 0: |
| 31 | print 'Retrieving %r...' % (pages[i],) |
| 32 | out_bound -= 1 |
| 33 | skip_read = False |
| 34 | resp.begin() |
| 35 | if debuglevel > 0: |
| 36 | print ' %d %s' % (resp.status, resp.reason) |
| 37 | if 200 <= resp.status < 300: |
| 38 | # Ok |
| 39 | data[i] = resp.read() |
| 40 | cookie = resp.getheader('Set-Cookie') |
| 41 | if cookie is not None: |
| 42 | headers['Cookie'] = cookie |
| 43 | skip_read = True |
| 44 | finished[i] = True |
| 45 | respobjs[i] = None |
| 46 | elif 300 <= resp.status < 400: |
| 47 | # Redirect |
| 48 | loc = resp.getheader('Location') |
| 49 | respobjs[i] = None |
| 50 | parsed = loc and urlparse.urlparse(loc) |
| 51 | if not parsed: |
| 52 | # Missing or empty location header |
| 53 | data[i] = (resp.status, resp.reason) |
| 54 | finished[i] = True |
| 55 | elif parsed.netloc != '' and parsed.netloc != host: |
| 56 | # Redirect to another host |
| 57 | data[i] = (resp.status, resp.reason, loc) |
| 58 | finished[i] = True |
| 59 | else: |
| 60 | path = urlparse.urlunparse(parsed._replace(scheme='',netloc='',fragment='')) |
| 61 | if debuglevel > 0: |