Worker thread procedure. Test how long it takes to return the mirror index page, and stuff the results into resultQueue.
(workQueue, resultQueue)
| 59 | workQueue.put(url) |
| 60 | |
| 61 | def TestUrl(workQueue, resultQueue): |
| 62 | |
| 63 | ''' Worker thread procedure. Test how long it takes to return the |
| 64 | mirror index page, and stuff the results into resultQueue.''' |
| 65 | |
| 66 | def SubthreadProc(url, result): |
| 67 | |
| 68 | ''' Subthread procedure. Actually get the mirror index page |
| 69 | in a subthread, so that we can time out using join rather than |
| 70 | wait for a very slow server. Passing in a list for result |
| 71 | lets us simulate pass-by-reference, since callers cannot get |
| 72 | the return code from a Python thread.''' |
| 73 | |
| 74 | startTime = time.time() |
| 75 | try: |
| 76 | data = urllib.urlopen(url).read() |
| 77 | except Exception: |
| 78 | # Could be a socket error or an HTTP error--either way, we |
| 79 | # don't care--it's a failure to us. |
| 80 | result.append(-1) |
| 81 | else: |
| 82 | elapsed = int((time.time() - startTime) * 1000) |
| 83 | result.append(elapsed) |
| 84 | |
| 85 | |
| 86 | while 1: |
| 87 | # Contine pulling data from the work queue until it's empty |
| 88 | try: |
| 89 | url = workQueue.get(0) |
| 90 | except Queue.Empty: |
| 91 | # work queue is empty--exit the thread proc. |
| 92 | return |
| 93 | |
| 94 | # Create a single subthread to do the actual work |
| 95 | result = [] |
| 96 | subThread = threading.Thread(target=SubthreadProc, args=(url, result)) |
| 97 | |
| 98 | # Daemonize the subthread so that even if a few are hanging |
| 99 | # around when the process is done, the process will exit. |
| 100 | subThread.setDaemon(True) |
| 101 | |
| 102 | # Run the subthread and wait for it to finish, or time out |
| 103 | subThread.start() |
| 104 | subThread.join(HTTP_TIMEOUT) |
| 105 | |
| 106 | if [] == result: |
| 107 | # Subthread hasn't give a result yet. Consider it timed out. |
| 108 | resultQueue.put((url, "TIMEOUT")) |
| 109 | elif -1 == result[0]: |
| 110 | # Subthread returned an error from geturl. |
| 111 | resultQueue.put((url, "FAILED")) |
| 112 | else: |
| 113 | # Subthread returned a time. Store it. |
| 114 | resultQueue.put((url, result[0])) |
| 115 | |
| 116 | |
| 117 | workers = [] |