(url, level)
| 12 | |
| 13 | # recursively download images starting from the root URL |
| 14 | def downloadImages(url, level): # the root URL is level 0 |
| 15 | # do not go to other websites |
| 16 | global website |
| 17 | netloc = urlparse.urlsplit(url).netloc.split('.') |
| 18 | if netloc[-2] + netloc[-1] != website: |
| 19 | return |
| 20 | |
| 21 | global urlList |
| 22 | if url in urlList: # prevent using the same URL again |
| 23 | return |
| 24 | |
| 25 | try: |
| 26 | hdr = {'User-Agent': 'Mozilla/5.0'} |
| 27 | req = urllib2.Request(url,headers=hdr) |
| 28 | urlContent = urllib2.urlopen(req).read() |
| 29 | urlList.append(url) |
| 30 | print url |
| 31 | except: |
| 32 | return |
| 33 | |
| 34 | soup = BeautifulSoup(''.join(urlContent)) |
| 35 | # find and download all images |
| 36 | imgTags = soup.findAll('img') |
| 37 | for imgTag in imgTags: |
| 38 | imgUrl = imgTag['src'] |
| 39 | imgUrl = url[ : url.find(".com") + 4] + imgUrl if (imgUrl[ : 4] != "http") else imgUrl |
| 40 | # download only the proper image files |
| 41 | if imgUrl.lower().endswith('.jpeg') or \ |
| 42 | imgUrl.lower().endswith('.jpg') or \ |
| 43 | imgUrl.lower().endswith('.gif') or \ |
| 44 | imgUrl.lower().endswith('.png') or \ |
| 45 | imgUrl.lower().endswith('.bmp'): |
| 46 | try: |
| 47 | hdr = {'User-Agent': 'Mozilla/5.0'} |
| 48 | req = urllib2.Request(imgUrl,headers=hdr) |
| 49 | imgData = urllib2.urlopen(req).read() |
| 50 | global minImageFileSize |
| 51 | if len(imgData) >= minImageFileSize: |
| 52 | print " " + imgUrl |
| 53 | fileName = basename(urlparse.urlsplit(imgUrl)[2]) |
| 54 | output = open(os.path.join(downloadLocationPath, fileName),'wb') |
| 55 | output.write(imgData) |
| 56 | output.close() |
| 57 | except Exception, e: |
| 58 | print str(e) |
| 59 | # pass |
| 60 | |
| 61 | |
| 62 | |
| 63 | # if there are links on the webpage then recursively repeat |
| 64 | if level > 0: |
| 65 | linkTags = soup.findAll('a') |
| 66 | if len(linkTags) > 0: |
| 67 | for linkTag in linkTags: |
| 68 | try: |
| 69 | linkUrl = linkTag['href'] |
| 70 | downloadImages(linkUrl, level - 1) |
| 71 | except Exception, e: |
no test coverage detected