| 17 | return sorted(self.items(),key=lambda x: (x[1], x[0]), reverse=reverse) |
| 18 | |
| 19 | def tinyTable(url): |
| 20 | global cur,conn |
| 21 | cur.execute('SELECT id,page,retrieved_at FROM TinyTable WHERE URL = ?', (url, )) |
| 22 | try: |
| 23 | row = cur.fetchone() |
| 24 | print 'DATE',row[2] |
| 25 | return row[1] |
| 26 | except: |
| 27 | row = None |
| 28 | print 'Retrieving', url |
| 29 | |
| 30 | data = urllib.urlopen (url).read() |
| 31 | if row != None: |
| 32 | cur.execute("UPDATE TinyTable SET page=?,retrieved_at=datetime('now') WHERE id=?", (unicode(data, 'utf-8'), row[0])) |
| 33 | else: |
| 34 | cur.execute("INSERT INTO TinyTable (url, page, retrieved_at) VALUES (?, ?, datetime('now'))",(url, unicode(data, 'utf-8'))) |
| 35 | conn.commit() |
| 36 | return data |
| 37 | |
| 38 | cururl = 'https://ctools.umich.edu/access/wiki/site/f57681b8-6db9-46cf-aad1-3a0bdd621138/home.html' |
| 39 | urls = list() |