| 27 | |
| 28 | # Load all sites from data/sites.json |
| 29 | def load(self, cleanup=True, startup=False): |
| 30 | self.log.debug("Loading sites...") |
| 31 | self.loaded = False |
| 32 | from .Site import Site |
| 33 | address_found = [] |
| 34 | added = 0 |
| 35 | # Load new adresses |
| 36 | try: |
| 37 | json_path = "%s/sites.json" % config.data_dir |
| 38 | data = json.load(open(json_path)) |
| 39 | except Exception as err: |
| 40 | raise Exception("Unable to load %s: %s" % (json_path, err)) |
| 41 | |
| 42 | for address, settings in data.items(): |
| 43 | if address not in self.sites: |
| 44 | if os.path.isfile("%s/%s/content.json" % (config.data_dir, address)): |
| 45 | # Root content.json exists, try load site |
| 46 | s = time.time() |
| 47 | try: |
| 48 | site = Site(address, settings=settings) |
| 49 | site.content_manager.contents.get("content.json") |
| 50 | except Exception as err: |
| 51 | self.log.debug("Error loading site %s: %s" % (address, err)) |
| 52 | continue |
| 53 | self.sites[address] = site |
| 54 | self.log.debug("Loaded site %s in %.3fs" % (address, time.time() - s)) |
| 55 | added += 1 |
| 56 | elif startup: |
| 57 | # No site directory, start download |
| 58 | self.log.debug("Found new site in sites.json: %s" % address) |
| 59 | gevent.spawn(self.need, address, settings=settings) |
| 60 | added += 1 |
| 61 | |
| 62 | address_found.append(address) |
| 63 | |
| 64 | # Remove deleted adresses |
| 65 | if cleanup: |
| 66 | for address in list(self.sites.keys()): |
| 67 | if address not in address_found: |
| 68 | del(self.sites[address]) |
| 69 | self.log.debug("Removed site: %s" % address) |
| 70 | |
| 71 | # Remove orpan sites from contentdb |
| 72 | content_db = ContentDb.getContentDb() |
| 73 | for row in content_db.execute("SELECT * FROM site").fetchall(): |
| 74 | address = row["address"] |
| 75 | if address not in self.sites: |
| 76 | self.log.info("Deleting orphan site from content.db: %s" % address) |
| 77 | |
| 78 | try: |
| 79 | content_db.execute("DELETE FROM site WHERE ?", {"address": address}) |
| 80 | except Exception as err: |
| 81 | self.log.error("Can't delete site %s from content_db: %s" % (address, err)) |
| 82 | |
| 83 | if address in content_db.site_ids: |
| 84 | del content_db.site_ids[address] |
| 85 | if address in content_db.sites: |
| 86 | del content_db.sites[address] |