| 30 | |
| 31 | @flag.admin |
| 32 | def actionFeedQuery(self, to, limit=10, day_limit=3): |
| 33 | from Site import SiteManager |
| 34 | rows = [] |
| 35 | stats = [] |
| 36 | |
| 37 | total_s = time.time() |
| 38 | num_sites = 0 |
| 39 | |
| 40 | for address, site_data in list(self.user.sites.items()): |
| 41 | feeds = site_data.get("follow") |
| 42 | if not feeds: |
| 43 | continue |
| 44 | if type(feeds) is not dict: |
| 45 | self.log.debug("Invalid feed for site %s" % address) |
| 46 | continue |
| 47 | num_sites += 1 |
| 48 | for name, query_set in feeds.items(): |
| 49 | site = SiteManager.site_manager.get(address) |
| 50 | if not site or not site.storage.has_db: |
| 51 | continue |
| 52 | |
| 53 | s = time.time() |
| 54 | try: |
| 55 | query_raw, params = query_set |
| 56 | query_parts = re.split(r"UNION(?:\s+ALL|)", query_raw) |
| 57 | for i, query_part in enumerate(query_parts): |
| 58 | db_query = DbQuery(query_part) |
| 59 | if day_limit: |
| 60 | where = " WHERE %s > strftime('%%s', 'now', '-%s day')" % (db_query.fields.get("date_added", "date_added"), day_limit) |
| 61 | if "WHERE" in query_part: |
| 62 | query_part = re.sub("WHERE (.*?)(?=$| GROUP BY)", where+" AND (\\1)", query_part) |
| 63 | else: |
| 64 | query_part += where |
| 65 | query_parts[i] = query_part |
| 66 | query = " UNION ".join(query_parts) |
| 67 | |
| 68 | if ":params" in query: |
| 69 | query_params = map(helper.sqlquote, params) |
| 70 | query = query.replace(":params", ",".join(query_params)) |
| 71 | |
| 72 | res = site.storage.query(query + " ORDER BY date_added DESC LIMIT %s" % limit) |
| 73 | |
| 74 | except Exception as err: # Log error |
| 75 | self.log.error("%s feed query %s error: %s" % (address, name, Debug.formatException(err))) |
| 76 | stats.append({"site": site.address, "feed_name": name, "error": str(err)}) |
| 77 | continue |
| 78 | |
| 79 | for row in res: |
| 80 | row = dict(row) |
| 81 | if not isinstance(row["date_added"], (int, float, complex)): |
| 82 | self.log.debug("Invalid date_added from site %s: %r" % (address, row["date_added"])) |
| 83 | continue |
| 84 | if row["date_added"] > 1000000000000: # Formatted as millseconds |
| 85 | row["date_added"] = row["date_added"] / 1000 |
| 86 | if "date_added" not in row or row["date_added"] > time.time() + 120: |
| 87 | self.log.debug("Newsfeed item from the future from from site %s" % address) |
| 88 | continue # Feed item is in the future, skip it |
| 89 | row["site"] = address |