| 2005 | return WikiaTable |
| 2006 | |
| 2007 | def all(self, **kwargs): |
| 2008 | if kwargs.pop("batch", True): |
| 2009 | # We can take advantage of Wikia's search API to reduce bandwith. |
| 2010 | # Instead of executing a query to retrieve each article, |
| 2011 | # we query for a batch of (10) articles. |
| 2012 | iterator = self.list(_id="pageid", **kwargs) |
| 2013 | while True: |
| 2014 | batch, done = [], False |
| 2015 | try: |
| 2016 | for i in range(10): batch.append(iterator.next()) |
| 2017 | except StopIteration: |
| 2018 | done = True # No more articles, finish batch and raise StopIteration. |
| 2019 | url = URL(self._url.replace("api.php", "wikia.php"), method=GET, query={ |
| 2020 | "controller": "WikiaSearch", |
| 2021 | "method": "getPages", |
| 2022 | "ids": '|'.join(str(id) for id in batch), |
| 2023 | "format": "json" |
| 2024 | }) |
| 2025 | kwargs.setdefault("unicode", True) |
| 2026 | kwargs.setdefault("cached", True) |
| 2027 | kwargs["timeout"] = 10 * (1 + len(batch)) |
| 2028 | data = url.download(**kwargs) |
| 2029 | data = json.loads(data) |
| 2030 | for x in (data or {}).get("pages", {}).values(): |
| 2031 | yield WikiaArticle(title=x.get("title", ""), source=x.get("html", "")) |
| 2032 | if done: |
| 2033 | raise StopIteration |
| 2034 | for title in self.list(**kwargs): |
| 2035 | yield self.search(title, **kwargs) |
| 2036 | |
| 2037 | class WikiaArticle(MediaWikiArticle): |
| 2038 | def __repr__(self): |