| 10 | |
| 11 | |
| 12 | class GithubTaskTask(object): |
| 13 | def __init__(self, task_id, keyword): |
| 14 | self.task_id = task_id |
| 15 | self.keyword = keyword |
| 16 | self.collection = "github_task" |
| 17 | self.results = [] |
| 18 | |
| 19 | def search_result(self): |
| 20 | self.update_status("search") |
| 21 | results = github_search(keyword=self.keyword) |
| 22 | self.results.extend(results) |
| 23 | |
| 24 | def save_content(self): |
| 25 | self.update_status("fetch content-{}".format(len(self.results))) |
| 26 | for result in self.results: |
| 27 | if not isinstance(result, GithubResult): |
| 28 | continue |
| 29 | |
| 30 | if self.filter_result(result): |
| 31 | continue |
| 32 | |
| 33 | item = self.result_to_dict(result) |
| 34 | |
| 35 | utils.conn_db("github_result").insert_one(item) |
| 36 | |
| 37 | def result_to_dict(self, result): |
| 38 | item = result.to_dict() |
| 39 | item["human_content"] = result.human_content(self.keyword) |
| 40 | item["keyword"] = self.keyword |
| 41 | item["github_task_id"] = self.task_id |
| 42 | return item |
| 43 | |
| 44 | def filter_result(self, result: GithubResult): |
| 45 | path_keyword_list = ["open-app-filter/", "/adbyby", |
| 46 | "/adblock", "luci-app-dnsfilter/", |
| 47 | "Spider/", "/spider", "_files/", |
| 48 | "alexa_10k.json", "/WeWorkProviderTest.php"] |
| 49 | for path in path_keyword_list: |
| 50 | if path in result.path: |
| 51 | return True |
| 52 | |
| 53 | content_keyword_list = ["DOMAIN-SUFFIX", "HOST-SUFFIX", "name:[proto;sport;dport;host", |
| 54 | ' "websites": [', |
| 55 | "import android.app.Application;", |
| 56 | "import android.app.Activity;"] |
| 57 | for keyword in content_keyword_list: |
| 58 | if keyword in result.content: |
| 59 | return True |
| 60 | |
| 61 | return False |
| 62 | |
| 63 | def update_status(self, value): |
| 64 | query = {"_id": ObjectId(self.task_id)} |
| 65 | update = {"$set": {"status": value}} |
| 66 | utils.conn_db(self.collection).update_one(query, update) |
| 67 | |
| 68 | def set_start_time(self): |
| 69 | query = {"_id": ObjectId(self.task_id)} |