| 100 | |
| 101 | |
| 102 | class GithubTaskMonitor(GithubTaskTask): |
| 103 | def __init__(self, task_id, keyword, scheduler_id): |
| 104 | super().__init__(task_id, keyword) |
| 105 | self.scheduler_id = scheduler_id |
| 106 | self.hash_md5_list = [] |
| 107 | self.new_results = [] # 保存过滤后的结果 |
| 108 | |
| 109 | def init_md5_list(self): |
| 110 | query = {"github_scheduler_id": self.scheduler_id} |
| 111 | results = list(utils.conn_db("github_hash").find(query, {"hash_md5": 1})) |
| 112 | for result in results: |
| 113 | if result["hash_md5"] not in self.hash_md5_list: |
| 114 | self.hash_md5_list.append(result["hash_md5"]) |
| 115 | |
| 116 | def save_mongo(self): |
| 117 | cnt = 0 |
| 118 | self.update_status("fetch content") |
| 119 | for result in self.results: |
| 120 | if not isinstance(result, GithubResult): |
| 121 | continue |
| 122 | |
| 123 | if result.hash_md5 in self.hash_md5_list: |
| 124 | continue |
| 125 | |
| 126 | # 保存md5, 直接在过滤前，避免重复过滤 |
| 127 | self.hash_md5_list.append(result.hash_md5) |
| 128 | hash_data = {"hash_md5": result.hash_md5, "github_scheduler_id": self.scheduler_id} |
| 129 | utils.conn_db("github_hash").insert_one(hash_data) |
| 130 | |
| 131 | if self.filter_result(result): |
| 132 | continue |
| 133 | |
| 134 | item = self.result_to_dict(result) |
| 135 | item["github_scheduler_id"] = self.scheduler_id |
| 136 | item["update_date"] = utils.curr_date_obj() |
| 137 | cnt += 1 |
| 138 | self.new_results.append(result) |
| 139 | utils.conn_db("github_monitor_result").insert_one(item) |
| 140 | |
| 141 | logger.info("github_monitor save {} {}".format(self.keyword, cnt)) |
| 142 | |
| 143 | def build_repo_map(self): |
| 144 | repo_map = dict() |
| 145 | for result in self.new_results: |
| 146 | repo_name = result.repo_full_name |
| 147 | if repo_map.get(repo_name) is None: |
| 148 | repo_map[repo_name] = [result] |
| 149 | else: |
| 150 | repo_map[repo_name].append(result) |
| 151 | |
| 152 | return repo_map |
| 153 | |
| 154 | def build_html_report(self): |
| 155 | repo_map = self.build_repo_map() |
| 156 | repo_cnt = 0 |
| 157 | html = "<br/><br/> <div> 搜索: {} 仓库数：{} 结果数： {} </div>".format(self.keyword, |
| 158 | len(repo_map.keys()), len(self.new_results)) |
| 159 | for repo_name in repo_map: |
no outgoing calls
no test coverage detected