@summary: 监控任务状态 --------- --------- @result:
(self)
| 155 | self._parsers.append(parser) |
| 156 | |
| 157 | def start_monitor_task(self): |
| 158 | """ |
| 159 | @summary: 监控任务状态 |
| 160 | --------- |
| 161 | --------- |
| 162 | @result: |
| 163 | """ |
| 164 | if not self._parsers: # 不是多模版模式， 将自己注入到parsers，自己为模版 |
| 165 | self._is_more_parsers = False |
| 166 | self._parsers.append(self) |
| 167 | |
| 168 | elif len(self._parsers) <= 1: |
| 169 | self._is_more_parsers = False |
| 170 | |
| 171 | # 添加任务 |
| 172 | for parser in self._parsers: |
| 173 | parser.add_task() |
| 174 | |
| 175 | while True: |
| 176 | try: |
| 177 | # 检查redis中是否有任务 任务小于_min_task_count 则从mysql中取 |
| 178 | tab_requests = setting.TAB_REQUESTS.format(redis_key=self._redis_key) |
| 179 | todo_task_count = self._redisdb.zget_count(tab_requests) |
| 180 | |
| 181 | tasks = [] |
| 182 | if todo_task_count < self._min_task_count: |
| 183 | tasks = self.get_task(todo_task_count) |
| 184 | if not tasks: |
| 185 | if not todo_task_count: |
| 186 | if self._keep_alive: |
| 187 | log.info("任务均已做完，爬虫常驻, 等待新任务") |
| 188 | time.sleep(self._check_task_interval) |
| 189 | continue |
| 190 | elif self.have_alive_spider(): |
| 191 | log.info("任务均已做完，但还有爬虫在运行，等待爬虫结束") |
| 192 | time.sleep(self._check_task_interval) |
| 193 | continue |
| 194 | elif not self.related_spider_is_done(): |
| 195 | continue |
| 196 | else: |
| 197 | log.info("任务均已做完，爬虫结束") |
| 198 | break |
| 199 | |
| 200 | else: |
| 201 | log.info("redis 中尚有%s条积压任务，暂时不派发新任务" % todo_task_count) |
| 202 | |
| 203 | if not tasks: |
| 204 | if todo_task_count >= self._min_task_count: |
| 205 | # log.info('任务正在进行 redis中剩余任务 %s' % todo_task_count) |
| 206 | pass |
| 207 | else: |
| 208 | log.info("无待做种子 redis中剩余任务 %s" % todo_task_count) |
| 209 | else: |
| 210 | # make start requests |
| 211 | self.distribute_task(tasks) |
| 212 | log.info(f"添加任务到redis成功 共{len(tasks)}条") |
| 213 | |
| 214 | except Exception as e: |
nothing calls this directly
no test coverage detected