(self, request)
| 490 | log.exception(e) |
| 491 | |
| 492 | def deal_request(self, request): |
| 493 | response = None |
| 494 | |
| 495 | for parser in self._parsers: |
| 496 | if parser.name == request.parser_name: |
| 497 | try: |
| 498 | self.__class__._total_task_count += 1 |
| 499 | # 记录需下载的文档 |
| 500 | self.record_download_status( |
| 501 | ParserControl.DOWNLOAD_TOTAL, parser.name |
| 502 | ) |
| 503 | |
| 504 | # 解析request |
| 505 | if request.auto_request: |
| 506 | request_temp = None |
| 507 | response = None |
| 508 | |
| 509 | # 下载中间件 |
| 510 | if request.download_midware: |
| 511 | if isinstance(request.download_midware, (list, tuple)): |
| 512 | request_temp = request |
| 513 | for download_midware in request.download_midware: |
| 514 | download_midware = ( |
| 515 | download_midware |
| 516 | if callable(download_midware) |
| 517 | else tools.get_method(parser, download_midware) |
| 518 | ) |
| 519 | request_temp = download_midware(request_temp) |
| 520 | else: |
| 521 | download_midware = ( |
| 522 | request.download_midware |
| 523 | if callable(request.download_midware) |
| 524 | else tools.get_method( |
| 525 | parser, request.download_midware |
| 526 | ) |
| 527 | ) |
| 528 | request_temp = download_midware(request) |
| 529 | elif request.download_midware != False: |
| 530 | request_temp = parser.download_midware(request) |
| 531 | |
| 532 | # 请求 |
| 533 | if request_temp: |
| 534 | if ( |
| 535 | isinstance(request_temp, (tuple, list)) |
| 536 | and len(request_temp) == 2 |
| 537 | ): |
| 538 | request_temp, response = request_temp |
| 539 | |
| 540 | if not isinstance(request_temp, Request): |
| 541 | raise Exception( |
| 542 | "download_midware need return a request, but received type: {}".format( |
| 543 | type(request_temp) |
| 544 | ) |
| 545 | ) |
| 546 | request = request_temp |
| 547 | |
| 548 | if response is None: |
| 549 | response = ( |
no test coverage detected