(self, request)
| 75 | return cls._failed_task_count, cls._success_task_count, cls._total_task_count |
| 76 | |
| 77 | def deal_request(self, request): |
| 78 | response = None |
| 79 | request_redis = request["request_redis"] |
| 80 | request = request["request_obj"] |
| 81 | |
| 82 | del_request_redis_after_item_to_db = False |
| 83 | del_request_redis_after_request_to_db = False |
| 84 | |
| 85 | for parser in self._parsers: |
| 86 | if parser.name == request.parser_name: |
| 87 | used_download_midware_enable = False |
| 88 | try: |
| 89 | self.__class__._total_task_count += 1 |
| 90 | # 记录需下载的文档 |
| 91 | self.record_download_status( |
| 92 | ParserControl.DOWNLOAD_TOTAL, parser.name |
| 93 | ) |
| 94 | |
| 95 | # 解析request |
| 96 | if request.auto_request: |
| 97 | request_temp = None |
| 98 | response = None |
| 99 | |
| 100 | # 下载中间件 |
| 101 | if request.download_midware: |
| 102 | if isinstance(request.download_midware, (list, tuple)): |
| 103 | request_temp = request |
| 104 | for download_midware in request.download_midware: |
| 105 | download_midware = ( |
| 106 | download_midware |
| 107 | if callable(download_midware) |
| 108 | else tools.get_method(parser, download_midware) |
| 109 | ) |
| 110 | request_temp = download_midware(request_temp) |
| 111 | else: |
| 112 | download_midware = ( |
| 113 | request.download_midware |
| 114 | if callable(request.download_midware) |
| 115 | else tools.get_method( |
| 116 | parser, request.download_midware |
| 117 | ) |
| 118 | ) |
| 119 | request_temp = download_midware(request) |
| 120 | elif request.download_midware != False: |
| 121 | request_temp = parser.download_midware(request) |
| 122 | |
| 123 | # 请求 |
| 124 | if request_temp: |
| 125 | if ( |
| 126 | isinstance(request_temp, (tuple, list)) |
| 127 | and len(request_temp) == 2 |
| 128 | ): |
| 129 | request_temp, response = request_temp |
| 130 | |
| 131 | if not isinstance(request_temp, Request): |
| 132 | raise Exception( |
| 133 | "download_midware need return a request, but received type: {}".format( |
| 134 | type(request_temp) |
no test coverage detected