Method process_request

selenium/scrapy_splash/middleware.py:260–365

(self, request, spider)

        return self.crawler.spider.state[self.remote_keys_key]

    def process_request(self, request, spider):
        if 'splash' not in request.meta:
            return

        if request.method not in {'GET', 'POST'}:
            logger.warning(
                "Currently only GET and POST requests are supported by "
                "SplashMiddleware; %(request)s will be handled without Splash",
                {'request': request},
                extra={'spider': spider}
            )
            return request

        if request.meta.get("_splash_processed"):
            # don't process the same request more than once
            return

        splash_options = request.meta['splash']
        request.meta['_splash_processed'] = True

        slot_policy = splash_options.get('slot_policy', self.slot_policy)
        self._set_download_slot(request, request.meta, slot_policy)

        args = splash_options.setdefault('args', {})

        if '_replaced_args' in splash_options:
            # restore arguments before sending request to the downloader
            load_args = {}
            save_args = []
            local_arg_fingerprints = {}
            for name in splash_options['_replaced_args']:
                fp = args[name]
                # Use remote Splash argument cache: if Splash key
                # for a value is known then don't send the value to Splash;
                # if it is unknown then try to save the value on server using
                # ``save_args``.
                if fp in self._remote_keys:
                    load_args[name] = self._remote_keys[fp]
                    del args[name]
                else:
                    save_args.append(name)
                    args[name] = self._argument_values[fp]

                local_arg_fingerprints[name] = fp

            if load_args:
                args['load_args'] = load_args
            if save_args:
                args['save_args'] = save_args
            splash_options['_local_arg_fingerprints'] = local_arg_fingerprints

            del splash_options['_replaced_args']  # ??

        args.setdefault('url', request.url)
        if request.method == 'POST':
            args.setdefault('http_method', request.method)
            # XXX: non-UTF8 request bodies are not supported now
            args.setdefault('body', request.body.decode('utf8'))