Method getPage

lib/request/connect.py:271–1025

This method connects to the target URL or proxy and returns the target URL page content

(**kwargs)

    @staticmethod
    def getPage(**kwargs):
        """
        This method connects to the target URL or proxy and returns
        the target URL page content
        """

        if conf.offline:
            return None, None, None

        url = kwargs.get("url", None) or conf.url
        get = kwargs.get("get", None)
        post = kwargs.get("post", None)
        method = kwargs.get("method", None)
        cookie = kwargs.get("cookie", None)
        ua = kwargs.get("ua", None) or conf.agent
        referer = kwargs.get("referer", None) or conf.referer
        direct_ = kwargs.get("direct", False)
        multipart = kwargs.get("multipart", None)
        silent = kwargs.get("silent", False)
        raise404 = kwargs.get("raise404", True)
        timeout = kwargs.get("timeout", None) or conf.timeout
        auxHeaders = kwargs.get("auxHeaders", None)
        response = kwargs.get("response", False)
        ignoreTimeout = kwargs.get("ignoreTimeout", False) or kb.ignoreTimeout or conf.ignoreTimeouts
        refreshing = kwargs.get("refreshing", False)
        retrying = kwargs.get("retrying", False)
        crawling = kwargs.get("crawling", False)
        checking = kwargs.get("checking", False)
        skipRead = kwargs.get("skipRead", False)
        finalCode = kwargs.get("finalCode", False)
        chunked = kwargs.get("chunked", False) or conf.chunked

        if isinstance(conf.delay, (int, float)) and conf.delay > 0:
            time.sleep(conf.delay)

        start = time.time()

        threadData = getCurrentThreadData()
        with kb.locks.request:
            kb.requestCounter += 1
            threadData.lastRequestUID = kb.requestCounter

            if conf.proxyFreq:
                if kb.requestCounter % conf.proxyFreq == 0:
                    conf.proxy = None

                    warnMsg = "changing proxy"
                    logger.warning(warnMsg)

                    setHTTPHandlers()

        if conf.dummy or conf.murphyRate and randomInt() % conf.murphyRate == 0:
            if conf.murphyRate:
                time.sleep(randomInt() % (MAX_MURPHY_SLEEP_TIME + 1))

            page, headers, code = randomStr(int(randomInt()), alphabet=[_unichr(_) for _ in xrange(256)]), None, None if not conf.murphyRate else randomInt(3)

            threadData.lastPage = page