Function fetch_data

buku.py:4347–4428

Handle server connection and redirections. Parameters ---------- url : str URL to fetch. http_head : bool If True, send only HTTP HEAD request. Default is False. Returns ------- FetchResult (url, title, desc, keywords, mime, bad, fetch_status)

(
        url: str,
        http_head: bool = False
)

def fetch_data(
        url: str,
        http_head: bool = False
) -> FetchResult:
    """Handle server connection and redirections.

    Parameters
    ----------
    url : str
        URL to fetch.
    http_head : bool
        If True, send only HTTP HEAD request. Default is False.

    Returns
    -------
    FetchResult
        (url, title, desc, keywords, mime, bad, fetch_status)
    """

    page_status = None
    page_url = url
    page_title = ''
    page_desc = ''
    page_keys = ''
    exception = False

    if is_nongeneric_url(url) or is_bad_url(url):
        return FetchResult(url, bad=True)

    if is_ignored_mime(url) or http_head:
        method = 'HEAD'
    else:
        method = 'GET'

    if not MYHEADERS:
        gen_headers()

    try:
        manager = get_PoolManager()

        while True:
            resp = manager.request(method, url, retries=Retry(redirect=10))
            page_status = resp.status

            if resp.status == 200:
                if method == 'GET':
                    for retry in resp.retries.history:
                        if retry.status not in PERMANENT_REDIRECTS:
                            break
                        page_status, page_url = retry.status, retry.redirect_location
                    page_title, page_desc, page_keys = get_data_from_page(resp)
            elif resp.status == 403 and url.endswith('/'):
                # HTTP response Forbidden
                # Handle URLs in the form of https://www.domain.com/
                # which fail when trying to fetch resource '/'
                # retry without trailing '/'

                LOGDBG('Received status 403: retrying...')