Method downloadFile

core/downloader/downloader_http.go:186–227

Download file and change the charset of page charset.

(p *page.Page, req *request.Request)

// Download file and change the charset of page charset.
func (this *HttpDownloader) downloadFile(p *page.Page, req *request.Request) (*page.Page, string) {
    var err error
    var urlstr string
    if urlstr = req.GetUrl(); len(urlstr) == 0 {
        mlog.LogInst().LogError("url is empty")
        p.SetStatus(true, "url is empty")
        return p, ""
    }

    client := &http.Client{
        CheckRedirect: req.GetRedirectFunc(),
    }
    httpreq, err := http.NewRequest(req.GetMethod(), req.GetUrl(), strings.NewReader(req.GetPostdata()))
    if header := req.GetHeader(); header != nil {
        httpreq.Header = req.GetHeader()
    }
    if cookies := req.GetCookies(); cookies != nil {
        for i := range cookies {
            httpreq.AddCookie(cookies[i])
        }
    }

    var resp *http.Response
    if resp, err = client.Do(httpreq); err != nil {
        if e, ok := err.(*url.Error); ok && e.Err != nil && e.Err.Error() == "normal" {
            //  normal
        } else {
            mlog.LogInst().LogError(err.Error())
            p.SetStatus(true, err.Error())
            return p, ""
        }
    }

    p.SetHeader(resp.Header)
    p.SetCookies(resp.Cookies())

    // get converter to utf-8
    bodyStr := this.changeCharsetEncodingAuto(resp.Header.Get("Content-Type"), resp.Body)

    defer resp.Body.Close()
    return p, bodyStr
}

func (this *HttpDownloader) downloadHtml(p *page.Page, req *request.Request) *page.Page {
    var err error