Function LoadURL

query.go:94–137

LoadURL loads the HTML document from the specified URL. Default enabling gzip on a HTTP request.

(url string)

// LoadURL loads the HTML document from the specified URL. Default enabling gzip on a HTTP request.
func LoadURL(url string) (*html.Node, error) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	// Enable gzip compression.
	req.Header.Add("Accept-Encoding", "gzip")
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	var reader io.ReadCloser

	defer func() {
		if reader != nil {
			reader.Close()
		}
	}()
	encoding := resp.Header.Get("Content-Encoding")
	switch encoding {
	case "gzip":
		gzipReader, err := gzip.NewReader(resp.Body)
		if err != nil {
			return nil, err
		}
		reader = gzipReader
	case "deflate":
		zlibReader, err := zlib.NewReader(resp.Body)
		if err != nil {
			return nil, err
		}
		reader = zlibReader
	case "":
		reader = resp.Body
	default:
		return nil, fmt.Errorf("%s compression is not support", encoding)
	}

	r, err := charset.NewReader(reader, resp.Header.Get("Content-Type"))
	if err != nil {
		return nil, err
	}
	return html.Parse(r)
}

// LoadDoc loads the HTML document from the specified file path.
func LoadDoc(path string) (*html.Node, error) {