Function CrawlHtml2Markdown

utils/util.go:386–562

采集HTML并把相对链接和相对图片 内容类型，contType:0表示markdown，1表示html，2表示文本 force:是否是强力采集 intelligence:是否是智能提取，智能提取，使用html2article，否则提取body diySelecter:自定义选择器 注意：由于参数问题，采集并下载图片的话，在headers中加上key为"project"的字段，值为书籍的标识

(urlstr string, contType int, force bool, intelligence int, diySelector string, excludeSelector []string, links map[string]string, headers ...map[string]string)

// diySelecter:自定义选择器
// 注意：由于参数问题，采集并下载图片的话，在headers中加上key为"project"的字段，值为书籍的标识
func CrawlHtml2Markdown(urlstr string, contType int, force bool, intelligence int, diySelector string, excludeSelector []string, links map[string]string, headers ...map[string]string) (cont string, err error) {
	defer func() {
		if r := recover(); r != nil {
			err = errors.New(fmt.Sprint(r))
			return
		}
	}()

	//记录已经存在了的图片，避免重复图片出现重复采集的情况
	var existImage bool

	from := "\r\n<!-- 原文：" + urlstr + " -->"

	imageMap := make(map[string]string)

	if strings.Contains(urlstr, "bookstack.cn") {
		return
	}

	// 默认记录到数据库中的图片路径
	//save := src
	project := ""
	for _, header := range headers {
		if val, ok := header["project"]; ok {
			project = val
		}
	}

	if force { //强力模式
		cont, err = CrawlByChrome(urlstr, project)
	} else {
		req := util.BuildRequest("get", urlstr, "", "", "", true, false, headers...).SetTimeout(httpTimeout, httpTimeout)
		cont, err = req.String()
		if err != nil && transfer != "" {
			req := util.BuildRequest("get", transfer+urlstr, urlstr, "", "", true, false, headers...).SetTimeout(httpTimeout, httpTimeout)
			cont, err = req.String()
		}
	}

	cont = strings.Replace(cont, "¶", "", -1)

	if err != nil {
		return
	}

	var doc *goquery.Document
	doc, err = goquery.NewDocumentFromReader(strings.NewReader(cont))
	if err != nil {
		return
	}

	//遍历a标签替换相对链接
	doc.Find("a").Each(func(i int, selection *goquery.Selection) {
		//存在href，且不以http://和https://开头
		if href, ok := selection.Attr("href"); ok {
			href = JoinURL(urlstr, href)

			if link, ok := links[strings.TrimRight(href, "/")]; ok {