Function CrawlByChrome

utils/util.go:256–318

使用chrome采集网页HTML

(urlStr string, bookIdentify string)

// 使用chrome采集网页HTML
func CrawlByChrome(urlStr string, bookIdentify string) (cont string, err error) {
	if strings.Contains(strings.ToLower(urlStr), "bookstack") {
		return
	}
	var (
		args   []string
		b      []byte
		folder string
	)

	name := beego.AppConfig.DefaultString("chrome", "chromium-browser")
	ok, _ := beego.AppConfig.Bool("puppeteer")
	selector, isScreenshot := ScreenShotProjects.Load(bookIdentify)
	if ok || isScreenshot {
		name = "node" // 读取截屏信息
		args = []string{"crawl.js", "--url", urlStr}
		if isScreenshot {
			folder = fmt.Sprintf("cache/screenshots/" + bookIdentify + "/" + MD5Sub16(urlStr))
			os.MkdirAll(folder, os.ModePerm)
			args = append(args, "--folder", folder, "--selector", selector.(string))
		}
	} else { // chrome
		args = []string{"--headless", "--disable-gpu", "--dump-dom", "--no-sandbox", urlStr}
	}
	cmd := exec.Command(name, args...)
	expire := 2 * httpTimeout
	if isScreenshot {
		expire = 10 * httpTimeout
	}
	time.AfterFunc(expire, func() {
		if cmd.Process != nil && cmd.Process.Pid != 0 {
			cmd.Process.Kill()
		}
	})

	b, err = cmd.Output()
	cont = string(b)

	if isScreenshot {
		pngFile := filepath.Join(folder, "screenshot.png")
		jsonFile := filepath.Join(folder, "screenshot.json")
		imagesMap := cropScreenshot(selector.(string), jsonFile, pngFile)
		if len(imagesMap) > 0 {
			doc, errDoc := goquery.NewDocumentFromReader(strings.NewReader(cont))
			if errDoc != nil {
				beego.Error(errDoc)
			} else {
				for ele, images := range imagesMap {
					doc.Find(ele).Each(func(i int, selection *goquery.Selection) {
						if img, ok := images[i]; ok {
							htmlStr := fmt.Sprintf(`<div><img src="$%v"/></div>`, img)
							selection.AfterHtml(htmlStr)
							selection.Remove()
						}
					})
				}
				cont, err = doc.Find("body").Html()
			}