Method processChan

fetch.go:351–436

Goroutine for a host's worker, processing requests for all its URLs.

(ch <-chan Command, hostKey string)

// Goroutine for a host's worker, processing requests for all its URLs.
func (f *Fetcher) processChan(ch <-chan Command, hostKey string) {
	var (
		agent *robotstxt.Group
		wait  <-chan time.Time
		ttl   <-chan time.Time
		delay = f.CrawlDelay
	)

loop:
	for {
		select {
		case <-f.q.cancelled:
			break loop
		case v, ok := <-ch:
			if !ok {
				// Terminate this goroutine, channel is closed
				break loop
			}

			// Wait for the prescribed delay
			if wait != nil {
				<-wait
			}

			// was it cancelled during the wait? check again
			select {
			case <-f.q.cancelled:
				break loop
			default:
				// go on
			}

			switch r, ok := v.(robotCommand); {
			case ok:
				// This is the robots.txt request
				agent = f.getRobotAgent(r)
				// Initialize the crawl delay
				if agent != nil && agent.CrawlDelay > 0 {
					delay = agent.CrawlDelay
				}
				wait = time.After(delay)

			case agent == nil || agent.Test(v.URL().Path):
				// Path allowed, process the request
				res, err := f.doRequest(v)
				f.visit(v, res, err)
				// No delay on error - the remote host was not reached
				if err == nil {
					wait = time.After(delay)
				} else {
					wait = nil
				}

			default:
				// Path disallowed by robots.txt
				f.visit(v, nil, ErrDisallowed)
				wait = nil
			}