Method processQueue

fetch.go:264–348

processQueue runs the queue in its own goroutine.

()

// processQueue runs the queue in its own goroutine.
func (f *Fetcher) processQueue() {
loop:
	for v := range f.q.ch {
		if v == nil {
			// Special case, when the Queue is closed, a nil command is sent, use this
			// indicator to check for the closed signal, instead of looking on every loop.
			select {
			case <-f.q.closed:
				// Close signal, exit loop
				break loop
			default:
				// Keep going
			}
		}
		select {
		case <-f.q.cancelled:
			// queue got cancelled, drain
			continue
		default:
			// go on
		}

		// Get the URL to enqueue
		u := v.URL()

		// Check if a channel is already started for this host
		f.mu.Lock()
		in, ok := f.hosts[u.Host]
		if !ok {
			// Start a new channel and goroutine for this host.

			var rob *url.URL
			if !f.DisablePoliteness {
				// Must send the robots.txt request.
				rob = u.ResolveReference(robotsTxtParsedPath)
			}

			// Create the infinite queue: the in channel to send on, and the out channel
			// to read from in the host's goroutine, and add to the hosts map
			var out chan Command
			in, out = make(chan Command, 1), make(chan Command, 1)
			f.hosts[u.Host] = in
			f.mu.Unlock()
			f.q.wg.Add(1)
			// Start the infinite queue goroutine for this host
			go sliceIQ(in, out)
			// Start the working goroutine for this host
			go f.processChan(out, u.Host)

			if !f.DisablePoliteness {
				// Enqueue the robots.txt request first.
				in <- robotCommand{&Cmd{U: rob, M: "GET"}}
			}
		} else {
			f.mu.Unlock()
		}
		// Send the request
		in <- v