Get the robots.txt User-Agent-specific group.
(r robotCommand)
| 437 | |
| 438 | // Get the robots.txt User-Agent-specific group. |
| 439 | func (f *Fetcher) getRobotAgent(r robotCommand) *robotstxt.Group { |
| 440 | res, err := f.doRequest(r) |
| 441 | if err != nil { |
| 442 | // TODO: Ignore robots.txt request error? |
| 443 | fmt.Fprintf(os.Stderr, "fetchbot: error fetching robots.txt: %s\n", err) |
| 444 | return nil |
| 445 | } |
| 446 | if res.Body != nil { |
| 447 | defer res.Body.Close() |
| 448 | } |
| 449 | robData, err := robotstxt.FromResponse(res) |
| 450 | if err != nil { |
| 451 | // TODO : Ignore robots.txt parse error? |
| 452 | fmt.Fprintf(os.Stderr, "fetchbot: error parsing robots.txt: %s\n", err) |
| 453 | return nil |
| 454 | } |
| 455 | return robData.FindGroup(f.UserAgent) |
| 456 | } |
| 457 | |
| 458 | // Call the Handler for this Command. Closes the response's body. |
| 459 | func (f *Fetcher) visit(cmd Command, res *http.Response, err error) { |
no test coverage detected