Crawler is the core crawler engine.
| 26 | GetID() int // GetID returns the engine ID |
| 27 | } |
| 28 | crawler struct { |
| 29 | *spider.Spider // spider rule being executed |
| 30 | downloader.Downloader // shared downloader |
| 31 | pipeline.Pipeline // result collection and output pipeline |
| 32 | id int // engine ID |
| 33 | outType string // output type for pipeline |
| 34 | batchCap int // batch output capacity for pipeline |
| 35 | pause [2]int64 // [min request interval ms, max additional interval ms] |
| 36 | } |
| 37 | ) |
| 38 | |
| 39 | // New creates a new Crawler with the given ID, Downloader, and pipeline config. |
nothing calls this directly
no outgoing calls
no test coverage detected