NewCrawlerPool creates a new crawler pool with the given Downloader.
(dl downloader.Downloader)
| 35 | |
| 36 | // NewCrawlerPool creates a new crawler pool with the given Downloader. |
| 37 | func NewCrawlerPool(dl downloader.Downloader) CrawlerPool { |
| 38 | return &cq{ |
| 39 | status: status.RUN, |
| 40 | dl: dl, |
| 41 | all: make([]Crawler, 0, config.Conf().CrawlsCap), |
| 42 | } |
| 43 | } |
| 44 | |
| 45 | // SetPipelineConfig sets the output type and batch capacity for new crawlers. |
| 46 | func (cq *cq) SetPipelineConfig(outType string, batchCap int) { |