Collector collects spider results and writes them to the configured output backend.
| 16 | |
| 17 | // Collector collects spider results and writes them to the configured output backend. |
| 18 | type Collector struct { |
| 19 | *spider.Spider |
| 20 | DataChan chan data.DataCell |
| 21 | FileChan chan data.FileCell |
| 22 | dataBuf []data.DataCell |
| 23 | outType string |
| 24 | batchCap int |
| 25 | dataBatch uint64 |
| 26 | fileBatch uint64 |
| 27 | wait sync.WaitGroup |
| 28 | sum [4]uint64 |
| 29 | dataSumLock sync.RWMutex |
| 30 | fileSumLock sync.RWMutex |
| 31 | } |
| 32 | |
| 33 | // NewCollector creates a new Collector for the given spider. |
| 34 | func NewCollector(sp *spider.Spider, outType string, batchCap int) *Collector { |
nothing calls this directly
no outgoing calls
no test coverage detected