MCPcopy Create free account

hub / github.com/aizk/go-crawler / functions

Functions53 in github.com/aizk/go-crawler

⨍ Functions 53◇ Types & classes 17
↓ 2 callersMethodStart
()
pipeline/pipeline.go:4
↓ 2 callersMethodoutputData
()
pipeline/collector.go:62
↓ 1 callersMethodCollectFiles
停止收集
pipeline/pipeline.go:6
↓ 1 callersMethodCollectItems
(items []Item)
pipeline/pipeline.go:7
↓ 1 callersMethodDownload
(spider *spider.Spider, request *request.Request)
download/download.go:11
↓ 1 callersMethodGetFiles
()
page/page.go:14
↓ 1 callersMethodGetItems
解析返回对象中的文件和数据
page/page.go:13
↓ 1 callersMethodGetOneRequest
()
crawler/crawler.go:136
↓ 1 callersMethodGetRule
查找对应的规则
spider/spider.go:34
↓ 1 callersMethodGetRuleName
()
request/request.go:7
↓ 1 callersMethodInit
(spider *spider.Spider)
crawler/crawler.go:36
↓ 1 callersFunctionNewMatrix
(spiderName string)
scheduler/matrix.go:13
↓ 1 callersMethodParse
(ruleName string)
page/page.go:12
↓ 1 callersMethodProcess
(req *request.Request)
crawler/crawler.go:91
↓ 1 callersMethodPull
()
scheduler/matrix.go:36
↓ 1 callersMethodPush
(req *request.Request)
scheduler/matrix.go:21
↓ 1 callersMethodReleaseOneThread
()
crawler/crawler.go:143
↓ 1 callersMethodRun
初始化
crawler/crawler.go:15
↓ 1 callersMethodStop
()
crawler/crawler.go:16
↓ 1 callersMethodUseOneThread
()
crawler/crawler.go:140
↓ 1 callersMethodgetRule
(ruleName string)
page/page.go:79
↓ 1 callersMethodrun
()
crawler/crawler.go:63
MethodCanStop
()
crawler/crawler.go:18
MethodCanStop
()
crawler/crawler.go:132
MethodCollectFiles
(files []File)
pipeline/collector.go:47
MethodCollectItems
(items []Item)
pipeline/collector.go:55
MethodDownload
(spider *spider.Spider, request *request.Request)
download/download.go:17
MethodGetFiles
()
page/page.go:60
MethodGetID
()
crawler/crawler.go:17
MethodGetID
()
crawler/crawler.go:124
MethodGetItems
()
page/page.go:56
FunctionGetPage
(s *spider.Spider, request *request.Request)
page/page.go:36
MethodInit
(spider *spider.Spider)
crawler/crawler.go:14
FunctionInsertGlobalSchedulerMatrix
(spiderName string)
scheduler/shceduler.go:23
FunctionNew
(id int)
crawler/crawler.go:29
MethodParse
(ruleName string)
page/page.go:64
MethodPullRequest
取出请求
spider/spider.go:64
MethodPushRequest
添加请求
spider/spider.go:59
MethodRegister
()
spider/spider.go:29
FunctionReleasePage
(page *Page)
page/page.go:43
MethodRun
()
crawler/crawler.go:44
MethodSaveFailure
保存失败记录
spider/spider.go:74
MethodSaveSuccess
保持成功记录
spider/spider.go:69
MethodSetID
(id int)
crawler/crawler.go:128
MethodSpiderMatrixInit
初始化队列
spider/spider.go:53
MethodStart
()
spider/spider.go:39
MethodStart
()
pipeline/collector.go:19
MethodStop
()
spider/spider.go:46
MethodStop
启动收集
pipeline/pipeline.go:5
MethodStop
()
pipeline/collector.go:43
MethodStop
()
crawler/crawler.go:120
Functioninit
()
main.go:9
Functionmain
()
main.go:13