MCPcopy Create free account

hub / github.com/JFanZhao/spider / functions

Functions80 in github.com/JFanZhao/spider

↓ 13 callersMethodput
添加一条数据 @param tableName @param row @param columnFamily @param column @param data @throws IOException
src/main/java/cn/ivan/spider/utils/HBaseUtils.java:204
↓ 12 callersMethodadd
(String nextUrl)
src/main/java/cn/ivan/spider/reposity/Repository.java:9
↓ 7 callersMethodpoll
()
src/main/java/cn/ivan/spider/reposity/Repository.java:5
↓ 5 callersMethodgetText
获取指定标签的值 @param tagNode @param xPath @return
src/main/java/cn/ivan/spider/utils/HtmlUtils.java:39
↓ 4 callersMethodaddField
(String key,String value)
src/main/java/cn/ivan/spider/domain/Page.java:59
↓ 4 callersMethodgetUrl
()
src/main/java/cn/ivan/spider/domain/Page.java:62
↓ 4 callersMethodstart
()
src/main/java/cn/ivan/spider/Spider.java:60
↓ 2 callersMethodaddHigh
(String nextUrl)
src/main/java/cn/ivan/spider/reposity/Repository.java:7
↓ 2 callersMethodaddUrl
(String url)
src/main/java/cn/ivan/spider/domain/Page.java:38
↓ 2 callersMethoddownload
(String url)
src/main/java/cn/ivan/spider/download/Downloadable.java:6
↓ 2 callersMethodexecute
(JobExecutionContext arg0)
src/main/java/cn/ivan/spider/UrlJob.java:17
↓ 2 callersMethodgetAttributeByAttr
获取指定标签指定属性的值 @param tagNode @param Xpath @param attr @return
src/main/java/cn/ivan/spider/utils/HtmlUtils.java:49
↓ 2 callersMethodgetContent
根据url获取页面内容 @param url @return
src/main/java/cn/ivan/spider/utils/PageUtils.java:29
↓ 2 callersMethodgetTagNodeByXpath
获取指定的标签node @param tagNode @param xPath @return
src/main/java/cn/ivan/spider/utils/HtmlUtils.java:19
↓ 2 callersMethodgetTopDomain
获取url的顶级域名 @param url @return
src/main/java/cn/ivan/spider/utils/DomainUtils.java:15
↓ 2 callersMethodgetUrls
()
src/main/java/cn/ivan/spider/domain/Page.java:41
↓ 2 callersMethodgetValues
()
src/main/java/cn/ivan/spider/domain/Page.java:56
↓ 2 callersMethodprocess
(Page page)
src/main/java/cn/ivan/spider/process/Processable.java:6
↓ 2 callersMethodsetProcessable
(Processable processable)
src/main/java/cn/ivan/spider/Spider.java:145
↓ 2 callersMethodsetStoreable
(Storeable storeable)
src/main/java/cn/ivan/spider/Spider.java:112
↓ 2 callersMethodsetUrl
(String url)
src/main/java/cn/ivan/spider/domain/Page.java:66
↓ 2 callersMethodsleep
(long millons)
src/main/java/cn/ivan/spider/utils/SleepUtils.java:9
↓ 2 callersMethodstart
让当前进程一直运行
src/main/java/cn/ivan/spider/SpiderWatcher.java:78
↓ 2 callersMethodstore
(Page page)
src/main/java/cn/ivan/spider/store/Storeable.java:6
↓ 1 callersMethodaddHigh
(String nextUrl)
src/main/java/cn/ivan/spider/reposity/RandomRedisRepository.java:31
↓ 1 callersMethodaddHigh
@Title: addHigh @Description: 向队列仓库中添加url,在添加之前先截取url中的顶级域名,当作key值 @param nextUrl @throws
src/main/java/cn/ivan/spider/reposity/RandomQueueRepository.java:48
↓ 1 callersMethodcheck
配置检查
src/main/java/cn/ivan/spider/Spider.java:94
↓ 1 callersMethoddownload
下载页面 @param url @return
src/main/java/cn/ivan/spider/Spider.java:125
↓ 1 callersMethodgetContent
()
src/main/java/cn/ivan/spider/domain/Page.java:70
↓ 1 callersMethodgetGoodId
()
src/main/java/cn/ivan/spider/domain/Page.java:50
↓ 1 callersMethodlrange
模拟redis的lrange方法 @param key @param start @param end @return
src/main/java/cn/ivan/spider/utils/RedisUtils.java:33
↓ 1 callersMethodprocess
解析页面 @param page
src/main/java/cn/ivan/spider/Spider.java:136
↓ 1 callersMethodprocessProduct
解析商品详细信息 @param page @param rootNode
src/main/java/cn/ivan/spider/process/JDProcessable.java:50
↓ 1 callersMethodsetContent
(String content)
src/main/java/cn/ivan/spider/domain/Page.java:74
↓ 1 callersMethodsetDownloadable
(Downloadable downloadable)
src/main/java/cn/ivan/spider/Spider.java:115
↓ 1 callersMethodsetGoodId
(String goodId)
src/main/java/cn/ivan/spider/domain/Page.java:53
↓ 1 callersMethodsetRepository
(Repository repository)
src/main/java/cn/ivan/spider/Spider.java:152
↓ 1 callersMethodsetSeedUrl
(String url)
src/main/java/cn/ivan/spider/Spider.java:163
↓ 1 callersMethodstore
(Page page)
src/main/java/cn/ivan/spider/Spider.java:159
MethodHBaseUtils
构造函数 加载配置
src/main/java/cn/ivan/spider/utils/HBaseUtils.java:58
MethodRedisUtils
构造方法 初始化redis配置信息
src/main/java/cn/ivan/spider/utils/RedisUtils.java:18
MethodSpider
()
src/main/java/cn/ivan/spider/Spider.java:42
MethodSpiderWatcher
()
src/main/java/cn/ivan/spider/SpiderWatcher.java:20
Methodadd
(String lowKey, String url)
src/main/java/cn/ivan/spider/utils/RedisUtils.java:40
Methodadd
向低优先级队列中存入url
src/main/java/cn/ivan/spider/reposity/QueueRepository.java:41
Methodadd
(String nextUrl)
src/main/java/cn/ivan/spider/reposity/RedisRepository.java:19
Methodadd
(String nextUrl)
src/main/java/cn/ivan/spider/reposity/RandomRedisRepository.java:41
Methodadd
(String nextUrl)
src/main/java/cn/ivan/spider/reposity/RandomQueueRepository.java:58
MethodaddHigh
向高优先级队列中存入url
src/main/java/cn/ivan/spider/reposity/QueueRepository.java:35
MethodaddHigh
(String nextUrl)
src/main/java/cn/ivan/spider/reposity/RedisRepository.java:15
MethodcreateTable
创建一张表 @param tableName @param column @throws Exception
src/main/java/cn/ivan/spider/utils/HBaseUtils.java:240
MethoddeleteOneRecord
删除一条记录 @param tableName @param rowKey
src/main/java/cn/ivan/spider/utils/HBaseUtils.java:159
MethoddeleteTable
删除表 @param tableName @throws Exception
src/main/java/cn/ivan/spider/utils/HBaseUtils.java:142
Methoddownload
(String url)
src/main/java/cn/ivan/spider/download/HttpClientDownable.java:8
MethodgetALLData
获取表中所有数据 @param tableName @throws Exception
src/main/java/cn/ivan/spider/utils/HBaseUtils.java:176
MethodgetALLTable
查询所有表名 @return @throws Exception
src/main/java/cn/ivan/spider/utils/HBaseUtils.java:219
MethodgetDownloadable
()
src/main/java/cn/ivan/spider/Spider.java:155
MethodgetProcessable
()
src/main/java/cn/ivan/spider/Spider.java:141
MethodgetRepository
()
src/main/java/cn/ivan/spider/Spider.java:149
MethodgetRowFilter
rowFilter 的使用 @param tableName @param reg @throws Exception
src/main/java/cn/ivan/spider/utils/HBaseUtils.java:105
MethodgetScanData
扫描数据 @param tableName @param family @param qualifier @throws Exception
src/main/java/cn/ivan/spider/utils/HBaseUtils.java:122
MethodgetStoreable
()
src/main/java/cn/ivan/spider/Spider.java:108
Methodmain
(String[] args)
src/main/java/cn/ivan/spider/Spider.java:166
Methodmain
(String[] args)
src/main/java/cn/ivan/spider/UrlManager.java:17
Methodmain
(String[] args)
src/main/java/cn/ivan/spider/SpiderWatcher.java:70
Methodmain
(String[] args)
src/main/java/cn/ivan/spider/utils/HBaseUtils.java:77
Methodpoll
(String key)
src/main/java/cn/ivan/spider/utils/RedisUtils.java:47
Methodpoll
从url仓库中取url,先从高优先级队列中取,如果高优先级队列中没有url,则从低优先级队列中取
src/main/java/cn/ivan/spider/reposity/QueueRepository.java:25
Methodpoll
()
src/main/java/cn/ivan/spider/reposity/RedisRepository.java:7
Methodpoll
()
src/main/java/cn/ivan/spider/reposity/RandomRedisRepository.java:23
Methodpoll
@Title: poll @Description: 随机获取一个url @return @throws
src/main/java/cn/ivan/spider/reposity/RandomQueueRepository.java:34
Methodprocess
当监视器发现监控的父节点下面的子节点发生变化的时候,会调用这个方法 注意:watcher监视器是单次有效的,注册一次,只能使用一次,需要想要一直使用,则需要重复注册
src/main/java/cn/ivan/spider/SpiderWatcher.java:41
Methodprocess
(Page page)
src/main/java/cn/ivan/spider/process/JDProcessable.java:18
Methodrun
()
src/main/java/cn/ivan/spider/Spider.java:67
MethodsetUrls
(List<String> urls)
src/main/java/cn/ivan/spider/domain/Page.java:44
MethodsetValues
(Map<String, String> values)
src/main/java/cn/ivan/spider/domain/Page.java:47
Methodstore
(Page page)
src/main/java/cn/ivan/spider/store/ConsoleStoreable.java:7
Methodstore
(Page page)
src/main/java/cn/ivan/spider/store/HBaseStoreable.java:12
Methodtest
()
src/test/java/cn/ivan/spider/CuratorTest.java:13
MethodtestName
()
src/test/java/cn/ivan/spider/SpiderTest.java:12