Functions — github.com/CrawlScript/WebCollector

hub / github.com/CrawlScript/WebCollector / functions

Functions510 in github.com/CrawlScript/WebCollector

⨍ Functions 510◇ Types & classes 97

↓ 1 callersMethodmerge

()

src/main/java/cn/edu/hfut/dmic/webcollector/crawldb/DBManager.java:53

↓ 1 callersMethodmeta

(JsonObject metaData)

src/main/java/cn/edu/hfut/dmic/webcollector/model/CrawlDatums.java:122

↓ 1 callersMethodmetaAsBoolean

(String key)

src/main/java/cn/edu/hfut/dmic/webcollector/model/CrawlDatum.java:204

↓ 1 callersMethodmetaAsBoolean

(String key)

src/main/java/cn/edu/hfut/dmic/webcollector/model/MetaGetter.java:9

↓ 1 callersMethodmetaAsDouble

(String key)

src/main/java/cn/edu/hfut/dmic/webcollector/model/CrawlDatum.java:209

↓ 1 callersMethodmetaAsDouble

(String key)

src/main/java/cn/edu/hfut/dmic/webcollector/model/MetaGetter.java:10

↓ 1 callersMethodmetaAsInt

(String key)

src/main/java/cn/edu/hfut/dmic/webcollector/model/CrawlDatum.java:199

↓ 1 callersMethodmetaAsLong

(String key)

src/main/java/cn/edu/hfut/dmic/webcollector/model/CrawlDatum.java:214

↓ 1 callersMethodmyMethod

()

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoExceptionCrawler.java:26

↓ 1 callersMethodnextWithoutFilter

()

src/main/java/cn/edu/hfut/dmic/webcollector/crawldb/Generator.java:80

↓ 1 callersMethodobj

()

src/main/java/cn/edu/hfut/dmic/webcollector/model/Page.java:520

↓ 1 callersMethodopen

()

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/berkeley/BerkeleyDBManager.java:113

↓ 1 callersMethodparseLink

(Page page, CrawlDatums next)

src/main/java/cn/edu/hfut/dmic/webcollector/fetcher/VisitorMethodDispatcher.java:240

↓ 1 callersMethodrandomProxy

()

src/main/java/cn/edu/hfut/dmic/webcollector/net/Proxies.java:42

↓ 1 callersMethodread

(File file)

src/main/java/cn/edu/hfut/dmic/webcollector/util/FileUtils.java:109

↓ 1 callersMethodregisterOtherConfigurations

()

src/main/java/cn/edu/hfut/dmic/webcollector/crawler/Crawler.java:81

↓ 1 callersMethodsetContentElement

(Element contentElement)

src/main/java/cn/edu/hfut/dmic/contentextractor/News.java:83

↓ 1 callersMethodsetDBManager

设置任务管理器 @param dbManager 任务管理器

src/main/java/cn/edu/hfut/dmic/webcollector/crawler/Crawler.java:449

↓ 1 callersMethodsetExecuteCount

(int executeCount)

src/main/java/cn/edu/hfut/dmic/webcollector/model/CrawlDatum.java:176

↓ 1 callersMethodsetExecutor

(Executor executor)

src/main/java/cn/edu/hfut/dmic/webcollector/fetcher/Fetcher.java:74

↓ 1 callersMethodsetFilter

(GeneratorFilter filter)

src/main/java/cn/edu/hfut/dmic/webcollector/crawldb/Generator.java:110

↓ 1 callersMethodsetNextFilter

(NextFilter nextFilter)

src/main/java/cn/edu/hfut/dmic/webcollector/crawler/Crawler.java:457

↓ 1 callersMethodsetThreads

设置线程数 @param threads 线程数

src/main/java/cn/edu/hfut/dmic/webcollector/crawler/Crawler.java:373

↓ 1 callersMethodsetTime

(String time)

src/main/java/cn/edu/hfut/dmic/contentextractor/News.java:70

↓ 1 callersMethodsetTitle

(String title)

src/main/java/cn/edu/hfut/dmic/contentextractor/News.java:47

↓ 1 callersMethodsetUrl

(String url)

src/main/java/cn/edu/hfut/dmic/contentextractor/News.java:39

↓ 1 callersMethodstop

停止爬取

src/main/java/cn/edu/hfut/dmic/webcollector/fetcher/Fetcher.java:395

↓ 1 callersMethodstopFeeder

()

src/main/java/cn/edu/hfut/dmic/webcollector/fetcher/Fetcher.java:145

↓ 1 callersMethodtoString

()

src/main/java/cn/edu/hfut/dmic/webcollector/crawler/Crawler.java:461

↓ 1 callersMethodtype

(String type)

src/main/java/cn/edu/hfut/dmic/webcollector/model/CrawlDatums.java:205

↓ 1 callersMethodwriteFetchSegment

(CrawlDatum fetchDatum)

src/main/java/cn/edu/hfut/dmic/webcollector/crawldb/SegmentWriter.java:32

↓ 1 callersMethodwriteParseSegment

(CrawlDatums parseDatums)

src/main/java/cn/edu/hfut/dmic/webcollector/crawldb/SegmentWriter.java:34

MethodAbuyunDynamicProxyRequester

(String proxyUser, String proxyPass)

src/main/java/cn/edu/hfut/dmic/webcollector/example/AbuyunDynamicProxyRequester.java:43

MethodAutoParseCrawler

(boolean autoParse)

src/main/java/cn/edu/hfut/dmic/webcollector/crawler/AutoParseCrawler.java:52

MethodBerkeleyCrawler

(String crawlPath, Executor executor)

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/berkeley/BerkeleyCrawler.java:29

MethodBerkeleyDBManager

(String crawlPath)

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/berkeley/BerkeleyDBManager.java:51

MethodBerkeleyDBReader

(String crawlPath)

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/berkeley/BerkeleyDBReader.java:46

MethodBerkeleyGenerator

(Environment env)

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/berkeley/BerkeleyGenerator.java:49

MethodBreadthCrawler

构造一个基于RocksDB的爬虫 RocksDB文件夹为crawlPath，crawlPath中维护了历史URL等信息不同任务不要使用相同的crawlPath 两个使用相同crawlPath的爬虫并行爬取会产生错误 @param crawlPath RocksDB使用的文件夹 @param au

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/rocks/BreadthCrawler.java:43

MethodBreadthCrawler

构造一个基于伯克利DB的爬虫伯克利DB文件夹为crawlPath，crawlPath中维护了历史URL等信息不同任务不要使用相同的crawlPath 两个使用相同crawlPath的爬虫并行爬取会产生错误 @param crawlPath 伯克利DB使用的文件夹 @param autoPars

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/berkeley/BreadthCrawler.java:43

MethodContentExtractor

(Document doc)

src/main/java/cn/edu/hfut/dmic/contentextractor/ContentExtractor.java:49

MethodCounter

()

src/main/java/cn/edu/hfut/dmic/webcollector/util/Counter.java:29

MethodCrawlDatum

()

src/main/java/cn/edu/hfut/dmic/webcollector/model/CrawlDatum.java:68

MethodCrawlDatums

()

src/main/java/cn/edu/hfut/dmic/webcollector/model/CrawlDatums.java:35

MethodCrawler

()

src/main/java/cn/edu/hfut/dmic/webcollector/crawler/Crawler.java:41

MethodDefaultConfigured

()

src/main/java/cn/edu/hfut/dmic/webcollector/conf/DefaultConfigured.java:24

MethodDemoAnnotatedAutoNewsCrawler

@param crawlPath crawlPath is the path of the directory which maintains information of this crawler @param autoParse if autoParse is

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoAnnotatedAutoNewsCrawler.java:19

MethodDemoAnnotatedBingCrawler

(String keyword, int pageNum)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoAnnotatedBingCrawler.java:54

MethodDemoAnnotatedDepthCrawler

(String crawlPath, boolean autoParse)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoAnnotatedDepthCrawler.java:34

MethodDemoAnnotatedManualNewsCrawler

@param crawlPath crawlPath is the path of the directory which maintains information of this crawler @param autoParse if autoParse is

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoAnnotatedManualNewsCrawler.java:19

MethodDemoAnnotatedMatchTypeCrawler

()

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoAnnotatedMatchTypeCrawler.java:61

MethodDemoAnnotatedRedirectCrawler

(String keyword, int pageNum)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoAnnotatedRedirectCrawler.java:35

MethodDemoAutoNewsCrawler

@param crawlPath crawlPath is the path of the directory which maintains information of this crawler @param autoParse if autoParse is

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoAutoNewsCrawler.java:19

MethodDemoBingCrawler

(String keyword, int pageNum)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoBingCrawler.java:55

MethodDemoCookieCrawler

(String crawlPath)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoCookieCrawler.java:38

MethodDemoExceptionCrawler

@param crawlPath crawlPath is the path of the directory which maintains information of this crawler @param autoParse if autoParse is

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoExceptionCrawler.java:20

MethodDemoHashSetNextFilter

(String crawlPath)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoHashSetNextFilter.java:44

MethodDemoManualNewsCrawler

@param crawlPath crawlPath is the path of the directory which maintains information of this crawler @param autoParse if autoParse is

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoManualNewsCrawler.java:19

MethodDemoNextFilter

(String crawlPath, boolean autoParse)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoNextFilter.java:44

MethodDemoPostCrawler

假设我们要爬取三个链接 1)http://www.A.com/index.php 需要POST，并需要POST表单数据username:John 2)http://www.B.com/index.php?age=10 需要POST，数据直接在URL中，不需要附带数据 3)http://www.C.

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoPostCrawler.java:53

MethodDemoRandomProxyCrawler

(String crawlPath)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoRandomProxyCrawler.java:67

MethodDemoRedirectCrawler

(String keyword, int pageNum)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoRedirectCrawler.java:38

MethodFetchItem

(CrawlDatum datum)

src/main/java/cn/edu/hfut/dmic/webcollector/fetcher/Fetcher.java:85

MethodFileIdGenerator

(File baseDir)

src/main/java/cn/edu/hfut/dmic/webcollector/util/FileIdGenerator.java:46

MethodFileSystemOutput

(String root)

src/main/java/cn/edu/hfut/dmic/webcollector/util/FileSystemOutput.java:39

MethodGenerator

()

src/main/java/cn/edu/hfut/dmic/webcollector/crawldb/Generator.java:40

MethodInvalidAnnotatedVisitorMethodException

(String message)

src/main/java/cn/edu/hfut/dmic/webcollector/fetcher/Visitor.java:108

MethodLinks

()

src/main/java/cn/edu/hfut/dmic/webcollector/model/Links.java:38

MethodMyRequester

()

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoRandomProxyCrawler.java:32

MethodMysqlHelper

(String url, String username, String password, int initialSize, int maxPoolSize)

src/main/java/cn/edu/hfut/dmic/webcollector/util/MysqlHelper.java:36

MethodOkHttpRequester

()

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/net/OkHttpRequester.java:81

MethodPage

(CrawlDatum datum, String contentType, byte[] content)

src/main/java/cn/edu/hfut/dmic/webcollector/model/Page.java:323

MethodQueueFeeder

(FetchQueue queue, DBManager dbManager, GeneratorFilter generatorFilter, int size)

src/main/java/cn/edu/hfut/dmic/webcollector/fetcher/Fetcher.java:138

MethodRamCrawler

()

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/ram/RamCrawler.java:30

MethodRamDBManager

(RamDB ramDB)

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/ram/RamDBManager.java:39

MethodRamGenerator

(RamDB ramDB)

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/ram/RamGenerator.java:34

MethodRegexRule

()

src/main/java/cn/edu/hfut/dmic/webcollector/util/RegexRule.java:31

MethodRocksCrawler

(String crawlPath, Executor executor)

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/rocks/RocksCrawler.java:29

MethodRocksDBManager

(String crawlPath)

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/rocks/RocksDBManager.java:53

MethodRocksDBReader

(String crawlPath)

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/rocks/RocksDBReader.java:42

MethodRocksGenerator

(String crawlPath)

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/rocks/RocksGenerator.java:38

MethodTutorialCrawler

(String crawlPath, boolean autoParse)

src/main/java/cn/edu/hfut/dmic/webcollector/example/TutorialCrawler.java:49

MethodVisitorMethodDispatcher

(Visitor visitor, boolean autoParse, RegexRule regexRule)

src/main/java/cn/edu/hfut/dmic/webcollector/fetcher/VisitorMethodDispatcher.java:56

MethodaddHttpProxy

(String ip, int port)

src/main/java/cn/edu/hfut/dmic/webcollector/net/Proxies.java:65

MethodaddSuccessCode

(int successCode)

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/net/OkHttpRequester.java:39

MethodafterParse

(Page page, CrawlDatums next)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoAnnotatedBingCrawler.java:137

MethodafterParse

(Page page, CrawlDatums next)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoAnnotatedDepthCrawler.java:58

Methodattrs

获取网页中满足指定css选择器的所有元素的指定属性的集合例如通过attrs("img[src]","abs:src")可获取网页中所有图片的链接 @param cssSelector @param attrName @return

src/main/java/cn/edu/hfut/dmic/webcollector/model/Page.java:138

Methodauthenticate

(Route route, Response response)

src/main/java/cn/edu/hfut/dmic/webcollector/example/AbuyunDynamicProxyRequester.java:57

Methodclear

()

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/rocks/RocksDBManager.java:206

Methodclear

()

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/berkeley/BerkeleyDBManager.java:215

Methodclear

()

src/main/java/cn/edu/hfut/dmic/webcollector/model/CrawlDatums.java:192

Methodclose

()

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/rocks/RocksDBReader.java:63

Methodclose

()

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/rocks/RocksGenerator.java:56

Methodclose

()

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/berkeley/BerkeleyGenerator.java:66

Methodclose

()

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/berkeley/BerkeleyDBReader.java:68

Methodclose

()

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/ram/RamDBManager.java:67

MethodcloseSegmentWriter

()

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/rocks/RocksDBManager.java:146

MethodcloseSegmentWriter

()

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/berkeley/BerkeleyDBManager.java:159

MethodcloseSegmentWriter

()

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/ram/RamDBManager.java:132

← previous next →201–300 of 510, ranked by callers