Functions — github.com/CrawlScript/WebCollector

hub / github.com/CrawlScript/WebCollector / functions

Functions510 in github.com/CrawlScript/WebCollector

⨍ Functions 510◇ Types & classes 97

MethodconnectFailed

(URI uri, SocketAddress sa, IOException ioe)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoRandomProxyCrawler.java:59

Methodcopy

(File origin, File newfile)

src/main/java/cn/edu/hfut/dmic/webcollector/util/FileUtils.java:45

MethodcopyDefault

()

src/main/java/cn/edu/hfut/dmic/webcollector/conf/Configuration.java:216

MethodcopyMeta

()

src/main/java/cn/edu/hfut/dmic/webcollector/model/CrawlDatum.java:219

MethodcopyMeta

()

src/main/java/cn/edu/hfut/dmic/webcollector/model/Page.java:477

MethodcreateGenerator

()

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/rocks/RocksDBManager.java:214

MethodcreateGenerator

()

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/berkeley/BerkeleyDBManager.java:223

MethodcreateGenerator

()

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/ram/RamDBManager.java:57

MethodcreateOkHttpClientBuilder

()

src/main/java/cn/edu/hfut/dmic/webcollector/example/AbuyunDynamicProxyRequester.java:49

MethodcreateOkHttpClientBuilder

()

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoRandomProxyCrawler.java:40

MethodcreateRequestBuilder

(CrawlDatum crawlDatum)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoPostCrawler.java:65

MethodcreateRequestBuilder

(CrawlDatum crawlDatum)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoCookieCrawler.java:26

MethodeditDistance

(String word1, String word2)

src/main/java/cn/edu/hfut/dmic/contentextractor/ContentExtractor.java:399

Methodexecute

(CrawlDatum datum, CrawlDatums next)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoSeleniumCrawler.java:47

Methodexecute

(CrawlDatum datum, CrawlDatums next)

src/main/java/cn/edu/hfut/dmic/webcollector/crawler/AutoParseCrawler.java:85

Methodfilter

(CrawlDatum nextItem, CrawlDatum referer)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoNextFilter.java:74

Methodfilter

(CrawlDatum nextItem, CrawlDatum referer)

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/nextfilter/HashSetNextFilter.java:30

Methodfilter

(CrawlDatum datum)

src/main/java/cn/edu/hfut/dmic/webcollector/crawldb/StatusGeneratorFilter.java:7

MethodfilterByRegex

(RegexRule regexRule)

src/main/java/cn/edu/hfut/dmic/webcollector/model/Links.java:76

MethodgetBoolean

(String key)

src/main/java/cn/edu/hfut/dmic/webcollector/conf/Configuration.java:61

MethodgetConf

()

src/main/java/cn/edu/hfut/dmic/webcollector/conf/CommonConfigured.java:26

MethodgetContentByDoc

(Document doc)

src/main/java/cn/edu/hfut/dmic/contentextractor/ContentExtractor.java:463

MethodgetContentByUrl

(String url)

src/main/java/cn/edu/hfut/dmic/contentextractor/ContentExtractor.java:481

MethodgetContentElement

()

src/main/java/cn/edu/hfut/dmic/contentextractor/News.java:79

MethodgetContentElementByUrl

(String url)

src/main/java/cn/edu/hfut/dmic/contentextractor/ContentExtractor.java:456

MethodgetDBManager

()

src/main/java/cn/edu/hfut/dmic/webcollector/fetcher/Fetcher.java:417

MethodgetDBManager

返回任务管理器 @return 任务管理器

src/main/java/cn/edu/hfut/dmic/webcollector/crawler/Crawler.java:440

MethodgetDataSource

()

src/main/java/cn/edu/hfut/dmic/webcollector/util/MysqlHelper.java:61

MethodgetDouble

(String key)

src/main/java/cn/edu/hfut/dmic/webcollector/conf/Configuration.java:68

MethodgetException

()

src/main/java/cn/edu/hfut/dmic/webcollector/model/Page.java:429

MethodgetExecutor

()

src/main/java/cn/edu/hfut/dmic/webcollector/fetcher/Fetcher.java:70

MethodgetExecutor

获取每个爬取任务的最大执行次数 @return 每个爬取任务的最大执行次数

src/main/java/cn/edu/hfut/dmic/webcollector/crawler/Crawler.java:396

MethodgetFilter

()

src/main/java/cn/edu/hfut/dmic/webcollector/crawldb/Generator.java:106

MethodgetGeneratorFilter

()

src/main/java/cn/edu/hfut/dmic/webcollector/crawler/Crawler.java:333

MethodgetLong

(String key)

src/main/java/cn/edu/hfut/dmic/webcollector/conf/Configuration.java:65

MethodgetMaxExecuteCount

()

src/main/java/cn/edu/hfut/dmic/webcollector/conf/Configuration.java:112

MethodgetMaxExecuteCount

()

src/main/java/cn/edu/hfut/dmic/webcollector/crawler/Crawler.java:377

MethodgetMaxReceiveSize

()

src/main/java/cn/edu/hfut/dmic/webcollector/conf/Configuration.java:159

MethodgetMaxRedirect

()

src/main/java/cn/edu/hfut/dmic/webcollector/conf/Configuration.java:152

MethodgetNextFilter

()

src/main/java/cn/edu/hfut/dmic/webcollector/fetcher/Fetcher.java:426

MethodgetNextFilter

()

src/main/java/cn/edu/hfut/dmic/webcollector/crawler/Crawler.java:453

MethodgetRegexRule

获取正则规则 @return 正则规则

src/main/java/cn/edu/hfut/dmic/webcollector/crawler/AutoParseCrawler.java:126

MethodgetRequester

()

src/main/java/cn/edu/hfut/dmic/webcollector/crawler/AutoParseCrawler.java:157

MethodgetResponse

(String url)

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/net/OkHttpRequester.java:86

MethodgetResponse

(String url)

src/main/java/cn/edu/hfut/dmic/webcollector/net/CommonRequester.java:8

MethodgetSuccessCodeSet

()

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/net/OkHttpRequester.java:140

MethodgetTemplate

()

src/main/java/cn/edu/hfut/dmic/webcollector/util/MysqlHelper.java:53

MethodgetThreads

返回爬虫的线程数 @return 爬虫的线程数

src/main/java/cn/edu/hfut/dmic/webcollector/fetcher/Fetcher.java:404

MethodgetThreads

返回线程数 @return 线程数

src/main/java/cn/edu/hfut/dmic/webcollector/crawler/Crawler.java:364

MethodgetVisitor

获取Visitor @return Visitor

src/main/java/cn/edu/hfut/dmic/webcollector/crawler/AutoParseCrawler.java:144

Methodhead

(Node node, int i)

src/main/java/cn/edu/hfut/dmic/webcollector/util/JsoupUtils.java:37

Methodhead

(Node node, int i)

src/main/java/cn/edu/hfut/dmic/contentextractor/ContentExtractor.java:287

Methodinc

()

src/main/java/cn/edu/hfut/dmic/webcollector/util/Counter.java:37

MethodindexOf

(CrawlDatum datum)

src/main/java/cn/edu/hfut/dmic/webcollector/model/CrawlDatums.java:201

MethodindexOf

(String url)

src/main/java/cn/edu/hfut/dmic/webcollector/model/Links.java:192

MethodinitSegmentWriter

()

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/rocks/RocksDBManager.java:121

MethodinitSegmentWriter

()

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/berkeley/BerkeleyDBManager.java:137

MethodinitSegmentWriter

()

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/ram/RamDBManager.java:116

Methodinject

(CrawlDatum datum, boolean force)

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/rocks/RocksDBManager.java:71

Methodinject

(CrawlDatum datum, boolean force)

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/berkeley/BerkeleyDBManager.java:79

Methodinject

(CrawlDatum datum, boolean force)

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/ram/RamDBManager.java:71

Methodinject

(CrawlDatum datum, boolean force)

src/main/java/cn/edu/hfut/dmic/webcollector/crawldb/DBManager.java:49

MethodisAutoParse

()

src/main/java/cn/edu/hfut/dmic/webcollector/fetcher/VisitorMethodDispatcher.java:232

MethodisAutoParse

@return 返回是否自动抽取符合正则的链接并加入后续任务

src/main/java/cn/edu/hfut/dmic/webcollector/crawler/AutoParseCrawler.java:108

MethodisDBExists

()

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/rocks/RocksDBManager.java:200

MethodisDBExists

()

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/berkeley/BerkeleyDBManager.java:209

MethodisDBExists

()

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/ram/RamDBManager.java:44

MethodisEmpty

()

src/main/java/cn/edu/hfut/dmic/webcollector/util/RegexRule.java:48

MethodisResumable

返回是否断点爬取 @return 是否断点爬取

src/main/java/cn/edu/hfut/dmic/webcollector/crawler/Crawler.java:346

Methoditerator

()

src/main/java/cn/edu/hfut/dmic/webcollector/model/CrawlDatums.java:171

MethodjsonArray

()

src/main/java/cn/edu/hfut/dmic/webcollector/model/Page.java:109

Methodlist

()

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/rocks/RocksDBManager.java:57

Methodlist

()

src/main/java/cn/edu/hfut/dmic/webcollector/plugin/berkeley/BerkeleyDBManager.java:55

Methodmain

(String[] args)

src/test/java/cn/edu/hfut/dmic/webcollector/util/CrawlDatumTest.java:32

Methodmain

(String[] args)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoHashSetNextFilter.java:67

Methodmain

(String[] args)

src/main/java/cn/edu/hfut/dmic/webcollector/example/TutorialCrawler.java:87

Methodmain

(String[] args)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoRedirectCrawler.java:67

Methodmain

(String[] args)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoAnnotatedAutoNewsCrawler.java:61

Methodmain

(String[] args)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoAnnotatedRedirectCrawler.java:66

Methodmain

(String[] args)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoExceptionCrawler.java:43

Methodmain

(String[] args)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoBingCrawler.java:137

Methodmain

(String[] args)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoAutoNewsCrawler.java:60

Methodmain

(String[] args)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoTypeCrawler.java:75

Methodmain

@param args 参数 @throws Exception 异常

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoPostCrawler.java:131

Methodmain

(String[] args)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoAnnotatedBingCrawler.java:145

Methodmain

(String[] args)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoCookieCrawler.java:54

Methodmain

(String[] args)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoNextFilter.java:71

Methodmain

(String[] args)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoAnnotatedDepthCrawler.java:73

Methodmain

(String[] args)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoAnnotatedMatchTypeCrawler.java:100

Methodmain

(String[] args)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoMetaCrawler.java:81

Methodmain

(String[] args)

src/main/java/cn/edu/hfut/dmic/webcollector/example/AbuyunDynamicProxyRequester.java:66

Methodmain

(String[] args)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoSeleniumCrawler.java:45

Methodmain

(String[] args)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoRandomProxyCrawler.java:83

Methodmain

(String[] args)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoAnnotatedManualNewsCrawler.java:63

Methodmain

(String[] args)

src/main/java/cn/edu/hfut/dmic/webcollector/example/DemoManualNewsCrawler.java:57

Methodmain

(String[] args)

src/main/java/cn/edu/hfut/dmic/webcollector/util/FileIdGenerator.java:55

Methodmain

(String[] args)

src/main/java/cn/edu/hfut/dmic/webcollector/util/MD5Utils.java:48

Methodmain

(String[] args)

src/main/java/cn/edu/hfut/dmic/contentextractor/ContentExtractor.java:514

MethodmakeAbs

(Document doc,String url)

src/main/java/cn/edu/hfut/dmic/webcollector/util/JsoupUtils.java:31

MethodmatchContentType

判断当前Page的Http响应头的Content-Type是否符合正则 @param contentTypeRegex @return

src/main/java/cn/edu/hfut/dmic/webcollector/model/Page.java:98

← previous next →301–400 of 510, ranked by callers