()
| 381 | } |
| 382 | |
| 383 | private void doWebMode() { |
| 384 | reset(); |
| 385 | int numPagesPosted; |
| 386 | if (type != null) { |
| 387 | throw new IllegalArgumentException( |
| 388 | "Specifying content-type with \"--mode=web\" is not supported"); |
| 389 | } |
| 390 | |
| 391 | // Set Extracting handler as default |
| 392 | solrUpdateUrl = appendUrlPath(solrUpdateUrl, "/extract"); |
| 393 | |
| 394 | info("Posting web pages to Solr url " + solrUpdateUrl); |
| 395 | auto = true; |
| 396 | info( |
| 397 | "Entering auto mode. Indexing pages with content-types corresponding to file endings " |
| 398 | + fileTypes); |
| 399 | if (recursive > 0) { |
| 400 | if (recursive > MAX_WEB_DEPTH) { |
| 401 | recursive = MAX_WEB_DEPTH; |
| 402 | warn("Too large recursion depth for web mode, limiting to " + MAX_WEB_DEPTH + "..."); |
| 403 | } |
| 404 | if (delay < DEFAULT_WEB_DELAY) { |
| 405 | warn( |
| 406 | "Never crawl an external web site faster than every 10 seconds, your IP will probably be blocked"); |
| 407 | } |
| 408 | info("Entering recursive mode, depth=" + recursive + ", delay=" + delay + "s"); |
| 409 | } |
| 410 | numPagesPosted = postWebPages(args, 0, out); |
| 411 | info(numPagesPosted + " web pages indexed."); |
| 412 | } |
| 413 | |
| 414 | private void doStdinMode() { |
| 415 | info("POSTing stdin to " + solrUpdateUrl + "..."); |
no test coverage detected