Method webCrawl

solr/core/src/java/org/apache/solr/cli/PostTool.java:632–702

A very simple crawler, pulling URLs to fetch from a backlog and then recurses N levels deep if recursive>0. Links are parsed from HTML through first getting an XHTML version using SolrCell with extractOnly, and followed if they are local. The crawler pauses for a default delay of 10 seconds betwe

(int level, OutputStream out)

   * @return number of pages crawled on this level and below
   */
  protected int webCrawl(int level, OutputStream out) {
    int numPages = 0;
    LinkedHashSet<URI> stack = backlog.get(level);
    int rawStackSize = stack.size();
    stack.removeAll(visited);
    int stackSize = stack.size();
    LinkedHashSet<URI> subStack = new LinkedHashSet<>();
    info(
        "Entering crawl at level "
            + level
            + " ("
            + rawStackSize
            + " links total, "
            + stackSize
            + " new)");
    for (URI uri : stack) {
      try {
        visited.add(uri);
        URL url = uri.toURL();
        PostTool.PageFetcherResult result = pageFetcher.readPageFromUrl(url);
        if (result.httpStatus == 200) {
          url = (result.redirectUrl != null) ? result.redirectUrl : url;
          URI postUri =
              new URI(
                  appendParam(
                      solrUpdateUrl.toString(),
                      "literal.id="
                          + URLEncoder.encode(url.toString(), UTF_8)
                          + "&literal.url="
                          + URLEncoder.encode(url.toString(), UTF_8)));
          ByteBuffer content = result.content;
          boolean success =
              postData(
                  new ByteArrayInputStream(content.array(), content.arrayOffset(), content.limit()),
                  null,
                  out,
                  result.contentType,
                  postUri);
          if (success) {
            info("POSTed web resource " + url + " (depth: " + level + ")");
            Thread.sleep(delay * 1000L);
            numPages++;
            // Pull links from HTML pages only
            if (recursive > level && result.contentType.equals("text/html")) {
              Set<URI> children =
                  pageFetcher.getLinksFromWebPage(
                      url,
                      new ByteArrayInputStream(
                          content.array(), content.arrayOffset(), content.limit()),
                      result.contentType,
                      postUri);
              subStack.addAll(children);
            }
          } else {
            warn("An error occurred while posting " + uri);
          }
        } else {
          warn("The URL " + uri + " returned a HTTP result status of " + result.httpStatus);