Method resolveUrls

src/java/org/apache/nutch/tools/ResolveUrls.java:100–133

Creates a thread pool for resolving urls. Reads in the url file on the local filesystem. For each url it attempts to resolve it keeping a total account of the number resolved, errored, and the amount of time.

()

   * account of the number resolved, errored, and the amount of time.
   */
  public void resolveUrls() {

    try {

      // create a thread pool with a fixed number of threads
      pool = Executors.newFixedThreadPool(numThreads);

      // read in the urls file and loop through each line, one url per line
      BufferedReader buffRead = new BufferedReader(new InputStreamReader(new FileInputStream(urlsFile), StandardCharsets.UTF_8));
      String urlStr = null;
      while ((urlStr = buffRead.readLine()) != null) {

        // spin up a resolver thread per url
        LOG.info("Starting: " + urlStr);
        pool.execute(new ResolverThread(urlStr));
      }

      // close the file and wait for up to 60 seconds before shutting down
      // the thread pool to give urls time to finish resolving
      buffRead.close();
      pool.awaitTermination(60, TimeUnit.SECONDS);
    } catch (Exception e) {

      // on error shutdown the thread pool immediately
      pool.shutdownNow();
      LOG.info(StringUtils.stringifyException(e));
    }

    // shutdown the thread pool and log totals
    pool.shutdown();
    LOG.info("Total: " + numTotal.get() + ", Resovled: " + numResolved.get()
        + ", Errored: " + numErrored.get() + ", Average Time: "
        + totalTime.get() / numTotal.get());
  }

  /**
   * Create a new ResolveUrls with a file from the local file system.