Class Crawler

samples/src/main/java/com/example/retrofit/Crawler.java:48–158

A simple web crawler that uses a Retrofit service to turn URLs into webpages.

/** A simple web crawler that uses a Retrofit service to turn URLs into webpages. */
public final class Crawler {
  private final Set<HttpUrl> fetchedUrls =
      Collections.synchronizedSet(new LinkedHashSet<HttpUrl>());
  private final ConcurrentHashMap<String, AtomicInteger> hostnames = new ConcurrentHashMap<>();
  private final PageService pageService;

  public Crawler(PageService pageService) {
    this.pageService = pageService;
  }

  public void crawlPage(HttpUrl url) {
    // Skip hosts that we've visited many times.
    AtomicInteger hostnameCount = new AtomicInteger();
    AtomicInteger previous = hostnames.putIfAbsent(url.host(), hostnameCount);
    if (previous != null) hostnameCount = previous;
    if (hostnameCount.incrementAndGet() > 100) return;

    // Asynchronously visit URL.
    pageService
        .get(url)
        .enqueue(
            new Callback<Page>() {
              @Override
              public void onResponse(Call<Page> call, Response<Page> response) {
                if (!response.isSuccessful()) {
                  System.out.println(call.request().url() + ": failed: " + response.code());
                  return;
                }

                // Print this page's URL and title.
                Page page = response.body();
                HttpUrl base = response.raw().request().url();
                System.out.println(base + ": " + page.title);

                // Enqueue its links for visiting.
                for (String link : page.links) {
                  HttpUrl linkUrl = base.resolve(link);
                  if (linkUrl != null && fetchedUrls.add(linkUrl)) {
                    crawlPage(linkUrl);
                  }
                }
              }

              @Override
              public void onFailure(Call<Page> call, Throwable t) {
                System.out.println(call.request().url() + ": failed: " + t);
              }
            });
  }

  public static void main(String... args) throws Exception {
    Dispatcher dispatcher = new Dispatcher(Executors.newFixedThreadPool(20));
    dispatcher.setMaxRequests(20);
    dispatcher.setMaxRequestsPerHost(1);

    OkHttpClient okHttpClient =
        new OkHttpClient.Builder()
            .dispatcher(dispatcher)