Method HostDb

src/java/org/apache/nutch/host/HostDb.java:70–105

(Configuration conf)

  private AtomicLong lastFlush;

  public HostDb(Configuration conf) throws GoraException {
    try {
      hostStore = StorageUtils.createWebStore(conf, String.class, Host.class);
    } catch (ClassNotFoundException e) {
      throw new RuntimeException(e);
    }

    // Create a cache.
    // We add a removal listener to see if we need to flush the store,
    // in order to adhere to the put-flush-get semantic
    // ("read your own write") of DataStore.

    long lruSize = conf.getLong(HOSTDB_LRU_SIZE, DEFAULT_LRU_SIZE);
    int concurrencyLevel = conf.getInt(HOSTDB_CONCURRENCY_LEVEL,
        DEFAULT_HOSTDB_CONCURRENCY_LEVEL);
    RemovalListener<String, CacheHost> listener = new RemovalListener<String, CacheHost>() {
      @Override
      public void onRemoval(RemovalNotification<String, CacheHost> notification) {
        CacheHost removeFromCacheHost = notification.getValue();
        if (removeFromCacheHost != NULL_HOST) {
          if (removeFromCacheHost.timestamp < lastFlush.get()) {
            try {
              hostStore.flush();
            } catch (Exception e) {
              throw new RuntimeException(e);
            }
            lastFlush.set(System.currentTimeMillis());
          }
        }
      }
    };

    cache = CacheBuilder.newBuilder().maximumSize(lruSize)
        .removalListener(listener).concurrencyLevel(concurrencyLevel).build();
    lastFlush = new AtomicLong(System.currentTimeMillis());
  }

  public Host get(final String key) throws IOException {
    Callable<CacheHost> valueLoader = new Callable<CacheHost>() {