Interface Protocol

src/java/org/apache/nutch/protocol/Protocol.java:29–65

A retriever of url content. Implemented by protocol extensions.

/** A retriever of url content. Implemented by protocol extensions. */
public interface Protocol extends FieldPluggable, Configurable {
  /** The name of the extension point. */
  public final static String X_POINT_ID = Protocol.class.getName();

  /**
   * Property name. If in the current configuration this property is set to
   * true, protocol implementations should handle "politeness" limits
   * internally. If this is set to false, it is assumed that these limits are
   * enforced elsewhere, and protocol implementations should not enforce them
   * internally.
   */
  public final static String CHECK_BLOCKING = "protocol.plugin.check.blocking";

  /**
   * Property name. If in the current configuration this property is set to
   * true, protocol implementations should handle robot exclusion rules
   * internally. If this is set to false, it is assumed that these limits are
   * enforced elsewhere, and protocol implementations should not enforce them
   * internally.
   */
  public final static String CHECK_ROBOTS = "protocol.plugin.check.robots";

  /*
   * Returns the {@link Content} for a fetchlist entry.
   */
  ProtocolOutput getProtocolOutput(String url, WebPage page);

  /**
   * Retrieve robot rules applicable for this url.
   * 
   * @param url
   *          url to check
   * @param page
   * @return robot rules (specific for this url or default), never null
   */
  BaseRobotRules getRobotRules(String url, WebPage page);
}