Method main

src/java/org/apache/nutch/tools/DmozParser.java:368–432

Command-line access. User may add URLs via a flat text file or the structured DMOZ file. By default, we ignore Adult material (as categorized by DMOZ).

(String argv[])

   * by DMOZ).
   */
  public static void main(String argv[]) throws Exception {
    if (argv.length < 1) {
      System.err
          .println("Usage: DmozParser <dmoz_file> [-subset <subsetDenominator>] [-includeAdultMaterial] [-skew skew] [-snippet] [-topicFile <topic list file>] [-topic <topic> [-topic <topic> [...]]]");
      return;
    }

    //
    // Parse the command line, figure out what kind of
    // URL file we need to load
    //
    int subsetDenom = 1;
    int skew = 0;
    String dmozFile = argv[0];
    boolean includeAdult = false;
    boolean snippet = false;
    Pattern topicPattern = null;
    Vector<String> topics = new Vector<String>();

    Configuration conf = NutchConfiguration.create();
    store = StorageUtils.createWebStore(conf, String.class, WebPage.class);
    FileSystem fs = FileSystem.get(conf);
    try {
      for (int i = 1; i < argv.length; i++) {
        if ("-includeAdultMaterial".equals(argv[i])) {
          includeAdult = true;
        } else if ("-subset".equals(argv[i])) {
          subsetDenom = Integer.parseInt(argv[i + 1]);
          i++;
        } else if ("-topic".equals(argv[i])) {
          topics.addElement(argv[i + 1]);
          i++;
        } else if ("-topicFile".equals(argv[i])) {
          addTopicsFromFile(argv[i + 1], topics);
          i++;
        } else if ("-skew".equals(argv[i])) {
          skew = Integer.parseInt(argv[i + 1]);
          i++;
        } else if ("-snippet".equals(argv[i])) {
          snippet = true;
        }
      }

      DmozParser parser = new DmozParser();

      if (!topics.isEmpty()) {
        String regExp = new String("^(");
        int j = 0;
        for (; j < topics.size() - 1; ++j) {
          regExp = regExp.concat(topics.get(j));
          regExp = regExp.concat("|");
        }
        regExp = regExp.concat(topics.get(j));
        regExp = regExp.concat(").*");
        LOG.info("Topic selection pattern = " + regExp);
        topicPattern = Pattern.compile(regExp);
      }