Method generate

src/java/org/apache/nutch/crawl/GeneratorJob.java:280–315

Mark URLs ready for fetching. @param topN top threshold for maximum number of URLs permitted in a batch @param curTime the current time in milliseconds @param filter optional filtering of URLs within the generated batch @param norm optional normalization of URls w

(long topN, long curTime, boolean filter, boolean norm,
      boolean sitemap)

   * @throws Exception
   * */
  public String generate(long topN, long curTime, boolean filter, boolean norm,
      boolean sitemap) throws Exception {

    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT);
    long start = System.currentTimeMillis();
    LOG.info("GeneratorJob: starting at {}", sdf.format(start));
    LOG.info("GeneratorJob: Selecting best-scoring urls due for fetch.");
    LOG.info("GeneratorJob: starting");
    LOG.info("GeneratorJob: filtering: {}", filter);
    LOG.info("GeneratorJob: normalizing: {}", norm);
    if (sitemap) {
      LOG.info("GeneratorJob: sitemap: {}", sitemap);
    }
    if (topN != Long.MAX_VALUE) {
      LOG.info("GeneratorJob: topN: {}", topN);
    }
    String batchId = getConf().get(BATCH_ID);
    Map<String, Object> results = run(ToolUtil.toArgMap(Nutch.ARG_TOPN, topN,
        Nutch.ARG_CURTIME, curTime, Nutch.ARG_FILTER, filter,
        Nutch.ARG_NORMALIZE, norm, Nutch.ARG_BATCH, batchId, Nutch.ARG_SITEMAP, sitemap));
    if (batchId == null) {
      // use generated random batch id
      batchId = (String) results.get(BATCH_ID);
    }

    long finish = System.currentTimeMillis();
    long generateCount = (Long) results.get(GENERATE_COUNT);
    LOG.info("GeneratorJob: finished at {}, time elapsed: {}", 
        sdf.format(finish), TimingUtil.elapsedTime(start, finish));
    LOG.info("GeneratorJob: generated batch id: {} containing {} URLs", 
        batchId, generateCount);
    if (generateCount == 0) {
      return null;
    }
    return batchId;
  }

  /**
   * Runs generator from commandline