Method map

src/java/org/apache/nutch/crawl/GeneratorMapper.java:49–108

(String reversedUrl, WebPage page, Context context)

  private float scoreThreshold;

  @Override
  public void map(String reversedUrl, WebPage page, Context context)
      throws IOException, InterruptedException {
    String url = TableUtil.unreverseUrl(reversedUrl);

    if (Mark.GENERATE_MARK.checkMark(page) != null) {
      GeneratorJob.LOG.debug("Skipping {}; already generated", url);
      return;
    }

    // filter on distance
    if (maxDistance > -1) {
      CharSequence distanceUtf8 = page.getMarkers().get(DbUpdaterJob.DISTANCE);
      if (distanceUtf8 != null) {
        int distance = Integer.parseInt(distanceUtf8.toString());
        if (distance > maxDistance) {
          return;
        }
      }
    }

    // If filtering is on don't generate URLs that don't pass URLFilters
    try {
      if (normalise) {
        url = normalizers.normalize(url,
            URLNormalizers.SCOPE_GENERATE_HOST_COUNT);
      }
      if (filter && filters.filter(url) == null)
        return;
      if ((sitemap && !URLFilters.isSitemap(page)) || !sitemap && URLFilters
          .isSitemap(page))
        return;
    } catch (URLFilterException | MalformedURLException e) {
      GeneratorJob.LOG
      .warn("Couldn't filter url: {} ({})", url, e);
      return;
    }

    // check fetch schedule
    if (!schedule.shouldFetch(url, page, curTime)) {
      if (GeneratorJob.LOG.isDebugEnabled()) {
        GeneratorJob.LOG.debug("-shouldFetch rejected '" + url
            + "', fetchTime=" + page.getFetchTime() + ", curTime=" + curTime);
      }
      return;
    }
    float score = page.getScore();
    try {
      score = scoringFilters.generatorSortValue(url, page, score);
    } catch (ScoringFilterException e) {
      // ignore
    }

    // consider only entries with a score superior to the threshold
    if (scoreThreshold != Float.NaN && score < scoreThreshold)
      return;

    entry.set(url, score);