Method map

src/java/org/apache/nutch/crawl/DbUpdateMapper.java:55–98

(String key, WebPage page, Context context)

  private WebPageWritable pageWritable;

  @Override
  public void map(String key, WebPage page, Context context)
      throws IOException, InterruptedException {
    if (Mark.GENERATE_MARK.checkMark(page) == null) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("Skipping " + TableUtil.unreverseUrl(key)
        + "; not generated yet");
      }
      return;
    }

    String url = TableUtil.unreverseUrl(key);

    scoreData.clear();
    Map<CharSequence, CharSequence> outlinks = page.getOutlinks();
    addScoreData(page, outlinks);

    Map<CharSequence, CharSequence> sitemaps = page.getSitemaps();
    addScoreData(page, sitemaps);

    // TODO: Outlink filtering (i.e. "only keep the first n outlinks")
    try {
      scoringFilters.distributeScoreToOutlinks(url, page, scoreData,
          (outlinks == null ? 0 : outlinks.size()));
    } catch (ScoringFilterException e) {
      LOG.warn("Distributing score failed for URL: " + key + " exception:"
          + StringUtils.stringifyException(e));
    }

    urlWithScore.setUrl(key);
    urlWithScore.setScore(Float.MAX_VALUE);
    pageWritable.setWebPage(page);
    nutchWritable.set(pageWritable);
    context.write(urlWithScore, nutchWritable);

    for (ScoreDatum scoreDatum : scoreData) {
      String reversedOut = TableUtil.reverseUrl(scoreDatum.getUrl());
      scoreDatum.setUrl(url);
      urlWithScore.setUrl(reversedOut);
      urlWithScore.setScore(scoreDatum.getScore());
      nutchWritable.set(scoreDatum);
      context.write(urlWithScore, nutchWritable);
    }
  }

  private void addScoreData(WebPage page, Map<CharSequence, CharSequence> map) {
    if (map != null) {