| 81 | } |
| 82 | |
| 83 | public Map<String, Object> run(Map<String, Object> args) throws Exception { |
| 84 | String crawlId = (String) args.get(Nutch.ARG_CRAWL); |
| 85 | String batchId = (String) args.get(Nutch.ARG_BATCH); |
| 86 | numJobs = 1; |
| 87 | currentJobNum = 0; |
| 88 | |
| 89 | if (batchId == null) { |
| 90 | batchId = Nutch.ALL_BATCH_ID_STR; |
| 91 | } |
| 92 | getConf().set(Nutch.BATCH_NAME_KEY, batchId); |
| 93 | // job.setBoolean(ALL, updateAll); |
| 94 | ScoringFilters scoringFilters = new ScoringFilters(getConf()); |
| 95 | HashSet<WebPage.Field> fields = new HashSet<WebPage.Field>(FIELDS); |
| 96 | fields.addAll(scoringFilters.getFields()); |
| 97 | |
| 98 | currentJob = NutchJob.getInstance(getConf(), "update-table"); |
| 99 | if (crawlId != null) { |
| 100 | currentJob.getConfiguration().set(Nutch.CRAWL_ID_KEY, crawlId); |
| 101 | } |
| 102 | |
| 103 | // Partition by {url}, sort by {url,score} and group by {url}. |
| 104 | // This ensures that the inlinks are sorted by score when they enter |
| 105 | // the reducer. |
| 106 | |
| 107 | currentJob.setPartitionerClass(UrlOnlyPartitioner.class); |
| 108 | currentJob.setSortComparatorClass(UrlScoreComparator.class); |
| 109 | currentJob.setGroupingComparatorClass(UrlOnlyComparator.class); |
| 110 | |
| 111 | MapFieldValueFilter<String, WebPage> batchIdFilter = getBatchIdFilter(batchId); |
| 112 | StorageUtils.initMapperJob(currentJob, fields, UrlWithScore.class, |
| 113 | NutchWritable.class, DbUpdateMapper.class, batchIdFilter); |
| 114 | StorageUtils.initReducerJob(currentJob, DbUpdateReducer.class); |
| 115 | currentJob.waitForCompletion(true); |
| 116 | ToolUtil.recordJobStatus(null, currentJob, results); |
| 117 | return results; |
| 118 | } |
| 119 | |
| 120 | private MapFieldValueFilter<String, WebPage> getBatchIdFilter(String batchId) { |
| 121 | if (batchId.equals(Nutch.ALL_CRAWL_ID.toString())) { |