(Map<String, Object> args)
| 135 | } |
| 136 | |
| 137 | @Override |
| 138 | public Map<String, Object> run(Map<String, Object> args) throws Exception { |
| 139 | LOG.info("IndexingJob: starting"); |
| 140 | |
| 141 | String batchId = (String) args.get(Nutch.ARG_BATCH); |
| 142 | |
| 143 | Configuration conf = getConf(); |
| 144 | conf.set(GeneratorJob.BATCH_ID, batchId); |
| 145 | |
| 146 | Job job = NutchJob.getInstance(conf, "Indexer"); |
| 147 | // TODO: Figure out why this needs to be here |
| 148 | job.getConfiguration().setClass("mapreduce.job.output.key.comparator.class", |
| 149 | StringComparator.class, RawComparator.class); |
| 150 | |
| 151 | Collection<WebPage.Field> fields = getFields(job); |
| 152 | MapFieldValueFilter<String, WebPage> batchIdFilter = getBatchIdFilter(batchId); |
| 153 | StorageUtils.initMapperJob(job, fields, String.class, NutchDocument.class, |
| 154 | IndexerMapper.class, batchIdFilter); |
| 155 | job.setNumReduceTasks(0); |
| 156 | job.setOutputFormatClass(IndexerOutputFormat.class); |
| 157 | |
| 158 | job.waitForCompletion(true); |
| 159 | ToolUtil.recordJobStatus(null, job, results); |
| 160 | |
| 161 | IndexWriters writers = new IndexWriters(getConf()); |
| 162 | LOG.info(writers.describe()); |
| 163 | |
| 164 | writers.open(getConf()); |
| 165 | if (getConf().getBoolean(SolrConstants.COMMIT_INDEX, true)) { |
| 166 | writers.commit(); |
| 167 | } |
| 168 | LOG.info("IndexingJob: done."); |
| 169 | return results; |
| 170 | } |
| 171 | |
| 172 | private MapFieldValueFilter<String, WebPage> getBatchIdFilter(String batchId) { |
| 173 | if (batchId.equals(REINDEX.toString()) |
no test coverage detected