MCPcopy Create free account
hub / github.com/apache/nutch / run

Method run

src/java/org/apache/nutch/indexer/IndexingJob.java:137–170  ·  view source on GitHub ↗
(Map<String, Object> args)

Source from the content-addressed store, hash-verified

135 }
136
137 @Override
138 public Map<String, Object> run(Map<String, Object> args) throws Exception {
139 LOG.info("IndexingJob: starting");
140
141 String batchId = (String) args.get(Nutch.ARG_BATCH);
142
143 Configuration conf = getConf();
144 conf.set(GeneratorJob.BATCH_ID, batchId);
145
146 Job job = NutchJob.getInstance(conf, "Indexer");
147 // TODO: Figure out why this needs to be here
148 job.getConfiguration().setClass("mapreduce.job.output.key.comparator.class",
149 StringComparator.class, RawComparator.class);
150
151 Collection<WebPage.Field> fields = getFields(job);
152 MapFieldValueFilter<String, WebPage> batchIdFilter = getBatchIdFilter(batchId);
153 StorageUtils.initMapperJob(job, fields, String.class, NutchDocument.class,
154 IndexerMapper.class, batchIdFilter);
155 job.setNumReduceTasks(0);
156 job.setOutputFormatClass(IndexerOutputFormat.class);
157
158 job.waitForCompletion(true);
159 ToolUtil.recordJobStatus(null, job, results);
160
161 IndexWriters writers = new IndexWriters(getConf());
162 LOG.info(writers.describe());
163
164 writers.open(getConf());
165 if (getConf().getBoolean(SolrConstants.COMMIT_INDEX, true)) {
166 writers.commit();
167 }
168 LOG.info("IndexingJob: done.");
169 return results;
170 }
171
172 private MapFieldValueFilter<String, WebPage> getBatchIdFilter(String batchId) {
173 if (batchId.equals(REINDEX.toString())

Callers 1

mainMethod · 0.45

Calls 15

getInstanceMethod · 0.95
getFieldsMethod · 0.95
getBatchIdFilterMethod · 0.95
initMapperJobMethod · 0.95
recordJobStatusMethod · 0.95
describeMethod · 0.95
openMethod · 0.95
commitMethod · 0.95
toArgMapMethod · 0.95
waitForCompletionMethod · 0.80
errorMethod · 0.80
getMethod · 0.65

Tested by

no test coverage detected