MCPcopy Create free account
hub / github.com/apache/nutch / run

Method run

src/java/org/apache/nutch/parse/ParserJob.java:247–285  ·  view source on GitHub ↗
(Map<String, Object> args)

Source from the content-addressed store, hash-verified

245 }
246
247 @Override
248 public Map<String, Object> run(Map<String, Object> args) throws Exception {
249 String batchId = (String) args.get(Nutch.ARG_BATCH);
250 Boolean shouldResume = (Boolean) args.get(Nutch.ARG_RESUME);
251 Boolean force = (Boolean) args.get(Nutch.ARG_FORCE);
252 Boolean sitemap = (Boolean) args.get(Nutch.ARG_SITEMAP);
253
254 if (batchId != null) {
255 getConf().set(GeneratorJob.BATCH_ID, batchId);
256 }
257 if (shouldResume != null) {
258 getConf().setBoolean(RESUME_KEY, shouldResume);
259 }
260 if (force != null) {
261 getConf().setBoolean(FORCE_KEY, force);
262 }
263 if (sitemap != null) {
264 getConf().setBoolean(SITEMAP_PARSE, sitemap);
265 }
266 LOG.info("ParserJob: resuming:\t{}", getConf().getBoolean(RESUME_KEY, false));
267 LOG.info("ParserJob: forced reparse:\t {}", getConf().getBoolean(FORCE_KEY, false));
268 if (batchId == null || batchId.equals(Nutch.ALL_BATCH_ID_STR)) {
269 LOG.info("ParserJob: parsing all");
270 } else {
271 LOG.info("ParserJob: batchId:\t{}", batchId);
272 }
273 currentJob = NutchJob.getInstance(getConf(), "parse");
274
275 Collection<WebPage.Field> fields = getFields(currentJob);
276 MapFieldValueFilter<String, WebPage> batchIdFilter = getBatchIdFilter(batchId);
277 StorageUtils.initMapperJob(currentJob, fields, String.class, WebPage.class,
278 ParserMapper.class, batchIdFilter);
279 StorageUtils.initReducerJob(currentJob, IdentityPageReducer.class);
280 currentJob.setNumReduceTasks(0);
281
282 currentJob.waitForCompletion(true);
283 ToolUtil.recordJobStatus(null, currentJob, results);
284 return results;
285 }
286
287 private MapFieldValueFilter<String, WebPage> getBatchIdFilter(String batchId) {
288 if (batchId.equals(REPARSE.toString())

Callers 2

parseMethod · 0.95
mainMethod · 0.45

Calls 12

getConfMethod · 0.95
getInstanceMethod · 0.95
getFieldsMethod · 0.95
getBatchIdFilterMethod · 0.95
initMapperJobMethod · 0.95
initReducerJobMethod · 0.95
recordJobStatusMethod · 0.95
parseMethod · 0.95
waitForCompletionMethod · 0.80
getMethod · 0.65
setMethod · 0.45
equalsMethod · 0.45

Tested by

no test coverage detected