(Map<String, Object> args)
| 245 | } |
| 246 | |
| 247 | @Override |
| 248 | public Map<String, Object> run(Map<String, Object> args) throws Exception { |
| 249 | String batchId = (String) args.get(Nutch.ARG_BATCH); |
| 250 | Boolean shouldResume = (Boolean) args.get(Nutch.ARG_RESUME); |
| 251 | Boolean force = (Boolean) args.get(Nutch.ARG_FORCE); |
| 252 | Boolean sitemap = (Boolean) args.get(Nutch.ARG_SITEMAP); |
| 253 | |
| 254 | if (batchId != null) { |
| 255 | getConf().set(GeneratorJob.BATCH_ID, batchId); |
| 256 | } |
| 257 | if (shouldResume != null) { |
| 258 | getConf().setBoolean(RESUME_KEY, shouldResume); |
| 259 | } |
| 260 | if (force != null) { |
| 261 | getConf().setBoolean(FORCE_KEY, force); |
| 262 | } |
| 263 | if (sitemap != null) { |
| 264 | getConf().setBoolean(SITEMAP_PARSE, sitemap); |
| 265 | } |
| 266 | LOG.info("ParserJob: resuming:\t{}", getConf().getBoolean(RESUME_KEY, false)); |
| 267 | LOG.info("ParserJob: forced reparse:\t {}", getConf().getBoolean(FORCE_KEY, false)); |
| 268 | if (batchId == null || batchId.equals(Nutch.ALL_BATCH_ID_STR)) { |
| 269 | LOG.info("ParserJob: parsing all"); |
| 270 | } else { |
| 271 | LOG.info("ParserJob: batchId:\t{}", batchId); |
| 272 | } |
| 273 | currentJob = NutchJob.getInstance(getConf(), "parse"); |
| 274 | |
| 275 | Collection<WebPage.Field> fields = getFields(currentJob); |
| 276 | MapFieldValueFilter<String, WebPage> batchIdFilter = getBatchIdFilter(batchId); |
| 277 | StorageUtils.initMapperJob(currentJob, fields, String.class, WebPage.class, |
| 278 | ParserMapper.class, batchIdFilter); |
| 279 | StorageUtils.initReducerJob(currentJob, IdentityPageReducer.class); |
| 280 | currentJob.setNumReduceTasks(0); |
| 281 | |
| 282 | currentJob.waitForCompletion(true); |
| 283 | ToolUtil.recordJobStatus(null, currentJob, results); |
| 284 | return results; |
| 285 | } |
| 286 | |
| 287 | private MapFieldValueFilter<String, WebPage> getBatchIdFilter(String batchId) { |
| 288 | if (batchId.equals(REPARSE.toString()) |
no test coverage detected