Runs generator @param args map of arguments @return results @throws Exception
(Map<String, Object> args)
| 197 | * @throws Exception |
| 198 | */ |
| 199 | @Override |
| 200 | public Map<String, Object> run(Map<String, Object> args) throws Exception { |
| 201 | String batchId = (String) args.get(Nutch.ARG_BATCH); |
| 202 | if (batchId == null) { |
| 203 | batchId = randomBatchId(); |
| 204 | } |
| 205 | getConf().set(BATCH_ID, batchId); |
| 206 | |
| 207 | // map to inverted subset due for fetch, sort by score |
| 208 | Long topN = null; |
| 209 | try { |
| 210 | topN = (Long) args.get(Nutch.ARG_TOPN); |
| 211 | } catch(Exception e) { |
| 212 | topN = Long.parseLong(args.get(Nutch.ARG_TOPN).toString()); |
| 213 | } |
| 214 | Long curTime = (Long) args.get(Nutch.ARG_CURTIME); |
| 215 | if (curTime == null) { |
| 216 | curTime = System.currentTimeMillis(); |
| 217 | } |
| 218 | Boolean filter = (Boolean) args.get(Nutch.ARG_FILTER); |
| 219 | Boolean norm = (Boolean) args.get(Nutch.ARG_NORMALIZE); |
| 220 | Boolean sitemap = (Boolean) args.get(Nutch.ARG_SITEMAP); |
| 221 | |
| 222 | // map to inverted subset due for fetch, sort by score |
| 223 | getConf().setLong(GENERATOR_CUR_TIME, curTime); |
| 224 | if (topN != null) |
| 225 | getConf().setLong(GENERATOR_TOP_N, topN); |
| 226 | if (filter != null) |
| 227 | getConf().setBoolean(GENERATOR_FILTER, filter); |
| 228 | if (sitemap != null) |
| 229 | getConf().setBoolean(GENERATOR_SITEMAP, sitemap); |
| 230 | |
| 231 | getConf().setLong(Nutch.GENERATE_TIME_KEY, System.currentTimeMillis()); |
| 232 | if (norm != null) |
| 233 | getConf().setBoolean(GENERATOR_NORMALISE, norm); |
| 234 | String mode = getConf().get(GENERATOR_COUNT_MODE, |
| 235 | GENERATOR_COUNT_VALUE_HOST); |
| 236 | if (GENERATOR_COUNT_VALUE_HOST.equalsIgnoreCase(mode)) { |
| 237 | getConf().set(URLPartitioner.PARTITION_MODE_KEY, |
| 238 | URLPartitioner.PARTITION_MODE_HOST); |
| 239 | } else if (GENERATOR_COUNT_VALUE_DOMAIN.equalsIgnoreCase(mode)) { |
| 240 | getConf().set(URLPartitioner.PARTITION_MODE_KEY, |
| 241 | URLPartitioner.PARTITION_MODE_DOMAIN); |
| 242 | } else { |
| 243 | LOG.warn("Unknown generator.max.count mode '" + mode + "', using mode=" |
| 244 | + GENERATOR_COUNT_VALUE_HOST); |
| 245 | getConf().set(GENERATOR_COUNT_MODE, GENERATOR_COUNT_VALUE_HOST); |
| 246 | getConf().set(URLPartitioner.PARTITION_MODE_KEY, |
| 247 | URLPartitioner.PARTITION_MODE_HOST); |
| 248 | } |
| 249 | numJobs = 1; |
| 250 | currentJobNum = 0; |
| 251 | currentJob = NutchJob.getInstance(getConf(), "generate: " + getConf().get(BATCH_ID)); |
| 252 | Collection<WebPage.Field> fields = getFields(currentJob); |
| 253 | StorageUtils.initMapperJob(currentJob, fields, SelectorEntry.class, |
| 254 | WebPage.class, GeneratorMapper.class, SelectorEntryPartitioner.class, |
| 255 | true); |
| 256 | StorageUtils.initReducerJob(currentJob, GeneratorReducer.class); |
no test coverage detected