Parses given web page and stores parsed content within page. Puts a meta-redirect to outlinks. @param url @param page
(String url, WebPage page)
| 329 | * @param page |
| 330 | */ |
| 331 | public void process(String url, WebPage page) { |
| 332 | if (status(url, page)) { |
| 333 | return; |
| 334 | } |
| 335 | Parse parse; |
| 336 | try { |
| 337 | parse = parse(url, page); |
| 338 | } catch (ParserNotFound e) { |
| 339 | // do not print stacktrace for the fact that some types are not mapped. |
| 340 | LOG.warn("No suitable parser found: " + e.getMessage()); |
| 341 | return; |
| 342 | } catch (final Exception e) { |
| 343 | LOG.warn("Error parsing: " + url + ": " |
| 344 | + StringUtils.stringifyException(e)); |
| 345 | return; |
| 346 | } |
| 347 | |
| 348 | if (parse == null) { |
| 349 | return; |
| 350 | } |
| 351 | |
| 352 | ParseStatus pstatus = parse.getParseStatus(); |
| 353 | page.setParseStatus(pstatus); |
| 354 | if (ParseStatusUtils.isSuccess(pstatus)) { |
| 355 | if (pstatus.getMinorCode() == ParseStatusCodes.SUCCESS_REDIRECT) { |
| 356 | successRedirect(url, page, pstatus); |
| 357 | } else { |
| 358 | page.setText(new Utf8(parse.getText())); |
| 359 | page.setTitle(new Utf8(parse.getTitle())); |
| 360 | |
| 361 | setSignature(page); |
| 362 | |
| 363 | if (page.getOutlinks() != null) { |
| 364 | page.getOutlinks().clear(); |
| 365 | } |
| 366 | String fromHost; |
| 367 | if (ignoreExternalLinks) { |
| 368 | try { |
| 369 | fromHost = new URL(url).getHost().toLowerCase(Locale.ROOT); |
| 370 | } catch (final MalformedURLException e) { |
| 371 | fromHost = null; |
| 372 | } |
| 373 | } else { |
| 374 | fromHost = null; |
| 375 | } |
| 376 | int validCount = 0; |
| 377 | |
| 378 | final Outlink[] outlinks = parse.getOutlinks(); |
| 379 | int outlinksToStore = Math.min(maxOutlinks, outlinks.length); |
| 380 | for (int i = 0; validCount < outlinksToStore |
| 381 | && i < outlinks.length; i++, validCount++) { |
| 382 | String toUrl = outlinks[i].getToUrl(); |
| 383 | if (toUrl.length() > maxTargetLength) { |
| 384 | continue; // skip it |
| 385 | } |
| 386 | String toHost; |
| 387 | if (ignoreExternalLinks) { |
| 388 | try { |