(String[] args)
| 459 | }; |
| 460 | |
| 461 | public int run(String[] args) throws Exception { |
| 462 | if (args.length < 1) { |
| 463 | System.err |
| 464 | .println("Usage: WebTableReader (-stats | -url [url] | -dump <out_dir> [-regex regex]) \n \t \t [-crawlId <id>] [-content] [-headers] [-links] [-text]"); |
| 465 | System.err |
| 466 | .println(" -crawlId <id> - the id to prefix the schemas to operate on, \n \t \t (default: storage.crawl.id)"); |
| 467 | System.err |
| 468 | .println(" -stats [-sort] - print overall statistics to System.out"); |
| 469 | System.err.println(" [-sort] - list status sorted by host"); |
| 470 | System.err |
| 471 | .println(" -url <url> - print information on <url> to System.out"); |
| 472 | System.err |
| 473 | .println(" -dump <out_dir> [-regex regex] - dump the webtable to a text file in \n \t \t <out_dir>"); |
| 474 | System.err.println(" -content - dump also raw content"); |
| 475 | System.err.println(" -headers - dump protocol headers"); |
| 476 | System.err.println(" -links - dump links"); |
| 477 | System.err.println(" -text - dump extracted text"); |
| 478 | System.err |
| 479 | .println(" [-regex] - filter on the URL of the webtable entry"); |
| 480 | return -1; |
| 481 | } |
| 482 | String param = null; |
| 483 | boolean content = false; |
| 484 | boolean links = false; |
| 485 | boolean text = false; |
| 486 | boolean headers = false; |
| 487 | boolean toSort = false; |
| 488 | String regex = ".+"; |
| 489 | Op op = null; |
| 490 | try { |
| 491 | for (int i = 0; i < args.length; i++) { |
| 492 | if (args[i].equals("-url")) { |
| 493 | param = args[++i]; |
| 494 | op = Op.READ; |
| 495 | // read(param); |
| 496 | // return 0; |
| 497 | } else if (args[i].equals("-stats")) { |
| 498 | op = Op.STAT; |
| 499 | } else if (args[i].equals("-sort")) { |
| 500 | toSort = true; |
| 501 | } else if (args[i].equals("-dump")) { |
| 502 | op = Op.DUMP; |
| 503 | param = args[++i]; |
| 504 | } else if (args[i].equals("-content")) { |
| 505 | content = true; |
| 506 | } else if (args[i].equals("-headers")) { |
| 507 | headers = true; |
| 508 | } else if (args[i].equals("-links")) { |
| 509 | links = true; |
| 510 | } else if (args[i].equals("-text")) { |
| 511 | text = true; |
| 512 | } else if (args[i].equals("-regex")) { |
| 513 | regex = args[++i]; |
| 514 | } else if (args[i].equals("-crawlId")) { |
| 515 | getConf().set(Nutch.CRAWL_ID_KEY, args[++i]); |
| 516 | } |
| 517 | } |
| 518 | if (op == null) { |
no test coverage detected