| 123 | } |
| 124 | |
| 125 | public int run(String[] args) throws Exception { |
| 126 | String plugins = "protocol-http|parse-tika|scoring-opic|urlfilter-regex|urlnormalizer-pass"; |
| 127 | int seeds = 1; |
| 128 | int depth = 10; |
| 129 | int threads = 10; |
| 130 | // boolean delete = true; |
| 131 | long topN = Long.MAX_VALUE; |
| 132 | |
| 133 | if (args.length == 0) { |
| 134 | System.err |
| 135 | .println("Usage: Benchmark [-crawlId <id>] [-seeds NN] [-depth NN] [-threads NN] [-maxPerHost NN] [-plugins <regex>]"); |
| 136 | System.err |
| 137 | .println("\t-crawlId id\t the id to prefix the schemas to operate on, (default: storage.crawl.id)"); |
| 138 | System.err |
| 139 | .println("\t-seeds NN\tcreate NN unique hosts in a seed list (default: 1)"); |
| 140 | System.err.println("\t-depth NN\tperform NN crawl cycles (default: 10)"); |
| 141 | System.err |
| 142 | .println("\t-threads NN\tuse NN threads per Fetcher task (default: 10)"); |
| 143 | // XXX what is the equivalent here? not an additional job... |
| 144 | // System.err.println("\t-keep\tkeep batchId data (default: delete after updatedb)"); |
| 145 | System.err.println("\t-plugins <regex>\toverride 'plugin.includes'."); |
| 146 | System.err.println("\tNOTE: if not specified, this is reset to: " |
| 147 | + plugins); |
| 148 | System.err |
| 149 | .println("\tNOTE: if 'default' is specified then a value set in nutch-default/nutch-site is used."); |
| 150 | System.err |
| 151 | .println("\t-maxPerHost NN\tmax. # of URLs per host in a fetchlist"); |
| 152 | return -1; |
| 153 | } |
| 154 | int maxPerHost = Integer.MAX_VALUE; |
| 155 | for (int i = 0; i < args.length; i++) { |
| 156 | if (args[i].equals("-crawlId")) { |
| 157 | getConf().set(Nutch.CRAWL_ID_KEY, args[++i]); |
| 158 | } else if (args[i].equals("-seeds")) { |
| 159 | seeds = Integer.parseInt(args[++i]); |
| 160 | } else if (args[i].equals("-threads")) { |
| 161 | threads = Integer.parseInt(args[++i]); |
| 162 | } else if (args[i].equals("-depth")) { |
| 163 | depth = Integer.parseInt(args[++i]); |
| 164 | } else if (args[i].equals("-plugins")) { |
| 165 | plugins = args[++i]; |
| 166 | } else if (args[i].equalsIgnoreCase("-maxPerHost")) { |
| 167 | maxPerHost = Integer.parseInt(args[++i]); |
| 168 | } else { |
| 169 | LOG.error("Invalid argument: '" + args[i] + "'"); |
| 170 | return -1; |
| 171 | } |
| 172 | } |
| 173 | BenchmarkResults res = benchmark(seeds, depth, threads, maxPerHost, topN, |
| 174 | plugins); |
| 175 | System.out.println(res); |
| 176 | return 0; |
| 177 | } |
| 178 | |
| 179 | public BenchmarkResults benchmark(int seeds, int depth, int threads, |
| 180 | int maxPerHost, long topN, String plugins) throws Exception { |