Given two urls, a src and a destination of a redirect, it returns the representative url. This method implements an extended version of the algorithm used by the Yahoo! Slurp crawler described here: How
(String src, String dst, boolean temp)
| 281 | * @return String The representative url. |
| 282 | */ |
| 283 | public static String chooseRepr(String src, String dst, boolean temp) { |
| 284 | |
| 285 | // validate both are well formed urls |
| 286 | URL srcUrl; |
| 287 | URL dstUrl; |
| 288 | try { |
| 289 | srcUrl = new URL(src); |
| 290 | dstUrl = new URL(dst); |
| 291 | } catch (MalformedURLException e) { |
| 292 | return dst; |
| 293 | } |
| 294 | |
| 295 | // get the source and destination domain, host, and page |
| 296 | String srcDomain = URLUtil.getDomainName(srcUrl); |
| 297 | String dstDomain = URLUtil.getDomainName(dstUrl); |
| 298 | String srcHost = srcUrl.getHost(); |
| 299 | String dstHost = dstUrl.getHost(); |
| 300 | String srcFile = srcUrl.getFile(); |
| 301 | String dstFile = dstUrl.getFile(); |
| 302 | |
| 303 | // are the source and destination the root path url.com/ or url.com |
| 304 | boolean srcRoot = (srcFile.equals("/") || srcFile.length() == 0); |
| 305 | boolean destRoot = (dstFile.equals("/") || dstFile.length() == 0); |
| 306 | |
| 307 | // 1) different domain them keep dest, temp or perm |
| 308 | // a.com -> b.com* |
| 309 | // |
| 310 | // 2) permanent and root, keep src |
| 311 | // *a.com -> a.com?y=1 || *a.com -> a.com/xyz/index.html |
| 312 | // |
| 313 | // 3) permanent and not root and dest root, keep dest |
| 314 | // a.com/xyz/index.html -> a.com* |
| 315 | // |
| 316 | // 4) permanent and neither root keep dest |
| 317 | // a.com/xyz/index.html -> a.com/abc/page.html* |
| 318 | // |
| 319 | // 5) temp and root and dest not root keep src |
| 320 | // *a.com -> a.com/xyz/index.html |
| 321 | // |
| 322 | // 7) temp and not root and dest root keep dest |
| 323 | // a.com/xyz/index.html -> a.com* |
| 324 | // |
| 325 | // 8) temp and neither root, keep shortest, if hosts equal by path else by |
| 326 | // hosts. paths are first by length then by number of / separators |
| 327 | // a.com/xyz/index.html -> a.com/abc/page.html* |
| 328 | // *www.a.com/xyz/index.html -> www.news.a.com/xyz/index.html |
| 329 | // |
| 330 | // 9) temp and both root keep shortest sub domain |
| 331 | // *www.a.com -> www.news.a.com |
| 332 | |
| 333 | // if we are dealing with a redirect from one domain to another keep the |
| 334 | // destination |
| 335 | if (!srcDomain.equals(dstDomain)) { |
| 336 | return dst; |
| 337 | } |
| 338 | |
| 339 | // if it is a permanent redirect |
| 340 | if (!temp) { |