This class uses a "chained filter" pattern to run defined normalizers. Different lists of normalizers may be defined for different "scopes", or contexts where they are used (note however that they need to be activated first through plugin.include property). There is one global scope de
| 89 | * @author Andrzej Bialecki |
| 90 | */ |
| 91 | public final class URLNormalizers { |
| 92 | |
| 93 | /** |
| 94 | * Default scope. If no scope properties are defined then the configuration |
| 95 | * for this scope will be used. |
| 96 | */ |
| 97 | public static final String SCOPE_DEFAULT = "default"; |
| 98 | /** Scope used by {@link org.apache.nutch.crawl.URLPartitioner}. */ |
| 99 | public static final String SCOPE_PARTITION = "partition"; |
| 100 | /** Scope used by {@link org.apache.nutch.crawl.GeneratorJob}. */ |
| 101 | public static final String SCOPE_GENERATE_HOST_COUNT = "generate_host_count"; |
| 102 | /** |
| 103 | * Scope used by {@link org.apache.nutch.fetcher.FetcherJob} when processing |
| 104 | * redirect URLs. |
| 105 | */ |
| 106 | public static final String SCOPE_FETCHER = "fetcher"; |
| 107 | /** Scope used when updating the CrawlDb with new URLs. */ |
| 108 | public static final String SCOPE_CRAWLDB = "crawldb"; |
| 109 | /** Scope used when updating the LinkDb with new URLs. */ |
| 110 | public static final String SCOPE_LINKDB = "linkdb"; |
| 111 | /** Scope used by {@link org.apache.nutch.crawl.InjectorJob}. */ |
| 112 | public static final String SCOPE_INJECT = "inject"; |
| 113 | /** |
| 114 | * Scope used when constructing new {@link org.apache.nutch.parse.Outlink} |
| 115 | * instances. |
| 116 | */ |
| 117 | public static final String SCOPE_OUTLINK = "outlink"; |
| 118 | |
| 119 | private static final Logger LOG = LoggerFactory |
| 120 | .getLogger(MethodHandles.lookup().lookupClass()); |
| 121 | |
| 122 | /* Empty extension list for caching purposes. */ |
| 123 | private final List<Extension> EMPTY_EXTENSION_LIST = Collections.emptyList(); |
| 124 | |
| 125 | private final URLNormalizer[] EMPTY_NORMALIZERS = new URLNormalizer[0]; |
| 126 | |
| 127 | private Configuration conf; |
| 128 | |
| 129 | private ExtensionPoint extensionPoint; |
| 130 | |
| 131 | private URLNormalizer[] normalizers; |
| 132 | |
| 133 | private int loopCount; |
| 134 | |
| 135 | public URLNormalizers(Configuration conf, String scope) { |
| 136 | this.conf = conf; |
| 137 | this.extensionPoint = PluginRepository.get(conf).getExtensionPoint( |
| 138 | URLNormalizer.X_POINT_ID); |
| 139 | ObjectCache objectCache = ObjectCache.get(conf); |
| 140 | |
| 141 | if (this.extensionPoint == null) { |
| 142 | throw new RuntimeException("x point " + URLNormalizer.X_POINT_ID |
| 143 | + " not found."); |
| 144 | } |
| 145 | |
| 146 | normalizers = (URLNormalizer[]) objectCache |
| 147 | .getObject(URLNormalizer.X_POINT_ID + "_" + scope); |
| 148 | if (normalizers == null) { |