Method normalize

pattern/web/__init__.py:3222–3228

Called from Crawler.crawl() to normalize URLs. For example: return url.split("?")[0]

(self, url)

        return False

    def normalize(self, url):
        """ Called from Crawler.crawl() to normalize URLs.
            For example: return url.split("?")[0]
        """
        # All links pass through here (visited or not).
        # This can be a place to count backlinks.
        return url

    def follow(self, link):
        """ Called from Crawler.crawl() to determine if it should follow this link.