Method tokenize

recipes/Python/464766_Trivia_Template/recipe-464766.py:43–55

tokenize(self) -> generator Split self into tokens and return generator object.

(self)

        return locals['_']

    def tokenize(self):
        """
        tokenize(self) -> generator

        Split self into tokens and return generator object.
        """
        for token in re.split('(<\?.*?\?>)', self):
            if token.startswith('<?') and token.endswith('?>'):
                yield token

            else:
                for ref in re.split('(###[\w_.]*)', token):
                    yield ref

    def compile(self):
        """