Method _tokenize

test/general/lm_eval/tasks/drop.py:268–269

(self, text)

        return str(float(text)) if self._is_number(text) else text

    def _tokenize(self, text):
        return re.split(" |-", text)

    def _normalize(self, answer):
        tokens = [