Method num_tokens

tasks/base_task.py:51–52

(self, index)

        return len(self._sizes)

    def num_tokens(self, index):
        return self.size(index)

    def size(self, index):
        """Return an example's size as a float or tuple. This value is used when