Method remove_extra_spaces

train/data.py:91–105

Remove double spaces in the prompt and generations to standardize spacing.

(self)

        return len(self.generations)
    
    def remove_extra_spaces(self):
        """
        Remove double spaces in the prompt and generations to standardize spacing.
        """
        def clean(text: str) -> str:
            return re.sub(r'[ \t]{2,}', ' ', text)

        # Clean the prompt
        for turn in self.prompt:
            turn['content'] = clean(turn['content'])

        # Clean the generations
        for x in self.generations:
            for turn in x:
                turn["content"] = clean(turn["content"])


class Dataset: