Class TextCleaner

scripts/eval/eval.py:216–280

Handles text cleaning operations for different datasets. This utility class provides dataset-specific text normalization and cleaning operations to ensure consistent transcript processing across different evaluation datasets, particularly for specialized corpora like CORAAL. Method

class TextCleaner:
    """Handles text cleaning operations for different datasets.

    This utility class provides dataset-specific text normalization and cleaning
    operations to ensure consistent transcript processing across different evaluation
    datasets, particularly for specialized corpora like CORAAL.

    Methods:
        clean_coraal_text: Specialized text cleaning for CORAAL dialect corpus
    """

    @staticmethod
    def clean_coraal_text(text: str) -> str:
        """Clean CORAAL dataset text according to corpus conventions.

        Applies CORAAL-specific text normalization including dialect word mapping,
        marker removal, and unintelligible segment handling. This ensures fair
        evaluation by standardizing transcript format.

        Args:
            text (str): Raw CORAAL transcript text with corpus-specific markup

        Returns:
            str: Cleaned transcript text ready for evaluation

        Note:
            CORAAL (Corpus of Regional African American Language) uses specific
            markup conventions for dialectal features, background noise, and
            unintelligible segments that need standardized handling.

        Example:
            >>> raw_text = "We(BR) aksed for [unintelligible] busses"
            >>> clean_text = TextCleaner.clean_coraal_text(raw_text)
            >>> print(clean_text)
            "We asked for  buses"
        """
        text = text.replace("[", "{").replace("]", "}")

        # Relabel CORAAL words
        replacements = {
            "busses": "buses",
            "aks": "ask",
            "aksing": "asking",
            "aksed": "asked",
        }
        words = text.split()
        words = [replacements.get(word, word) for word in words]
        text = " ".join(words)

        # Remove CORAAL flags and markers
        patterns_to_remove = [
            r"(?i)\/unintelligible\/",
            r"(?i)\/inaudible\/",
            r"\/RD(.*?)\/",
            r"\/(\?)\1*\/",
        ]
        for pattern in patterns_to_remove:
            text = re.sub(pattern, "", text)