Function detect_language

datastudio/utils/language.py:7–29

Detect if text is primarily Chinese or English. Args: text: Input text to analyze. Returns: "chinese", "english", or "unknown".

(text: str)

def detect_language(text: str) -> Literal["chinese", "english", "unknown"]:
    """
    Detect if text is primarily Chinese or English.

    Args:
        text: Input text to analyze.

    Returns:
        "chinese", "english", or "unknown".
    """
    # Remove punctuation and digits, keep only letters and Chinese characters
    clean_text = re.sub(r"[^\u4e00-\u9fff\u3400-\u4dbfa-zA-Z]", "", text)

    if not clean_text:
        return "unknown"

    # Count Chinese characters
    chinese_chars = re.findall(r"[\u4e00-\u9fff\u3400-\u4dbf]", clean_text)

    if len(chinese_chars) > 0:
        return "chinese"
    else:
        return "english"


def is_mixed_language(question: str, answer: str) -> bool: