Function get_normalization_props

scripts/test_helpers.py:177–241

Download and parse Unicode DerivedNormalizationProps.txt, caching in temp directory. Args: version: Unicode version string (e.g., "17.0.0") Returns: Dict mapping codepoints to their normalization properties. Properties include: NFC_QC, NFD_QC, NFKC_QC, NFKD_QC (Quic

(version: str = UNICODE_VERSION)

def get_normalization_props(version: str = UNICODE_VERSION) -> Dict[int, Dict[str, str]]:
    """Download and parse Unicode DerivedNormalizationProps.txt, caching in temp directory.

    Args:
        version: Unicode version string (e.g., "17.0.0")

    Returns:
        Dict mapping codepoints to their normalization properties.
        Properties include: NFC_QC, NFD_QC, NFKC_QC, NFKD_QC (Quick_Check values)
    """
    cache_path = os.path.join(
        tempfile.gettempdir(), f"DerivedNormalizationProps-{version}.txt"
    )

    if not os.path.exists(cache_path):
        url = f"https://www.unicode.org/Public/{version}/ucd/DerivedNormalizationProps.txt"
        print(f"Downloading Unicode {version} DerivedNormalizationProps.txt from {url}...")
        try:
            urllib.request.urlretrieve(url, cache_path)
            print(f"Cached to {cache_path}")
        except Exception as e:
            raise UnicodeDataDownloadError(
                f"Could not download DerivedNormalizationProps.txt from {url}: {e}"
            )
    else:
        print(f"Using cached Unicode {version} DerivedNormalizationProps.txt: {cache_path}")

    props: Dict[int, Dict[str, str]] = {}
    with open(cache_path, "r", encoding="utf-8") as f:
        for line in f:
            line = line.split("#")[0].strip()
            if not line:
                continue
            parts = line.split(";")
            if len(parts) < 2:
                continue
            try:
                cp_range = parts[0].strip()
                prop_value = parts[1].strip()

                # Parse property name and value (e.g., "NFC_QC" or "NFC_QC; N")
                if len(parts) >= 3:
                    prop_name = prop_value
                    prop_val = parts[2].strip()
                else:
                    # Properties like "Full_Composition_Exclusion" are boolean
                    prop_name = prop_value
                    prop_val = "Y"

                # Parse codepoint range
                if ".." in cp_range:
                    start, end = cp_range.split("..")
                    start_cp = int(start, 16)
                    end_cp = int(end, 16)
                else:
                    start_cp = end_cp = int(cp_range, 16)

                for cp in range(start_cp, end_cp + 1):