| 3927 | |
| 3928 | |
| 3929 | def test_utf8_normalize(): |
| 3930 | arr = pa.array(["01²3"]) |
| 3931 | assert pc.utf8_normalize(arr, form="NFC") == arr |
| 3932 | assert pc.utf8_normalize(arr, form="NFKC") == pa.array(["0123"]) |
| 3933 | assert pc.utf8_normalize(arr, "NFD") == arr |
| 3934 | assert pc.utf8_normalize(arr, "NFKD") == pa.array(["0123"]) |
| 3935 | # GH-51225: composing forms must compose, not only decompose |
| 3936 | composed = pa.array(["\u00e9", "\ud55c", None]) |
| 3937 | decomposed = pa.array(["e\u0301", "\u1112\u1161\u11ab", None]) |
| 3938 | for form in ("NFC", "NFKC"): |
| 3939 | assert pc.utf8_normalize(decomposed, form=form) == composed |
| 3940 | assert pc.utf8_normalize(composed, form=form) == composed |
| 3941 | for form in ("NFD", "NFKD"): |
| 3942 | assert pc.utf8_normalize(composed, form=form) == decomposed |
| 3943 | assert pc.utf8_normalize(decomposed, form=form) == decomposed |
| 3944 | with pytest.raises( |
| 3945 | ValueError, |
| 3946 | match='"NFZ" is not a valid Unicode normalization form'): |
| 3947 | pc.utf8_normalize(arr, form="NFZ") |
| 3948 | |
| 3949 | |
| 3950 | def test_random(): |