()
| 40 | |
| 41 | |
| 42 | def main(): |
| 43 | out = sys.argv[1] |
| 44 | # per-character NFKC |
| 45 | nfkc = [] |
| 46 | seq = [] |
| 47 | for c in range(MAXCP): |
| 48 | if is_surrogate(c): |
| 49 | continue |
| 50 | ch = chr(c) |
| 51 | m = unicodedata.normalize("NFKC", ch) |
| 52 | if m != ch: |
| 53 | nfkc.append((c, len(seq), len(m))) |
| 54 | seq.extend(ord(x) for x in m) |
| 55 | # canonical composition pairs: a + b -> c when NFC composes them |
| 56 | comp = [] |
| 57 | for c in range(MAXCP): |
| 58 | if is_surrogate(c): |
| 59 | continue |
| 60 | d = unicodedata.decomposition(chr(c)) |
| 61 | if not d or d.startswith("<"): |
| 62 | continue |
| 63 | parts = [int(x, 16) for x in d.split()] |
| 64 | if len(parts) != 2: |
| 65 | continue |
| 66 | a, b = parts |
| 67 | if unicodedata.normalize("NFC", chr(a) + chr(b)) == chr(c): |
| 68 | comp.append((a, b, c)) |
| 69 | comp.sort() |
| 70 | with open(out, "w", encoding="utf-8") as f: |
| 71 | f.write("/*\n * pdf_unicode.c -- Unicode %s facts for the PDF mention scanner (generated).\n" |
| 72 | " *\n * The field test's scanner ran on CPython's str and re; these tables are what\n" |
| 73 | " * CPython %s answers: NFKC of each code point, the canonical compositions,\n" |
| 74 | " * str.isspace, the re module's \\w (str.isalnum or '_'), str.isalpha, and the\n" |
| 75 | " * one-character str.isupper / str.islower / titlecase. Generated by\n" |
| 76 | " * scripts/gen-pdf-unicode.py; do not edit by hand. Data of the Unicode\n" |
| 77 | " * Character Database, (c) Unicode, Inc., Unicode License v3\n" |
| 78 | " * (THIRD_PARTY.md).\n */\n" |
| 79 | % (unicodedata.unidata_version, sys.version.split()[0])) |
| 80 | f.write("#include \"pdf_unicode.h\"\n\n") |
| 81 | f.write("const pdf_nfkc_t PDF_NFKC[] = {\n") |
| 82 | for i in range(0, len(nfkc), 4): |
| 83 | f.write(" " + " ".join("{0x%X, %d, %d}," % e for e in nfkc[i:i + 4]) + "\n") |
| 84 | f.write("};\nconst int PDF_NFKC_COUNT = (int)(sizeof(PDF_NFKC) / sizeof(PDF_NFKC[0]));\n\n") |
| 85 | f.write("const uint32_t PDF_NFKC_SEQ[] = {\n") |
| 86 | for i in range(0, len(seq), 10): |
| 87 | f.write(" " + " ".join("0x%X," % x for x in seq[i:i + 10]) + "\n") |
| 88 | f.write("};\n\n") |
| 89 | f.write("const pdf_ucomp_t PDF_UCOMP[] = {\n") |
| 90 | for i in range(0, len(comp), 3): |
| 91 | f.write(" " + " ".join("{0x%X, 0x%X, 0x%X}," % e for e in comp[i:i + 3]) + "\n") |
| 92 | f.write("};\nconst int PDF_UCOMP_COUNT = (int)(sizeof(PDF_UCOMP) / sizeof(PDF_UCOMP[0]));\n\n") |
| 93 | emit_ranges(f, "PDF_U_SPACE", ranges(lambda c: c.isspace())) |
| 94 | emit_ranges(f, "PDF_U_WORD", ranges(lambda c: c.isalnum() or c == "_")) |
| 95 | emit_ranges(f, "PDF_U_ALPHA", ranges(lambda c: c.isalpha())) |
| 96 | emit_ranges(f, "PDF_U_UPPER", ranges(lambda c: c.isupper())) |
| 97 | emit_ranges(f, "PDF_U_LOWER", ranges(lambda c: c.islower())) |
| 98 | emit_ranges(f, "PDF_U_TITLE", ranges(lambda c: c.istitle() and not c.isupper())) |
| 99 | print("nfkc", len(nfkc), "seq", len(seq), "comp", len(comp)) |
no test coverage detected