MCPcopy Create free account
hub / github.com/DeusData/codebase-memory-mcp / main

Function main

scripts/gen-pdf-unicode.py:42–99  ·  view source on GitHub ↗
()

Source from the content-addressed store, hash-verified

40
41
42def main():
43 out = sys.argv[1]
44 # per-character NFKC
45 nfkc = []
46 seq = []
47 for c in range(MAXCP):
48 if is_surrogate(c):
49 continue
50 ch = chr(c)
51 m = unicodedata.normalize("NFKC", ch)
52 if m != ch:
53 nfkc.append((c, len(seq), len(m)))
54 seq.extend(ord(x) for x in m)
55 # canonical composition pairs: a + b -> c when NFC composes them
56 comp = []
57 for c in range(MAXCP):
58 if is_surrogate(c):
59 continue
60 d = unicodedata.decomposition(chr(c))
61 if not d or d.startswith("<"):
62 continue
63 parts = [int(x, 16) for x in d.split()]
64 if len(parts) != 2:
65 continue
66 a, b = parts
67 if unicodedata.normalize("NFC", chr(a) + chr(b)) == chr(c):
68 comp.append((a, b, c))
69 comp.sort()
70 with open(out, "w", encoding="utf-8") as f:
71 f.write("/*\n * pdf_unicode.c -- Unicode %s facts for the PDF mention scanner (generated).\n"
72 " *\n * The field test's scanner ran on CPython's str and re; these tables are what\n"
73 " * CPython %s answers: NFKC of each code point, the canonical compositions,\n"
74 " * str.isspace, the re module's \\w (str.isalnum or '_'), str.isalpha, and the\n"
75 " * one-character str.isupper / str.islower / titlecase. Generated by\n"
76 " * scripts/gen-pdf-unicode.py; do not edit by hand. Data of the Unicode\n"
77 " * Character Database, (c) Unicode, Inc., Unicode License v3\n"
78 " * (THIRD_PARTY.md).\n */\n"
79 % (unicodedata.unidata_version, sys.version.split()[0]))
80 f.write("#include \"pdf_unicode.h\"\n\n")
81 f.write("const pdf_nfkc_t PDF_NFKC[] = {\n")
82 for i in range(0, len(nfkc), 4):
83 f.write(" " + " ".join("{0x%X, %d, %d}," % e for e in nfkc[i:i + 4]) + "\n")
84 f.write("};\nconst int PDF_NFKC_COUNT = (int)(sizeof(PDF_NFKC) / sizeof(PDF_NFKC[0]));\n\n")
85 f.write("const uint32_t PDF_NFKC_SEQ[] = {\n")
86 for i in range(0, len(seq), 10):
87 f.write(" " + " ".join("0x%X," % x for x in seq[i:i + 10]) + "\n")
88 f.write("};\n\n")
89 f.write("const pdf_ucomp_t PDF_UCOMP[] = {\n")
90 for i in range(0, len(comp), 3):
91 f.write(" " + " ".join("{0x%X, 0x%X, 0x%X}," % e for e in comp[i:i + 3]) + "\n")
92 f.write("};\nconst int PDF_UCOMP_COUNT = (int)(sizeof(PDF_UCOMP) / sizeof(PDF_UCOMP[0]));\n\n")
93 emit_ranges(f, "PDF_U_SPACE", ranges(lambda c: c.isspace()))
94 emit_ranges(f, "PDF_U_WORD", ranges(lambda c: c.isalnum() or c == "_"))
95 emit_ranges(f, "PDF_U_ALPHA", ranges(lambda c: c.isalpha()))
96 emit_ranges(f, "PDF_U_UPPER", ranges(lambda c: c.isupper()))
97 emit_ranges(f, "PDF_U_LOWER", ranges(lambda c: c.islower()))
98 emit_ranges(f, "PDF_U_TITLE", ranges(lambda c: c.istitle() and not c.isupper()))
99 print("nfkc", len(nfkc), "seq", len(seq), "comp", len(comp))

Callers 1

gen-pdf-unicode.pyFile · 0.70

Calls 6

rangeFunction · 0.85
is_surrogateFunction · 0.85
emit_rangesFunction · 0.85
rangesFunction · 0.85
openFunction · 0.50
writeMethod · 0.45

Tested by

no test coverage detected