()
| 12 | |
| 13 | # see https://www.unicode.org/L2/L1999/UnicodeData.html |
| 14 | def unicode_data_iter(): |
| 15 | res = requests.get(UNICODE_DATA_URL) |
| 16 | res.raise_for_status() |
| 17 | data = res.content.decode() |
| 18 | |
| 19 | prev = [] |
| 20 | |
| 21 | for line in data.splitlines(): |
| 22 | # ej: 0000;<control>;Cc;0;BN;;;;;N;NULL;;;; |
| 23 | line = line.split(";") |
| 24 | |
| 25 | cpt = int(line[0], base=16) |
| 26 | assert cpt < MAX_CODEPOINTS |
| 27 | |
| 28 | cpt_lower = int(line[-2] or "0", base=16) |
| 29 | assert cpt_lower < MAX_CODEPOINTS |
| 30 | |
| 31 | cpt_upper = int(line[-3] or "0", base=16) |
| 32 | assert cpt_upper < MAX_CODEPOINTS |
| 33 | |
| 34 | categ = line[2].strip() |
| 35 | assert len(categ) == 2 |
| 36 | |
| 37 | bidir = line[4].strip() |
| 38 | assert len(categ) == 2 |
| 39 | |
| 40 | name = line[1] |
| 41 | if name.endswith(", First>"): |
| 42 | prev = (cpt, cpt_lower, cpt_upper, categ, bidir) |
| 43 | continue |
| 44 | if name.endswith(", Last>"): |
| 45 | assert prev[1:] == (0, 0, categ, bidir) |
| 46 | for c in range(prev[0], cpt): |
| 47 | yield (c, cpt_lower, cpt_upper, categ, bidir) |
| 48 | |
| 49 | yield (cpt, cpt_lower, cpt_upper, categ, bidir) |
| 50 | |
| 51 | |
| 52 | # see definition in unicode.h |
no test coverage detected