| 29 | } |
| 30 | |
| 31 | static uint32_t utf8_decode(const char * s, size_t remaining, int * len) { |
| 32 | if (remaining == 0) { *len = 0; return 0xFFFD; } |
| 33 | uint8_t c = (uint8_t)s[0]; |
| 34 | if (c < 0x80) { *len = 1; return c; } |
| 35 | if ((c & 0xE0) == 0xC0 && remaining >= 2) { |
| 36 | *len = 2; |
| 37 | return ((uint32_t)(c & 0x1F) << 6) | |
| 38 | ((uint32_t)((uint8_t)s[1]) & 0x3F); |
| 39 | } |
| 40 | if ((c & 0xF0) == 0xE0 && remaining >= 3) { |
| 41 | *len = 3; |
| 42 | return ((uint32_t)(c & 0x0F) << 12) | |
| 43 | (((uint32_t)((uint8_t)s[1]) & 0x3F) << 6) | |
| 44 | ((uint32_t)((uint8_t)s[2]) & 0x3F); |
| 45 | } |
| 46 | if ((c & 0xF8) == 0xF0 && remaining >= 4) { |
| 47 | *len = 4; |
| 48 | return ((uint32_t)(c & 0x07) << 18) | |
| 49 | (((uint32_t)((uint8_t)s[1]) & 0x3F) << 12) | |
| 50 | (((uint32_t)((uint8_t)s[2]) & 0x3F) << 6) | |
| 51 | ((uint32_t)((uint8_t)s[3]) & 0x3F); |
| 52 | } |
| 53 | *len = 1; |
| 54 | return 0xFFFD; |
| 55 | } |
| 56 | |
| 57 | // Unicode character property tests (simplified — covers the ranges needed |
| 58 | // for Qwen3/3.5 tokenizer pre-split). |
no outgoing calls
no test coverage detected