Assume input is UTF-8 encoded Unicode code points. Will read 1 code point and return that as int. The number of bytes read is len and they are put in the bytes buffer. EOF is never counted as such and will not appear in bytes. Returns EOF upon end-of-file otherwise the Unicode code point. */
| 146 | Returns EOF upon end-of-file otherwise the Unicode code point. |
| 147 | */ |
| 148 | static int utf8_codepoint(int cc, int *len, int bytes[4]) |
| 149 | { |
| 150 | *len = 0; |
| 151 | if (cc == EOF) |
| 152 | return EOF; |
| 153 | /* cc must be single byte: */ |
| 154 | assert((cc & 0xFFFFFF00) == 0); |
| 155 | bytes[(*len)++] = cc; |
| 156 | if (cc < 0x80) /* ASCII */ |
| 157 | return cc; |
| 158 | /* cc = 0b1xxxxxxx */ |
| 159 | int cp, n; |
| 160 | if ((cc & 0xE0) == 0xC0) { /* cc = 0b110xxxxx */ |
| 161 | n = 2; |
| 162 | cp = cc & 0x1F; /* 5 bits */ |
| 163 | } |
| 164 | else if ((cc & 0xF0) == 0xE0) { /* cc = 0b1110xxxx */ |
| 165 | n = 3; |
| 166 | cp = cc & 0x0F; /* 4 bits */ |
| 167 | } |
| 168 | else if ((cc & 0xF8) == 0xF0) { /* cc = 0b11110xxx */ |
| 169 | n = 4; |
| 170 | cp = cc & 0x07; /* 3 bits */ |
| 171 | } |
| 172 | else { /* invalid utf-8 start byte */ |
| 173 | if (!nowarn) |
| 174 | fprintf(stderr, "(W): [%s:%u] Invalid UTF-8 start byte 0x%02x.\n", |
| 175 | filename, linenr, cc); |
| 176 | return cc; |
| 177 | } |
| 178 | /* collect all follow bytes: */ |
| 179 | while (--n) { /* at most 3 iterations */ |
| 180 | cc = get(); |
| 181 | if (cc == EOF) { /* unexpected EOF in utf-8 sequence */ |
| 182 | if (!nowarn) |
| 183 | fprintf(stderr, "(W): [%s:%u] Unexpected EOF in UTF-8 sequence.\n", |
| 184 | filename, linenr); |
| 185 | return EOF; |
| 186 | } |
| 187 | bytes[(*len)++] = cc; |
| 188 | if ((cc & 0xC0) != 0x80) { /* invalid utf-8 follow byte */ |
| 189 | if (!nowarn) |
| 190 | fprintf(stderr, "(W): [%s:%u] Invalid UTF-8 follow byte 0x%02x.\n", |
| 191 | filename, linenr, cc); |
| 192 | return cc; |
| 193 | } |
| 194 | cp <<= 6; |
| 195 | cp |= (cc & 0x3F); /* 6 bits */ |
| 196 | } |
| 197 | /* check for validness; not in surrogate range, etc. */ |
| 198 | if (cp >= 0xD800 && cp <= 0xDFFF || |
| 199 | cp > 0x10FFFF || cp == 0xFFFE || cp == 0xFFFF) { |
| 200 | /* invalid Unicode code point. */ |
| 201 | if (!nowarn) |
| 202 | fprintf(stderr, "(W): [%s:%u] Invalid Unicode code point 0x%04x.\n", |
| 203 | filename, linenr, cp); |
| 204 | } |
| 205 | return cp; |