----------------------------------------------------------------------------- Functions that converts one unicode codepoint at a time -----------------------------------------------------------------------------
| 295 | // Functions that converts one unicode codepoint at a time |
| 296 | //----------------------------------------------------------------------------- |
| 297 | UTF32 oneUTF8toUTF32( const UTF8* codepoint, U32 *unitsWalked) |
| 298 | { |
| 299 | PROFILE_SCOPE(oneUTF8toUTF32); |
| 300 | |
| 301 | // codepoints 6 codeunits long are read, but do not convert correctly, |
| 302 | // and are filtered out anyway. |
| 303 | |
| 304 | // early out for ascii |
| 305 | if(!(*codepoint & 0x0080)) |
| 306 | { |
| 307 | if (unitsWalked != NULL) |
| 308 | *unitsWalked = 1; |
| 309 | return (UTF32)*codepoint; |
| 310 | } |
| 311 | |
| 312 | U32 expectedByteCount; |
| 313 | UTF32 ret = 0; |
| 314 | U8 codeunit; |
| 315 | |
| 316 | // check the first byte ( a.k.a. codeunit ) . |
| 317 | U8 c = codepoint[0]; |
| 318 | c = c >> 1; |
| 319 | expectedByteCount = sgFirstByteLUT[c]; |
| 320 | if(expectedByteCount > 0) // 0 or negative is illegal to start with |
| 321 | { |
| 322 | // process 1st codeunit |
| 323 | ret |= sgByteMask8LUT[expectedByteCount] & codepoint[0]; // bug? |
| 324 | |
| 325 | // process trailing codeunits |
| 326 | for(U32 i=1;i<expectedByteCount; i++) |
| 327 | { |
| 328 | codeunit = codepoint[i]; |
| 329 | if( sgFirstByteLUT[codeunit>>1] == 0 ) |
| 330 | { |
| 331 | ret <<= 6; // shift up 6 |
| 332 | ret |= (codeunit & 0x3f); // mask in the low 6 bits of this codeunit byte. |
| 333 | } |
| 334 | else |
| 335 | { |
| 336 | // found a bad codepoint - did not get a medial where we wanted one. |
| 337 | // Dump the replacement, and claim to have parsed only 1 char, |
| 338 | // so that we'll dump a slew of replacements, instead of eating the next char. |
| 339 | ret = kReplacementChar; |
| 340 | expectedByteCount = 1; |
| 341 | break; |
| 342 | } |
| 343 | } |
| 344 | } |
| 345 | else |
| 346 | { |
| 347 | // found a bad codepoint - got a medial or an illegal codeunit. |
| 348 | // Dump the replacement, and claim to have parsed only 1 char, |
| 349 | // so that we'll dump a slew of replacements, instead of eating the next char. |
| 350 | ret = kReplacementChar; |
| 351 | expectedByteCount = 1; |
| 352 | } |
| 353 | |
| 354 | if(unitsWalked != NULL) |
no test coverage detected