Method Utf16ToUtf8

astyle/src/astyle_main.cpp:2236–2336

Adapted from SciTE Utf8_16.cxx. Copyright (C) 2002 Scott Kirkwood. Modified for Artistic Style by Jim Pattee. Convert a utf-16 file to utf-8.

//
// Convert a utf-16 file to utf-8.
size_t ASConsole::Utf16ToUtf8(char* utf16In, size_t inLen, FileEncoding encoding,
                              bool firstBlock, char* utf8Out) const
{
	typedef unsigned short utf16;	// 16 bits
	typedef unsigned char  ubyte;	// 8 bits
	enum { SURROGATE_LEAD_FIRST = 0xD800 };
	enum { SURROGATE_LEAD_LAST = 0xDBFF };
	enum { SURROGATE_TRAIL_FIRST = 0xDC00 };
	enum { SURROGATE_TRAIL_LAST = 0xDFFF };
	enum { SURROGATE_FIRST_VALUE = 0x10000 };
	enum eState { eStart, eSecondOf4Bytes, ePenultimate, eFinal };

	int nCur16 = 0;
	int nCur = 0;
	ubyte* pRead = reinterpret_cast<ubyte*>(utf16In);
	ubyte* pCur = reinterpret_cast<ubyte*>(utf8Out);
	const ubyte* pEnd = pRead + inLen;
	const ubyte* pCurStart = pCur;
	static eState eState = eStart;	// eState is retained for subsequent blocks
	if (firstBlock)
		eState = eStart;

	// the BOM will automatically be converted to utf-8
	while (pRead < pEnd)
	{
		switch (eState)
		{
		case eStart:
			if (pRead >= pEnd)
			{
				++pRead;
				break;
			}
			if (encoding == UTF_16LE)
			{
				nCur16 = *pRead++;
				nCur16 |= static_cast<utf16>(*pRead << 8);
			}
			else
			{
				nCur16 = static_cast<utf16>(*pRead++ << 8);
				nCur16 |= static_cast<utf16>(*pRead);
			}
			if (nCur16 >= SURROGATE_LEAD_FIRST && nCur16 <= SURROGATE_LEAD_LAST)
			{
				++pRead;
				int trail;
				if (encoding == UTF_16LE)
				{
					trail = *pRead++;
					trail |= static_cast<utf16>(*pRead << 8);
				}
				else
				{
					trail = static_cast<utf16>(*pRead++ << 8);
					trail |= static_cast<utf16>(*pRead);
				}
				nCur16 = (((nCur16 & 0x3ff) << 10) | (trail & 0x3ff)) + SURROGATE_FIRST_VALUE;