Function guessFileEncoding

src/eepp/ui/doc/textformat.cpp:390–456

}

TextFormat guessFileEncoding( IOStream& ins ) {
	auto start = ins.tell();
	ScopedOp op( [] {}, [&start, &ins] { ins.seek( start ); } );
	TextFileStats stats8;

	// Try UTF8 first:
	Uint32 numBytesRead =
		scanTextFile( stats8, ins, TextEncoding::get<UTF8>(), NumBytesForAutodetect );
	if ( numBytesRead == 0 )
		return { TextFormat::Encoding::UTF8, stats8.getNewLineType(), false };

	ins.seek( 0 );
	if ( stats8.numInvalidPoints() == 0 && stats8.numControl == 0 ) {
		// No UTF-8 encoding errors, and no weird control characters/nulls. Pick UTF-8.
		return { TextFormat::Encoding::UTF8, stats8.getNewLineType(), false };
	}

	// If more than 20% of the high bytes in UTF-8 are encoding errors, reinterpret UTF-8 as just
	// bytes.
	TextFormat::Encoding encoding8 = TextFormat::Encoding::UTF8;
	{
		Uint32 numHighBytes = numBytesRead - stats8.numPlainAscii - stats8.numControl;
		if ( stats8.numInvalidPoints() >= numHighBytes * 0.2f ) {
			// Too many UTF-8 errors. Consider it bytes.
			encoding8 = TextFormat::Encoding::Latin1;
			stats8.numPoints = numBytesRead;
			stats8.numValidPoints = numBytesRead;
		}
	}

	// Examine both UTF16 endianness:
	TextFileStats stats16_le;
	scanTextFile( stats16_le, ins, TextEncoding::get<UTF16_LE>(), NumBytesForAutodetect );
	ins.seek( 0 );

	TextFileStats stats16_be;
	scanTextFile( stats16_be, ins, TextEncoding::get<UTF16_BE>(), NumBytesForAutodetect );
	ins.seek( 0 );

	// Choose the better UTF16 candidate:
	TextFileStats* stats = &stats16_le;
	TextFormat::Encoding encoding = TextFormat::Encoding::UTF16LE;
	if ( stats16_be.getScore() > stats16_le.getScore() ) {
		stats = &stats16_be;
		encoding = TextFormat::Encoding::UTF16BE;
	}

	TextFileStats statsShiftJIS;
	statsShiftJIS.count16b = true;
	scanTextFile( statsShiftJIS, ins, TextEncoding::get<ShiftJIS>(), NumBytesForAutodetect );
	ins.seek( 0 );

	if ( statsShiftJIS.allValidPoints() && statsShiftJIS.num16bytes &&
		 statsShiftJIS.getScore() > stats->getScore() ) {
		stats = &statsShiftJIS;
		encoding = TextFormat::Encoding::Shift_JIS;
	}