Method BuildWordsArray

Src/stringdiffs.cpp:420–482

* @brief Break line into constituent words */

 * @brief Break line into constituent words
 */
std::vector<stringdiffs::word>
stringdiffs::BuildWordsArray(const String & str) const
{
	std::vector<word> words = { word(0, -1, 0, 0) }; // dummy;
	int i = 0, begin = 0;
	ICUBreakIterator *pIterChar = ICUBreakIterator::getCharacterBreakIterator(reinterpret_cast<const UChar *>(str.c_str()), static_cast<int32_t>(str.length()));

	size_t sLen = str.length();
	assert(sLen < INT_MAX);

	if (str.empty())
		return words;

	int break_type = 0;
	int prev_break_type = 0;
	int iLen = static_cast<int>(sLen);
	for (; i < iLen;)
	{
		break_type = dlword;
		tchar_t ch = str[i];
		if (ch == '\r' || ch == '\n')
		{
			if (m_eol_mode == EOL_AS_SPACE)
				break_type = dlspace;
			else
				break_type = dleol;
		}
		else if (isSafeWhitespace(ch))
		{
			break_type = dlspace;
		}
		else if (isWordBreak(m_breakType, str.c_str(), i))
		{
			break_type = dlbreak;
		}
		else if (m_ignore_numbers && tc::istdigit(ch))
		{
			break_type = dlnumber;
		}
		if (i > 0 && (break_type != prev_break_type || break_type == dlbreak || (prev_break_type == dleol && !(str[i - 1] == '\r' && ch == '\n'))))
		{
			words.push_back(word(begin, i - 1, prev_break_type, Hash(str, begin, i - 1, 0)));
			begin = i;
		}
		if (m_eol_mode == EOL_AS_SPACE && break_type == dlspace)
		{
			for (; i < iLen; i++)
			{
				tchar_t ch2 = str[i];
				if (ch2 != '\r' && ch2 != '\n' && !isSafeWhitespace(ch2))
					break;
			}
			pIterChar->preceding(i);
		}
		else
		{
			i = pIterChar->next();
		}