Function ProcessSentence

extensions/removerepeatphrase.cpp:42–75

constexpr wchar_t ERASED = 0xf246; // inside Unicode private use area

bool ProcessSentence(std::wstring& sentence, SentenceInfo sentenceInfo)
{
	if (sentenceInfo["text number"] == 0) return false;

	// This algorithm looks for repeating substrings (in other words, common prefixes among the set of suffixes) of the sentence with length > 6
	// It then looks for any regions of characters at least twice as long as the substring made up only of characters in the substring, and erases them
	// If this results in the substring being completely erased from the string, the substring is copied to the last location where it was located in the original string
	auto timeout = GetTickCount64() + 30'000; // give up if taking over 30 seconds
	std::vector<int> suffixArray = GenerateSuffixArray(sentence);
	for (int i = 0; i + 1 < sentence.size() && GetTickCount64() < timeout; ++i)
	{
		int commonPrefixLength = 0;
		for (int j = suffixArray[i], k = suffixArray[i + 1]; j < sentence.size() && k < sentence.size(); ++j, ++k)
			if (sentence[j] != ERASED && sentence[j] == sentence[k]) commonPrefixLength += 1;
			else break;

		if (commonPrefixLength > 6)
		{
			std::wstring substring(sentence, suffixArray[i], commonPrefixLength);
			bool substringCharMap[0x10000] = {};
			for (auto ch : substring) substringCharMap[ch] = true;

			for (int regionSize = 0, j = 0; j <= sentence.size(); ++j)
				if (substringCharMap[sentence[j]]) regionSize += 1;
				else if (regionSize >= commonPrefixLength * 2)
					while (regionSize > 0) sentence[j - regionSize--] = ERASED;
				else regionSize = 0;

			if (!wcsstr(sentence.c_str(), substring.c_str())) std::copy(substring.begin(), substring.end(), sentence.begin() + max(suffixArray[i], suffixArray[i + 1]));
		}
	}
	sentence.erase(std::remove(sentence.begin(), sentence.end(), ERASED), sentence.end());
	return true;
}

TEST(
	{