Method tokenizeLine

TheForceEngine/TFE_System/parser.cpp:169–232

Split a line into tokens using space, comma or equals as separators. Note strings with spaces still work, they need to be closed in quotes, which are removed upon tokenizing.

// Split a line into tokens using space, comma or equals as separators.
// Note strings with spaces still work, they need to be closed in quotes, which are removed upon tokenizing.
void TFE_Parser::tokenizeLine(const char* line, TokenList& tokens)
{
	tokens.clear();

	const size_t len = strlen(line);
	// first move past leading whitespace and ending white space.
	size_t start = 0, end = 0;
	for (size_t c = 0; c < len; c++)
	{
		if (!isWhitespace(line[c]))
		{
			if (start == 0 && end == 0) { start = c; }
			end = c + 1;
		}
	}

	// next start reading tokens.
	bool inQuote = false;
	char curToken[1024];
	size_t curTokenPos = 0;
	// TODO: Add an option to allow white space in tokens when not in quotes, but still remove trailing/ending whitespace.
	// This is useful for names.
	for (size_t c = start; c < end; c++)
	{
		if (line[c] == '"')
		{
			if (inQuote && curTokenPos == 0)
			{
				tokens.push_back("");
			}
			inQuote = !inQuote;
		}
		else if (!inQuote && (isWhitespace(line[c]) || isSeparator(line[c])))
		{
			curToken[curTokenPos] = 0;
			if (curTokenPos)
			{
				tokens.push_back(curToken);
			}
			curTokenPos = 0;
			curToken[0] = 0;
		}
		else if (!inQuote && m_enableColonSeperator && line[c] == ':')
		{
			curToken[curTokenPos] = 0;
			if (curTokenPos)
			{
				tokens.push_back(curToken);
			}
			curTokenPos = 0;
			curToken[0] = 0;
		}
		else
		{
			curToken[curTokenPos++] = line[c];
		}
	}