Function TokenizeText

Source/CoreLib/Tokenizer.cpp:304–634

		}

		List<Token> TokenizeText(const String & fileName, const String & text, Procedure<TokenizeErrorType, CodePosition> errorHandler)
		{
			int lastPos = 0, pos = 0;
			int line = 1, col = 0;
			String file = fileName;
			State state = State::Start;
			StringBuilder tokenBuilder;
			int tokenLine, tokenCol;
			List<Token> tokenList;
			LexDerivative derivative = LexDerivative::None;
            TokenFlags tokenFlags = TokenFlag::AtStartOfLine;
			auto InsertToken = [&](TokenType type)
			{
				derivative = LexDerivative::None;
				tokenList.Add(Token(type, tokenBuilder.ToString(), tokenLine, tokenCol, pos, file, tokenFlags));
                tokenFlags = 0;
				tokenBuilder.Clear();
			};
			auto ProcessTransferChar = [&](char nextChar)
			{
				switch (nextChar)
				{
				case '\\':
				case '\"':
				case '\'':
					tokenBuilder.Append(nextChar);
					break;
				case 't':
					tokenBuilder.Append('\t');
					break;
				case 's':
					tokenBuilder.Append(' ');
					break;
				case 'n':
					tokenBuilder.Append('\n');
					break;
				case 'r':
					tokenBuilder.Append('\r');
					break;
				case 'b':
					tokenBuilder.Append('\b');
					break;
				}
			};
			while (pos <= text.Length())
			{
				char curChar = (pos < text.Length() ? text[pos] : ' ');
				char nextChar = (pos < text.Length() - 1) ? text[pos + 1] : '\0';
				if (lastPos != pos)
				{
					if (curChar == '\n')
					{
						line++;
						col = 0;
					}
					else
						col++;
					lastPos = pos;