Method ReadToken

Source/Engine/Utilities/TextProcessing.cpp:95–182

}

void TextProcessing::ReadToken(Token* token)
{
    // Eat whitespaces
    EatWhiteSpaces();

    // Set token start but reset its length
    token->Start = _cursor;
    token->Length = 0;
    token->Separator = { 0, 0 };

    // Read until can
    while (_position < _length)
    {
        // Read char
        char c = moveForward();

        // Check if it's a separator char
        char c1 = PeekChar();
        for (int32 s = 0; s < Separators.Count(); s++)
        {
            if (Separators[s].C0 == c && Separators[s].C1 == c1)
            {
                /*if (*length == 0)
                {
                    // Skip separator after separator case and start searching for new token
                    *token = _cursor;
                    *length = -1;
                    break;
                }*/

                token->Separator = Separators[s];
                ReadChar();
                
                // Check for comments
                if (token->Separator == SingleLineComment)
                {
                    // Read whole line
                    ReadLine();

                    // Read another token
                    ReadToken(token);
                }
                else if (token->Separator == MultiLineCommentSeparator)
                {
                    // Read tokens until end sequence
                    char prev = ' ';
                    while (CanRead())
                    {
                        c = ReadChar();
                        if (prev == '*' && c == '/')
                            break;
                        prev = c;
                    }

                    // Check if comment is valid (has end before file end)
                    if (!CanRead())
                    {
                        LOG(Warning, "Missing multiline comment ending");