Method tokenize

glslang/MachineIndependent/Scan.cpp:832–941

Called by yylex to get the next token. Returning 0 implies end of input.

// Called by yylex to get the next token.
// Returning 0 implies end of input.
int TScanContext::tokenize(TPpContext* pp, TParserToken& token)
{
    do {
        parserToken = &token;
        TPpToken ppToken;
        int token = pp->tokenize(ppToken);
        if (token == EndOfInput)
            return 0;

        tokenText = ppToken.name;
        loc = ppToken.loc;
        parserToken->sType.lex.loc = loc;
        switch (token) {
        case ';':  afterType = false; afterBuffer = false; inDeclaratorList = false; afterDeclarator = false; angleBracketDepth = 0; squareBracketDepth = 0; parenDepth = 0; return SEMICOLON;
        case ',':
            // If we just processed a declarator (identifier after a type), this comma
            // indicates that we're in a declarator list. Note that 'afterDeclarator' is
            // only set when we are not inside a template parameter list, array expression,
            // or function parameter list.
            if (afterDeclarator) {
                inDeclaratorList = true;
            }
            afterType = false;
            afterDeclarator = false;
            return COMMA;
        case ':':                       return COLON;
        case '=':  afterType = false; inDeclaratorList = false; afterDeclarator = false; return EQUAL;
        case '(':  afterType = false; inDeclaratorList = false; afterDeclarator = false; parenDepth++; return LEFT_PAREN;
        case ')':  afterType = false; inDeclaratorList = false; afterDeclarator = false; if (parenDepth > 0) parenDepth--; return RIGHT_PAREN;
        case '.':  field = true;        return DOT;
        case '!':                       return BANG;
        case '-':                       return DASH;
        case '~':                       return TILDE;
        case '+':                       return PLUS;
        case '*':                       return STAR;
        case '/':                       return SLASH;
        case '%':                       return PERCENT;
        case '<':                       angleBracketDepth++; return LEFT_ANGLE;
        case '>':                       if (angleBracketDepth > 0) angleBracketDepth--; return RIGHT_ANGLE;
        case '|':                       return VERTICAL_BAR;
        case '^':                       return CARET;
        case '&':                       return AMPERSAND;
        case '?':                       return QUESTION;
        case '[':                       squareBracketDepth++; return LEFT_BRACKET;
        case ']':                       if (squareBracketDepth > 0) squareBracketDepth--; return RIGHT_BRACKET;
        case '{':  afterStruct = false; afterBuffer = false; inDeclaratorList = false; afterDeclarator = false; angleBracketDepth = 0; squareBracketDepth = 0; parenDepth = 0; return LEFT_BRACE;
        case '}':  inDeclaratorList = false; afterDeclarator = false; angleBracketDepth = 0; squareBracketDepth = 0; parenDepth = 0; return RIGHT_BRACE;
        case '\\':
            parseContext.error(loc, "illegal use of escape character", "\\", "");
            break;

        case PPAtomAddAssign:          return ADD_ASSIGN;
        case PPAtomSubAssign:          return SUB_ASSIGN;
        case PPAtomMulAssign:          return MUL_ASSIGN;
        case PPAtomDivAssign:          return DIV_ASSIGN;
        case PPAtomModAssign:          return MOD_ASSIGN;

        case PpAtomRight:              return RIGHT_OP;