Function ReadToken

Source/SpireCore/Preprocessor.cpp:1696–1729

Read one token using the full preprocessor, with all its behaviors.

// Read one token using the full preprocessor, with all its behaviors.
static Token ReadToken(Preprocessor* preprocessor)
{
    for (;;)
    {
        // Look at the next raw token in the input.
        Token const& token = PeekRawToken(preprocessor);

        // If we have a directive (`#` at start of line) then handle it
        if ((token.Type == TokenType::Pound) && (token.flags & TokenFlag::AtStartOfLine))
        {
            // Skip the `#`
            AdvanceRawToken(preprocessor);

            // Create a context for parsing the directive
            PreprocessorDirectiveContext directiveContext;
            directiveContext.preprocessor = preprocessor;
            directiveContext.parseError = false;

            // Parse and handle the directive
            HandleDirective(&directiveContext);
            continue;
        }

        // otherwise, if we are currently in a skipping mode, then skip tokens
        if (IsSkipping(preprocessor))
        {
            AdvanceRawToken(preprocessor);
            continue;
        }

        // otherwise read a token, which may involve macro expansion
        return AdvanceToken(preprocessor);
    }
}

// intialize a preprocessor context, using the given sink for errros
static void InitializePreprocessor(