Function ProcessDataToken

code/AssetLib/FBX/FBXTokenizer.cpp:96–129

process a potential data token up to 'cur', adding it to 'output_tokens'. ------------------------------------------------------------------------------------------------

// process a potential data token up to 'cur', adding it to 'output_tokens'.
// ------------------------------------------------------------------------------------------------
void ProcessDataToken(TokenList &output_tokens, StackAllocator &token_allocator,
                      const char*& start, const char*& end,
                      unsigned int line,
                      unsigned int column,
                      TokenType type = TokenType_DATA,
                      bool must_have_token = false)
{
    if (start && end) {
        // sanity check:
        // tokens should have no whitespace outside quoted text and [start,end] should
        // properly delimit the valid range.
        bool in_double_quotes = false;
        for (const char* c = start; c != end + 1; ++c) {
            if (*c == '\"') {
                in_double_quotes = !in_double_quotes;
            }

            if (!in_double_quotes && IsSpaceOrNewLine(*c)) {
                TokenizeError("unexpected whitespace in token", line, column);
            }
        }

        if (in_double_quotes) {
            TokenizeError("non-terminated double quotes", line, column);
        }

        output_tokens.push_back(new_Token(start,end + 1,type,line,column));
    }
    else if (must_have_token) {
        TokenizeError("unexpected character, expected data token", line, column);
    }

    start = end = nullptr;
}

}