Function Tokenize

code/AssetLib/FBX/FBXTokenizer.cpp:134–242

------------------------------------------------------------------------------------------------

// ------------------------------------------------------------------------------------------------
void Tokenize(TokenList &output_tokens, const char *input, StackAllocator &token_allocator) {
	ai_assert(input);
	ASSIMP_LOG_DEBUG("Tokenizing ASCII FBX file");

    // line and column numbers numbers are one-based
    unsigned int line = 1;
    unsigned int column = 1;

    bool comment = false;
    bool in_double_quotes = false;
    bool pending_data_token = false;

    const char *token_begin = nullptr, *token_end = nullptr;
    for (const char* cur = input;*cur;column += (*cur == '\t' ? ASSIMP_FBX_TAB_WIDTH : 1), ++cur) {
        const char c = *cur;

        if (IsLineEnd(c)) {
            comment = false;

            column = 0;
            ++line;
        }

        if(comment) {
            continue;
        }

        if(in_double_quotes) {
            if (c == '\"') {
                in_double_quotes = false;
                token_end = cur;

                ProcessDataToken(output_tokens, token_allocator, token_begin, token_end, line, column);
                pending_data_token = false;
            }
            continue;
        }

        switch(c)
        {
        case '\"':
            if (token_begin) {
                TokenizeError("unexpected double-quote", line, column);
            }
            token_begin = cur;
            in_double_quotes = true;
            continue;

        case ';':
            ProcessDataToken(output_tokens, token_allocator, token_begin, token_end, line, column);
            comment = true;
            continue;

        case '{':
            ProcessDataToken(output_tokens, token_allocator, token_begin, token_end, line, column);
            output_tokens.push_back(new_Token(cur,cur+1,TokenType_OPEN_BRACKET,line,column));
            continue;