Method GetToken

sdk/angelscript/source/as_parser.cpp:1006–1043

BNF:18: COMMENT ::= ('//'[^#x0A]*) | ('/*'[^*]*'*/') // single token: starts with // and ends with new line or starts with /* and ends with */ BNF:18: WHITESPACE ::= [ #x09#x0A#x0D]+ // single token: spaces, tab, carriage return, line feed, and UTF8 byte-order-mark

// BNF:18: COMMENT       ::= ('//'[^#x0A]*) | ('/*'[^*]*'*/')  // single token:  starts with // and ends with new line or starts with /* and ends with */
// BNF:18: WHITESPACE    ::= [ #x09#x0A#x0D]+                  // single token:  spaces, tab, carriage return, line feed, and UTF8 byte-order-mark
void asCParser::GetToken(sToken *token)
{
	// Check if the token has already been parsed
	if( lastToken.pos == sourcePos )
	{
		*token = lastToken;
		sourcePos += token->length;

		if( token->type == ttWhiteSpace ||
			token->type == ttOnelineComment ||
			token->type == ttMultilineComment )
			GetToken(token);

		return;
	}

	// Parse new token
	size_t sourceLength = script->codeLength;
	do
	{
		if( sourcePos >= sourceLength )
		{
			token->type = ttEnd;
			token->length = 0;
		}
		else
			token->type = engine->tok.GetToken(&script->code[sourcePos], sourceLength - sourcePos, &token->length);

		token->pos = sourcePos;

		// Update state
		sourcePos += token->length;
	}
	// Filter out whitespace and comments
	while( token->type == ttWhiteSpace ||
	       token->type == ttOnelineComment ||
	       token->type == ttMultilineComment );
}

void asCParser::SetPos(size_t pos)
{