Method _binary_to_token

bytecode/gdscript_v2_tokenizer_buffer.cpp:106–158

}

GDScriptV2TokenizerCompat::Token GDScriptV2TokenizerBufferCompat::_binary_to_token(const uint8_t *p_buffer, int p_token_index) {
	Token token;
	const uint8_t *b = p_buffer;

	uint32_t token_type = decode_uint32(b);
	token.type = decomp->get_global_token((token_type & TOKEN_MASK));
	if (token_type & TOKEN_BYTE_MASK) {
		b += 4;
	} else {
		b++;
	}
	// token.start_line = decode_uint32(b);
	token.start_line = get_token_line(token_lines, p_token_index);
	token.end_line = decode_uint32(b);
	token.start_column = get_token_line(token_columns, p_token_index);
	token.end_column = token.start_column;

	token.literal = token.get_name();
	if (token.type == Token::Type::G_TK_CONST_NAN) {
		token.literal = String("NAN"); // Special case since name and notation are different.
	}

	switch (token.type) {
		case GDScriptV2TokenizerCompat::Token::Type::G_TK_ANNOTATION:
		case GDScriptV2TokenizerCompat::Token::Type::G_TK_IDENTIFIER: {
			// Get name from map.
			int identifier_pos = token_type >> TOKEN_BITS;
			if (unlikely(identifier_pos >= identifiers.size())) {
				Token error;
				error.type = Token::Type::G_TK_ERROR;
				error.literal = "Identifier index out of bounds.";
				return error;
			}
			token.literal = identifiers[identifier_pos];
		} break;
		case GDScriptV2TokenizerCompat::Token::Type::G_TK_ERROR:
		case GDScriptV2TokenizerCompat::Token::Type::G_TK_CONSTANT: {
			// Get literal from map.
			int constant_pos = token_type >> TOKEN_BITS;
			if (unlikely(constant_pos >= constants.size())) {
				Token error;
				error.type = Token::Type::G_TK_ERROR;
				error.literal = "Constant index out of bounds.";
				return error;
			}
			token.literal = constants[constant_pos];
		} break;
		default:
			break;
	}

	return token;
}

Error GDScriptV2TokenizerBufferCompat::set_code_buffer(const Vector<uint8_t> &p_buffer) {
	const uint8_t *buf = p_buffer.ptr();