Method from_model_config_cpp

tensorrt_llm/runtime/generation.py:659–691

Create a partially initialized ModelConfig instance from a given ModelConfig CPP binding instance. Note that each of these classes have fields that don't exist in the other, so the created ModelConfigPython won't have all of its fields initialized.

(cls, model_config_cpp)

    @classmethod
    def from_model_config_cpp(cls, model_config_cpp) -> 'ModelConfig':
        """Create a partially initialized ModelConfig instance from a given ModelConfig CPP binding instance.

        Note that each of these classes have fields that don't exist in the other, so the created ModelConfigPython
        won't have all of its fields initialized.
        """
        return cls(
            max_batch_size=model_config_cpp.max_batch_size,
            max_beam_width=model_config_cpp.max_beam_width,
            vocab_size=model_config_cpp.vocab_size,
            num_layers=model_config_cpp.num_layers(),
            num_heads=model_config_cpp.num_heads,
            num_kv_heads=model_config_cpp.num_kv_heads(0),
            hidden_size=model_config_cpp.hidden_size,
            remove_input_padding=model_config_cpp.use_packed_input,
            kv_cache_type=model_config_cpp.kv_cache_type,
            cross_attention=model_config_cpp.use_cross_attention,
            head_size=model_config_cpp.head_size,
            max_prompt_embedding_table_size=model_config_cpp.
            max_prompt_embedding_table_size,
            quant_mode=QuantMode(model_config_cpp.quant_mode.value),
            gather_context_logits=model_config_cpp.compute_context_logits,
            gather_generation_logits=model_config_cpp.compute_generation_logits,
            gpt_attention_plugin=model_config_cpp.use_gpt_attention_plugin,
            dtype=binding_to_str_dtype(model_config_cpp.data_type),
            num_kv_heads_per_layer=model_config_cpp.num_kv_heads_per_layer,
            tokens_per_block=model_config_cpp.tokens_per_block,
            lora_plugin=model_config_cpp.use_lora_plugin,
            layer_types=[
                binding_layer_type_to_str(lt)
                for lt in model_config_cpp.layer_types
            ],
        )


@dataclass