(self, config: ChatGLMConfig, device=None, empty_init=True)
| 732 | |
| 733 | class ChatGLMModel(ChatGLMPreTrainedModel): |
| 734 | def __init__(self, config: ChatGLMConfig, device=None, empty_init=True): |
| 735 | super().__init__(config) |
| 736 | if empty_init: |
| 737 | init_method = skip_init |
| 738 | else: |
| 739 | init_method = default_init |
| 740 | init_kwargs = {} |
| 741 | if device is not None: |
| 742 | init_kwargs["device"] = device |
| 743 | self.embedding = init_method(Embedding, config, **init_kwargs) |
| 744 | self.num_layers = config.num_layers |
| 745 | self.multi_query_group_num = config.multi_query_group_num |
| 746 | self.kv_channels = config.kv_channels |
| 747 | |
| 748 | # Rotary positional embeddings |
| 749 | self.seq_length = config.seq_length |
| 750 | rotary_dim = ( |
| 751 | config.hidden_size // config.num_attention_heads if config.kv_channels is None else config.kv_channels |
| 752 | ) |
| 753 | |
| 754 | self.rotary_pos_emb = RotaryEmbedding(rotary_dim // 2, original_impl=config.original_rope, device=device, |
| 755 | dtype=config.torch_dtype) |
| 756 | self.encoder = init_method(GLMTransformer, config, **init_kwargs) |
| 757 | self.output_layer = init_method(nn.Linear, config.hidden_size, config.padded_vocab_size, bias=False, |
| 758 | dtype=config.torch_dtype, **init_kwargs) |
| 759 | self.pre_seq_len = config.pre_seq_len |
| 760 | self.prefix_projection = config.prefix_projection |
| 761 | if self.pre_seq_len is not None: |
| 762 | for param in self.parameters(): |
| 763 | param.requires_grad = False |
| 764 | self.prefix_tokens = torch.arange(self.pre_seq_len).long() |
| 765 | self.prefix_encoder = PrefixEncoder(config) |
| 766 | self.dropout = torch.nn.Dropout(0.1) |
| 767 | |
| 768 | def get_input_embeddings(self): |
| 769 | return self.embedding.word_embeddings |
nothing calls this directly
no test coverage detected