(self, params)
| 253 | |
| 254 | class Transformer(nn.Module): |
| 255 | def __init__(self, params): |
| 256 | super().__init__() |
| 257 | self.params = params |
| 258 | self.vocab_size = params.vocab_size |
| 259 | self.n_layers = params.n_layer |
| 260 | |
| 261 | self.word_embeddings = nn.Embedding(params.vocab_size, params.hidden_size) |
| 262 | |
| 263 | self.h = torch.nn.ModuleList() |
| 264 | for layer_id in range(params.n_layer): |
| 265 | self.h.append(TransformerBlock(layer_id, params)) |
| 266 | |
| 267 | self.ln_f = nn.LayerNorm(params.hidden_size, eps=params.layer_norm_epsilon) |
| 268 | |
| 269 | @torch.inference_mode() |
| 270 | def forward(self, tokens: torch.Tensor, start_pos: int): |
nothing calls this directly
no test coverage detected