(self, tokens)
| 114 | self.output = nn.Linear(args.dim, args.vocab_size, bias=False) |
| 115 | |
| 116 | def forward(self, tokens): |
| 117 | _bsz, seq_len = tokens.size() |
| 118 | assert seq_len <= self.max_seq_len |
| 119 | h = self.tok_embeddings(tokens) |
| 120 | pos = torch.arange(0, seq_len, device=tokens.device) |
| 121 | p = self.pos_embeddings(pos) # positional embeddings of shape (seq_len, dim) |
| 122 | h = h + p |
| 123 | h = self.dropout(h) |
| 124 | for layer in self.layers: |
| 125 | h = layer(h) |
| 126 | h = self.norm(h) |
| 127 | output = self.output(h).float() |
| 128 | return output |
| 129 | |
| 130 | def reset_parameters(self): |
| 131 | self.tok_embeddings.reset_parameters() |
nothing calls this directly
no outgoing calls
no test coverage detected