| 90 | )"; |
| 91 | |
| 92 | struct Transformer { |
| 93 | Tensor qkv; // modelDim * 3 * qkvDim |
| 94 | Tensor rmsNormPre; // modelDim |
| 95 | Tensor rmsNormPost; // modelDim |
| 96 | Tensor out; // 3 * qkvDim * modelDim |
| 97 | Tensor mlp1; // modelDim * (2 * hidden_width * modelDim) |
| 98 | Tensor mlp2; // modelDim * (2 * hidden_width * modelDim) |
| 99 | }; |
| 100 | |
| 101 | struct Activations { |
| 102 | Tensor qkv; // batchSize * 3 * nHeads * qkvDim |
nothing calls this directly
no outgoing calls
no test coverage detected