| 67 | } ParameterTensors; |
| 68 | |
| 69 | void fill_in_parameter_sizes(size_t* param_sizes, GPT2Config config) { |
| 70 | size_t Vp = config.padded_vocab_size; |
| 71 | size_t C = config.channels; |
| 72 | size_t maxT = config.max_seq_len; |
| 73 | size_t L = config.num_layers; |
| 74 | param_sizes[0] = Vp * C; // wte |
| 75 | param_sizes[1] = maxT * C; // wpe |
| 76 | param_sizes[2] = L * C; // ln1w |
| 77 | param_sizes[3] = L * C; // ln1b |
| 78 | param_sizes[4] = L * (3 * C) * C; // qkvw |
| 79 | param_sizes[5] = L * (3 * C); // qkvb |
| 80 | param_sizes[6] = L * C * C; // attprojw |
| 81 | param_sizes[7] = L * C; // attprojb |
| 82 | param_sizes[8] = L * C; // ln2w |
| 83 | param_sizes[9] = L * C; // ln2b |
| 84 | param_sizes[10] = L * (4 * C) * C; // fcw |
| 85 | param_sizes[11] = L * (4 * C); // fcb |
| 86 | param_sizes[12] = L * C * (4 * C); // fcprojw |
| 87 | param_sizes[13] = L * C; // fcprojb |
| 88 | param_sizes[14] = C; // lnfw |
| 89 | param_sizes[15] = C; // lnfb |
| 90 | } |
| 91 | |
| 92 | // allocate memory for the parameters and point the individual tensors to the right places |
| 93 | float* malloc_and_point_parameters(ParameterTensors* params, size_t* param_sizes) { |
no outgoing calls
no test coverage detected