| 322 | } |
| 323 | |
| 324 | double Block::active_bytes(size_t pos) const { |
| 325 | double bytes_per_weight = bits_per_weight(_config->weight_quant, _config->block_size[0] * _config->block_size[1]) / 8.0; |
| 326 | |
| 327 | double bytes = 0; |
| 328 | bytes += _rms_att_weight->size; |
| 329 | bytes += _rms_ffn_weight->size; |
| 330 | if (_config->n_routed_experts > 0 && _w1->ndim() == 3) { |
| 331 | bytes += _moegate->size; |
| 332 | if (_moegate_bias) bytes += _moegate_bias->size; |
| 333 | // bytes_per_weight accounts for scales and other quantization schemes |
| 334 | bytes += _config->n_active_routed * 3 * _config->dim * _config->moe_intermediate_size * bytes_per_weight; // w1, w2, w3 |
| 335 | } else { |
| 336 | bytes += _w1->size + _w2->size + _w3->size; // w1, w2, w3 |
| 337 | if (_s1) { |
| 338 | bytes += _s1->size; |
| 339 | bytes += _s2->size; |
| 340 | bytes += _s3->size; |
| 341 | } |
| 342 | } |
| 343 | if (_config->n_shared_experts > 0) { |
| 344 | if (_shared_w1) bytes += _shared_w1->size; |
| 345 | if (_shared_s1) bytes += _shared_s1->size; |
| 346 | if (_shared_w2) bytes += _shared_w2->size; |
| 347 | if (_shared_s2) bytes += _shared_s2->size; |
| 348 | if (_shared_w3) bytes += _shared_w3->size; |
| 349 | if (_shared_s3) bytes += _shared_s3->size; |
| 350 | } |
| 351 | return bytes; |
| 352 | } |
| 353 | |
| 354 | BlockMHA::BlockMHA( |
| 355 | int layer_i, |
no test coverage detected