Method Block

src/model.cpp:149–286

};

Block::Block(
  int layer_i,
  const std::shared_ptr<Config> config,
  const Tensor* rms_att_weight,
  const Tensor* rms_ffn_weight,
  const Tensor* w1,
  const Tensor* s1,
  const Tensor* w2,
  const Tensor* s2,
  const Tensor* w3,
  const Tensor* s3,
  const Tensor* shared_w1,
  const Tensor* shared_s1,
  const Tensor* shared_w2,
  const Tensor* shared_s2,
  const Tensor* shared_w3,
  const Tensor* shared_s3,
  const Tensor* moegate,
  const Tensor* moegate_bias
) : _layer_i(layer_i), _config(config) {
  switch (config->weight_quant) {
    case Quant::F32:
    case Quant::F16:
    case Quant::F8E5M2:
    case Quant::Q2_K:
    case Quant::Q3_K: {
      break;
    }
    default: {
      std::cerr << "FATAL: unsupported weight quantization: " << quant_to_string(config->weight_quant) << std::endl;
      assert(false);
      break;
    }
  }

  _rms_att_weight = check_tensor(
    rms_att_weight, Quant::F32, {config->dim, 0, 0, 0}, __LINE__
  );
  _rms_ffn_weight = check_tensor(
    rms_ffn_weight, Quant::F32, {config->dim, 0, 0, 0}, __LINE__
  );

  bool need_block_scales = _config->weight_quant == Quant::F8E5M2;
  int b0 = config->block_size[0];
  int b1 = config->block_size[1];

  if (config->n_routed_experts > 0 && layer_i >= config->first_k_dense_replace) {
    _moegate = check_tensor(
      moegate, Quant::F32, {config->n_routed_experts, config->dim, 0, 0}, __LINE__
    );
    if (moegate_bias != nullptr) {
      _moegate_bias = check_tensor(
        moegate_bias, Quant::F32, {config->n_routed_experts, 0, 0, 0}, __LINE__
      );
    }
    _w1 = check_tensor(
      w1, config->weight_quant, {config->n_routed_experts, config->moe_intermediate_size, config->dim, 0}, __LINE__
    );