Function bind_transformer

src/models/miocodec/weights.cpp:369–415

}

MioCodecTransformerWeights bind_transformer(
    MioCodecWeights & weights,
    const engine::assets::TensorSource & source,
    const std::string & prefix,
    int64_t dim,
    int64_t layers,
    int64_t heads,
    int64_t window_size,
    bool use_adaln,
    int64_t condition_dim = 0,
    std::optional<int64_t> output_dim = std::nullopt,
    engine::assets::TensorStorageType storage_type = engine::assets::TensorStorageType::F32) {
    MioCodecTransformerWeights transformer;
    transformer.dim = dim;
    transformer.heads = heads;
    transformer.head_dim = dim / heads;
    transformer.window_size = window_size;
    transformer.intermediate_dim = swiglu_hidden_dim(dim);
    transformer.use_adaln = use_adaln;
    transformer.layers.reserve(static_cast<size_t>(layers));
    for (int64_t layer = 0; layer < layers; ++layer) {
        const std::string layer_prefix = prefix + ".layers." + std::to_string(layer);
        MioCodecTransformerLayerWeights w;
        w.qkv_proj = bind_qkv_linear(weights, source, layer_prefix, dim, storage_type);
        w.out_proj = bind_linear(weights, layer_prefix + ".attention.wo", dim, dim, false);
        w.feed_forward_w1 = bind_linear(weights, layer_prefix + ".feed_forward.w1", dim, transformer.intermediate_dim, false);
        w.feed_forward_w2 = bind_linear(weights, layer_prefix + ".feed_forward.w2", transformer.intermediate_dim, dim, false);
        w.feed_forward_w3 = bind_linear(weights, layer_prefix + ".feed_forward.w3", dim, transformer.intermediate_dim, false);
        if (use_adaln) {
            w.attention_adaln = bind_adaln(weights, layer_prefix + ".attention_norm", condition_dim, 3 * dim);
            w.feed_forward_adaln = bind_adaln(weights, layer_prefix + ".ffn_norm", condition_dim, 3 * dim);
        } else {
            w.attention_norm = bind_norm(weights, layer_prefix + ".attention_norm", dim);
            w.feed_forward_norm = bind_norm(weights, layer_prefix + ".ffn_norm", dim);
        }
        transformer.layers.push_back(std::move(w));
    }
    if (use_adaln) {
        transformer.adaln_norm = bind_adaln(weights, prefix + ".norm", condition_dim, 2 * dim);
    } else {
        transformer.norm = bind_norm(weights, prefix + ".norm", dim);
    }
    if (output_dim.has_value()) {
        transformer.output_projection = bind_linear(weights, prefix + ".output_proj", dim, *output_dim);
    }
    return transformer;
}

MioCodecSnakeBetaWeights bind_snake_beta(
    MioCodecWeights & weights,