Method Vevo2ARDecodeGraph

src/models/vevo2/ar.cpp:709–791

                core::TensorShape::from_dims({1, cache_steps + 1, config.num_key_value_heads, dim})));
            auto out = decoder_layer_with_static_cache_tail(
                build_ctx,
                graph,
                x,
                position_values,
                layer,
                config,
                cache_keys.back(),
                cache_values.back(),
                mask);
            x = out.output;
            key_sources.push_back(ggml_view_1d(ctx.get(), out.key.tensor, config.num_key_value_heads * dim, 0));
            value_sources.push_back(ggml_view_1d(ctx.get(), out.value.tensor, config.num_key_value_heads * dim, 0));
        }
        step_cache = runtime::TransformerKVCache(
            cache_steps + 1,
            config.num_key_value_heads * dim,
            std::move(cache_keys),
            std::move(cache_values));
        build_transfer_views(config.num_key_value_heads * dim);
        x = modules::RMSNormModule({config.hidden_size, config.rms_norm_eps, true, false})
                .build(build_ctx, x, this->weights->norm);
        logits = modules::LinearModule({config.hidden_size, config.vocab_size, false})
                     .build(build_ctx, x, {this->weights->lm_head, std::nullopt})
                     .tensor;
        ggml_set_output(logits);
        ggml_build_forward_expand(graph, logits);
        buffer = ggml_backend_alloc_ctx_tensors(ctx.get(), backend);
        if (buffer == nullptr) {
            throw std::runtime_error("failed to allocate Vevo2 AR decode graph");
        }
        attention_mask_values.assign(static_cast<size_t>(cache_steps + 1), ggml_fp32_to_fp16(-INFINITY));
    }

    ~Vevo2ARDecodeGraph() {
        engine::core::release_backend_graph_resources(backend, graph);
        if (buffer != nullptr) {
            ggml_backend_buffer_free(buffer);
        }
    }

    bool can_run(const Vevo2ARWeights & other_weights, int64_t required_steps) const noexcept {
        return weights.get() == &other_weights && cache_steps >= required_steps;
    }

    void import_state(const runtime::TransformerKVState & state) {
        step_cache.import_state(state);
    }

    std::vector<float> run_step(int32_t token, const Vevo2ARConfig & config) {
        if (step_cache.valid_steps() >= cache_steps) {
            throw std::runtime_error("Vevo2 AR decode cache exhausted");
        }
        ggml_backend_tensor_set(token_id, &token, 0, sizeof(int32_t));
        const int32_t position = static_cast<int32_t>(step_cache.current_end());
        ggml_backend_tensor_set(positions, &position, 0, sizeof(int32_t));
        const auto masked = ggml_fp32_to_fp16(-INFINITY);
        const auto visible = ggml_fp32_to_fp16(0.0F);
        std::fill(attention_mask_values.begin(), attention_mask_values.end(), masked);