Method runVideo

transformers/diffusion/engine/src/wan_diffusion.cpp:274–383

}

bool WanDiffusion::runVideo(const std::string& prompt, const std::string& outputDir, int width, int height, int frames,
                            int steps, int seed, float cfgScale, std::function<void(int)> progressCallback) {
    AUTOTIME;
    if (mModules.size() < 3 || !mModules[0] || !mModules[1] || !mModules[2]) {
        MNN_ERROR("Wan modules are not loaded. Please call load() first.\n");
        return false;
    }
    if (width <= 0 || height <= 0 || frames <= 0) {
        MNN_ERROR("Wan video shape must be positive, got width=%d height=%d frames=%d\n", width, height, frames);
        return false;
    }
    const int vaeScaleFactor = 8;
    const int transformerPatchSize = 2;
    const int spatialAlignment = vaeScaleFactor * transformerPatchSize;
    if (width % spatialAlignment != 0 || height % spatialAlignment != 0) {
        MNN_ERROR("Wan width and height must be multiples of %d, got %dx%d\n", spatialAlignment, width, height);
        return false;
    }
    if (steps <= 0) {
        MNN_PRINT("Warning: Wan steps must be positive; using 1\n");
        steps = 1;
    }

    int seqLen = mMaxTextLen;
    VARP encoderAttentionMask;
    auto encoderHiddenStates = encodePrompt(prompt, &seqLen, &encoderAttentionMask);
    if (encoderHiddenStates.get() == nullptr) {
        return false;
    }
    if (mMemoryMode != 1) {
        mModules[0].reset();
        MNN::Express::Executor::getGlobalExecutor()->gc(MNN::Express::Executor::FULL);
    }

    int latentFrames = std::max(1, (frames + 3) / 4);
    int latentH = height / vaeScaleFactor;
    int latentW = width / vaeScaleFactor;
    int latentChannels = mLatentChannels;
    int latentSize = latentChannels * latentFrames * latentH * latentW;

    std::vector<float> noise(latentSize);
    int realSeed = seed < 0 ? std::random_device()() : seed;
    std::mt19937 rng(realSeed);
    std::normal_distribution<float> normal(0.0f, 1.0f);
    for (int i = 0; i < latentSize; ++i) {
        noise[i] = normal(rng);
    }

    VARP latent = _Input({1, latentChannels, latentFrames, latentH, latentW}, NCHW, halide_type_of<float>());
    ::memcpy(latent->writeMap<float>(), noise.data(), noise.size() * sizeof(float));

    mTimesteps.resize(steps);
    if (steps == 1) {
        mTimesteps[0] = 1000.0f;
    } else {
        float shift = 3.0f;
        for (int i = 0; i < steps; ++i) {
            float tLinear = 1.0f + i * (0.001f - 1.0f) / (float)(steps - 1);