Method encode_prompt

transformers/diffusion/engine/src/diffusion_sd35.cpp:162–246

}

std::pair<VARP, VARP> DiffusionSD35::encode_prompt(const std::string& prompt) {
    auto run_encoder = [&](int module_index, VARP input_ids, const char* name) {
        auto outputs = mModules[module_index]->onForward({input_ids});
        std::vector<VARP> safe_outputs;
        for (auto& out : outputs) {
            auto info = out->getInfo();
            const void* ptr = out->readMap<void>();
            auto new_var = _Const(ptr, info->dim, info->order, info->type);
            safe_outputs.push_back(new_var);
        }
        if (mMemoryMode != 1) {
            mModules[module_index].reset();
            MNN_PRINT("%s Module unloaded.\n", name);
            MNN::Express::Executor::getGlobalExecutor()->gc(MNN::Express::Executor::FULL);
        }
        return safe_outputs;
    };

    // 1. CLIP L
    loadModule(0);
    auto ids1 = mTokenizer1->encode(prompt, mMaxTextLen);
    VARP input_ids1 = _Input({2, mMaxTextLen}, NCHW, halide_type_of<int>());
    int* inputs1_ptr = input_ids1->writeMap<int>();
    memset(inputs1_ptr, 0, 2 * mMaxTextLen * sizeof(int));
    if (ids1.size() > 0) {
        size_t copy_size = std::min(ids1.size(), (size_t)(2 * mMaxTextLen));
        memcpy(inputs1_ptr, ids1.data(), copy_size * sizeof(int));
    }

    auto out1 = run_encoder(0, input_ids1, "CLIP L");
    auto clip_l_hidden = out1[0]; // (2, 77, 768)
    auto clip_l_pooled = out1[1]; // (2, 768)

    // 2. CLIP G
    loadModule(1);
    auto ids2 = mTokenizer2->encode(prompt, mMaxTextLen);
    VARP input_ids2 = _Input({2, mMaxTextLen}, NCHW, halide_type_of<int>());
    int* inputs2_ptr = input_ids2->writeMap<int>();
    memset(inputs2_ptr, 0, 2 * mMaxTextLen * sizeof(int));
    if (ids2.size() > 0) {
        size_t copy_size = std::min(ids2.size(), (size_t)(2 * mMaxTextLen));
        memcpy(inputs2_ptr, ids2.data(), copy_size * sizeof(int));
    }

    auto out2 = run_encoder(1, input_ids2, "CLIP G");
    auto clip_g_hidden = out2[0]; // (2, 77, 1280)
    auto clip_g_pooled = out2[1]; // (2, 1280)

    // 3. T5
    VARP t5_hidden;
    loadModule(2);
    if (mModules[2]) {
        auto ids3 = mTokenizer3->encode(prompt, mMaxTextLenT5);
        VARP input_ids3 = _Input({2, mMaxTextLenT5}, NCHW, halide_type_of<int>());

        // Uncond for T5 is empty string
        auto ids3_uncond = mTokenizer3->encode("", mMaxTextLenT5);