Method encode

transformers/diffusion/engine/src/tokenizer.cpp:75–91

}

std::vector<int> MtokTokenizer::encode(const std::string& sentence, int maxlen) {
#if defined(MNN_DIFFUSION_WITH_LLM_TOKENIZER)
    if (mStyle == Style::kPair) {
        std::vector<int> ids(maxlen * 2, 0);
        auto uncond = encodeSingle("", maxlen);
        auto cond = encodeSingle(sentence, maxlen);
        std::copy(uncond.begin(), uncond.end(), ids.begin());
        std::copy(cond.begin(), cond.end(), ids.begin() + maxlen);
        return ids;
    }
    return encodeSingle(sentence, maxlen);
#else
    (void)sentence;
    (void)maxlen;
    return {};
#endif
}

} // namespace DIFFUSION
} // namespace MNN