Method load

transformers/diffusion/engine/src/tokenizer.cpp:27–41

}

bool MtokTokenizer::load(const std::string& filePath) {
#if defined(MNN_DIFFUSION_WITH_LLM_TOKENIZER)
    auto mtokPath = BuildMtokPath(filePath);
    delete mTokenizer;
    mTokenizer = MNN::Transformer::Tokenizer::createTokenizer(mtokPath);
    if (mTokenizer == nullptr) {
        MNN_ERROR("Failed to create tokenizer from %s\n", mtokPath.c_str());
    }
    return mTokenizer != nullptr;
#else
    (void)filePath;
    MNN_ERROR("Diffusion tokenizer requires MNN_BUILD_LLM=ON and tokenizer.mtok\n");
    return false;
#endif
}

std::vector<int> MtokTokenizer::encodeSingle(const std::string& sentence, int maxlen) const {
#if defined(MNN_DIFFUSION_WITH_LLM_TOKENIZER)