Function BuildMtokPath

transformers/diffusion/engine/src/tokenizer.cpp:14–16

namespace DIFFUSION {

static std::string BuildMtokPath(const std::string& filePath) {
    return filePath + "/tokenizer.mtok";
}

MtokTokenizer::MtokTokenizer(Style style, int bosId, int eosId) : mStyle(style), mBosId(bosId), mEosId(eosId) {}