Method Tokenizer

src/common/tokenizer/tokenizer.cpp:17–48

\brief Constructor \param model_path the model path

/// \brief Constructor
/// \param model_path the model path
Tokenizer::Tokenizer(const std::string& model_path) {
    #ifdef _WIN32
    std::ifstream fs(model_path + "\\tokenizer.json", std::ios::in | std::ios::binary);
    #else
    std::ifstream fs(model_path + "/tokenizer.json", std::ios::in | std::ios::binary);
    #endif
    if (fs.fail()) {
        #ifdef _WIN32
        std::cerr << "Cannot open " << model_path + "\\tokenizer.json" << std::endl;
        #else
        std::cerr << "Cannot open " << model_path + "/tokenizer.json" << std::endl;
        #endif
        exit(1);
    }
    std::string data;
    fs.seekg(0, std::ios::end);
    size_t size = static_cast<size_t>(fs.tellg());
    fs.seekg(0, std::ios::beg);
    data.resize(size);
    fs.read(data.data(), size);
    this->tokenizer = tokenizers::Tokenizer::FromBlobJSON(data);
    fs.close();
    std::string decoder_type;
    nlohmann::json data_json = nlohmann::json::parse(data);
    JSON_GET(decoder_type, data_json["decoder"], "type", "ByteLevel", std::string);
    if (decoder_type == "ByteLevel") {
        this->is_doubled_encoded = true;
    }
    else {
        this->is_doubled_encoded = false;
    }
}

/// \brief Destructor
Tokenizer::~Tokenizer() = default;