Function parse_vocab_format

cactus/engine/engine_tokenizer.cpp:30–34

}

TokenizerRuntimeConfig::VocabFormat parse_vocab_format(const std::string& value) {
    if (value == "id_tab_token") return TokenizerRuntimeConfig::VocabFormat::ID_TAB_TOKEN;
    if (value == "line_token") return TokenizerRuntimeConfig::VocabFormat::LINE_TOKEN;
    return TokenizerRuntimeConfig::VocabFormat::UNKNOWN;
}

TokenizerRuntimeConfig::Normalizer parse_normalizer(const std::string& value) {
    if (value == "metaspace") return TokenizerRuntimeConfig::Normalizer::METASPACE;