Function parse_normalizer

cactus/engine/engine_tokenizer.cpp:36–40

}

TokenizerRuntimeConfig::Normalizer parse_normalizer(const std::string& value) {
    if (value == "metaspace") return TokenizerRuntimeConfig::Normalizer::METASPACE;
    if (value == "byte_level") return TokenizerRuntimeConfig::Normalizer::BYTE_LEVEL;
    return TokenizerRuntimeConfig::Normalizer::NONE;
}

TokenizerRuntimeConfig::Decoder parse_decoder(const std::string& value) {
    if (value == "replace_metaspace") return TokenizerRuntimeConfig::Decoder::REPLACE_METASPACE;