Function load_tokenizer

cake-core/src/models/common/text_model.rs:19–49

Load the tokenizer and resolve EOS token ID(s). `default_eos_token` is the model-specific fallback (e.g. "<|eot_id|>" for LLaMA, "<|endoftext|>" for Qwen2).

(
    ctx: &Context,
    default_eos_token: &str,
)

/// `default_eos_token` is the model-specific fallback (e.g. "<|eot_id|>" for LLaMA,
/// "<|endoftext|>" for Qwen2).
pub fn load_tokenizer(
    ctx: &Context,
    default_eos_token: &str,
) -> Result<(Tokenizer, Option<EosTokenId>)> {
    // For GGUF files, look for tokenizer.json in the same directory as the .gguf file
    let tokenizer_filename = if ctx.data_path.is_file() {
        ctx.data_path
            .parent()
            .unwrap_or(&ctx.data_path)
            .join("tokenizer.json")
    } else {
        ctx.data_path.join("tokenizer.json")
    };

    log::info!("loading tokenizer from {}", tokenizer_filename.display());

    let tokenizer = Tokenizer::from_file(tokenizer_filename).map_err(anyhow::Error::msg)?;

    let config = ctx.config.as_ref().expect("No config specified");

    let eos_token_id = if config.eos_token_id.is_some() {
        config.eos_token_id.clone()
    } else {
        // Fallback: try to resolve from tokenizer vocabulary
        tokenizer
            .token_to_id(default_eos_token)
            .map(EosTokenId::Single)
    };

    Ok((tokenizer, eos_token_id))
}

/// Apply repeat penalty entirely on GPU to avoid costly GPU↔CPU round-trips.
///