Function encode_chat_messages

deepspec/data/parser.py:183–200

(
    tokenizer,
    messages,
    *,
    add_generation_prompt: bool,
    enable_thinking: bool | None = None,
)

def encode_chat_messages(
    tokenizer,
    messages,
    *,
    add_generation_prompt: bool,
    enable_thinking: bool | None = None,
) -> torch.LongTensor:
    conversation_text = render_chat_messages(
        tokenizer,
        messages,
        add_generation_prompt=add_generation_prompt,
        enable_thinking=enable_thinking,
    )
    return tokenizer(
        conversation_text,
        return_tensors="pt",
        add_special_tokens=False,
    ).input_ids


def preprocess_record(