Function run_MTP

examples/llm-api/llm_speculative_decoding.py:18–32

(model: Optional[str] = None)

def run_MTP(model: Optional[str] = None):
    spec_config = MTPDecodingConfig(num_nextn_predict_layers=1,
                                    use_relaxed_acceptance_for_thinking=True,
                                    relaxed_topk=10,
                                    relaxed_delta=0.01)

    llm = LLM(
        # You can change this to a local model path if you have the model downloaded
        model=model or "nvidia/DeepSeek-R1-FP4",
        speculative_config=spec_config,
    )

    for prompt in prompts:
        response = llm.generate(prompt, SamplingParams(max_tokens=10))
        print(response.outputs[0].text)


def run_Eagle3():