Function profile_text_generation

tests/profile_inference.py:33–72

Profile text generation inference.

()

def profile_text_generation():
    """Profile text generation inference."""
    import torch

    from questions.inference_server.model_cache import ModelCache
    from questions.models import GenerateParams
    from questions.text_generator_inference import fast_inference

    print("Setting up profiling environment...")

    model_cache = ModelCache()

    # Test parameters
    params = GenerateParams(
        text="Once upon a time in a galaxy far, far away",
        max_length=100,
        temperature=0.7,
        top_p=0.9,
        top_k=40,
        number_of_results=1,
        model="any",
    )

    # Warmup
    print("Running warmup...")
    for _ in range(2):
        fast_inference(params, model_cache)
        if torch.cuda.is_available():
            torch.cuda.synchronize()

    print("Starting profiled runs...")

    # Profile multiple runs
    for i in range(5):
        result = fast_inference(params, model_cache)
        if torch.cuda.is_available():
            torch.cuda.synchronize()
        print(f"  Run {i+1}/5 complete")

    return result


def main():