Function pipe_quant_42

tests/test_lmdeploy/test_quant_policy.py:58–76

Create pipeline with quant_policy=QuantPolicy.TURBO_QUANT. This fixture has class scope so large model instances are released before later FP8 accuracy tests allocate their own pipelines.

(model_id)

@pytest.fixture(scope='class')
def pipe_quant_42(model_id):
    """Create pipeline with quant_policy=QuantPolicy.TURBO_QUANT.

    This fixture has class scope so large model instances are released before later FP8 accuracy tests allocate their
    own pipelines.
    """
    engine_config = PytorchEngineConfig(
        tp=1,
        cache_max_entry_count=0.05,
        quant_policy=QuantPolicy.TURBO_QUANT,  # K=4bit, V=2bit mixed precision
    )
    pipe = pipeline(model_id, backend_config=engine_config, log_level='INFO')
    yield pipe
    # Cleanup
    pipe.close()
    del pipe
    gc.collect()
    if torch.cuda.is_available() and torch.cuda.device_count() > 0:
        torch.cuda.empty_cache()


# =============================================================================