(self, base_model_id: str)
| 84 | ) |
| 85 | class LoraBaseModel: |
| 86 | def __init__(self, base_model_id: str): |
| 87 | model_dir = merged_model_cache_dir(base_model_id) |
| 88 | logging.info(f"Loading base model {model_dir}") |
| 89 | self.seen_models = set() |
| 90 | |
| 91 | self.engine = AsyncLLMEngine.from_engine_args( |
| 92 | AsyncEngineArgs( |
| 93 | model=base_model_id, |
| 94 | enable_lora=True, |
| 95 | max_loras=MAX_INPUTS, |
| 96 | max_lora_rank=8, |
| 97 | max_cpu_loras=500, |
| 98 | ) |
| 99 | ) |
| 100 | |
| 101 | @modal.method() |
| 102 | async def generate(self, request: Input) -> Output: |
nothing calls this directly
no test coverage detected