(self, huggingface_model_id: str)
| 99 | ) |
| 100 | class Model: |
| 101 | def __init__(self, huggingface_model_id: str): |
| 102 | model_dir = merged_model_cache_dir(huggingface_model_id) |
| 103 | cache_model_weights(huggingface_model_id, model_dir) |
| 104 | |
| 105 | logging.info("Preloading model") |
| 106 | read_all_files(model_dir) |
| 107 | |
| 108 | logging.info(f"Loading model from volume {model_dir}") |
| 109 | self.engine = AsyncLLMEngine.from_engine_args(AsyncEngineArgs(model=model_dir)) |
| 110 | |
| 111 | @modal.method() |
| 112 | async def generate(self, request: Input) -> Output: |
nothing calls this directly
no test coverage detected