(
self,
model_path: str,
model_config: ModelConfig,
cache_config: CacheConfig,
backend_config: BackendConfig,
misc_config: MiscConfig,
dist_ctx: DistContext,
device_ctx: DeviceContext,
adapters: dict[str, str] = None,
specdecode_config: SpecDecodeConfig = None,
trust_remote_code: bool = False
)
| 217 | """ |
| 218 | |
| 219 | def __init__( |
| 220 | self, |
| 221 | model_path: str, |
| 222 | model_config: ModelConfig, |
| 223 | cache_config: CacheConfig, |
| 224 | backend_config: BackendConfig, |
| 225 | misc_config: MiscConfig, |
| 226 | dist_ctx: DistContext, |
| 227 | device_ctx: DeviceContext, |
| 228 | adapters: dict[str, str] = None, |
| 229 | specdecode_config: SpecDecodeConfig = None, |
| 230 | trust_remote_code: bool = False |
| 231 | ): |
| 232 | |
| 233 | self.model_config = model_config |
| 234 | self.cache_config = cache_config |
| 235 | # use raw tokenizer |
| 236 | if dist_ctx.dist_config.world_size > 1: |
| 237 | monkey_patch_hf_modules_cache() |
| 238 | self.tokenizer = Tokenizer(model_path, trust_remote_code=trust_remote_code).model.model |
| 239 | |
| 240 | # asyncio |
| 241 | self._pre_in_que = None |
| 242 | self._in_que = None |
| 243 | self._out_que = None |
| 244 | self._background_task = None |
| 245 | self._preprocess_task = None |
| 246 | self.tasks = set() |
| 247 | |
| 248 | # cuda stream |
| 249 | self.stream = torch.cuda.Stream() |
| 250 | self.out_stream = torch.cuda.Stream() |
| 251 | self.cache_stream = torch.cuda.Stream() |
| 252 | |
| 253 | self.dist_ctx = dist_ctx |
| 254 | self.device_ctx = device_ctx |
| 255 | |
| 256 | device = 'cuda' |
| 257 | self.backend_config = backend_config |
| 258 | self.misc_config = misc_config |
| 259 | self.dist_config = dist_ctx.dist_config |
| 260 | rank = dist_ctx.rank |
| 261 | |
| 262 | self.model_path = model_path |
| 263 | self.adapters = adapters |
| 264 | self.device = device |
| 265 | self.rank = rank |
| 266 | |
| 267 | tp = self.dist_config.tp |
| 268 | world_size = self.dist_config.world_size |
| 269 | self.tp = tp |
| 270 | self.world_size = world_size |
| 271 | self.need_output = rank % self.dist_config.attn_tp == 0 |
| 272 | |
| 273 | self.patched_model = None |
| 274 | self.cache_engine = None |
| 275 | self.state_cache_engine = None |
| 276 | self.profiler: AgentProfiler = None |
nothing calls this directly
no test coverage detected