Configuration for EPLB manager.
| 1321 | |
| 1322 | |
| 1323 | class EPLBConfig: |
| 1324 | """ |
| 1325 | Configuration for EPLB manager. |
| 1326 | """ |
| 1327 | |
| 1328 | def __init__( |
| 1329 | self, |
| 1330 | args, |
| 1331 | ): |
| 1332 | if args is None: |
| 1333 | args = {} |
| 1334 | |
| 1335 | # enable eplb |
| 1336 | self.enable_eplb: bool = False |
| 1337 | # redundant experts num |
| 1338 | self.redundant_experts_num: int = 0 |
| 1339 | # expert ip shm size |
| 1340 | self.redundant_expert_ip_shm_size: int = 1024 |
| 1341 | # expert meta dir |
| 1342 | self.redundant_expert_meta_dir: str = "/tmp/redundant_expert_meta" |
| 1343 | # expert api user and password |
| 1344 | self.redundant_expert_api_user: str = "" |
| 1345 | self.redundant_expert_api_password: str = "" |
| 1346 | # expert eplb strategy |
| 1347 | self.redundant_expert_eplb_strategy: str = "" |
| 1348 | # expert dump workload interval |
| 1349 | self.redundant_expert_dump_workload_interval: int = 10 |
| 1350 | # expert async load model shmem size gb |
| 1351 | self.redundant_expert_async_load_model_shmem_size_gb: int = 0 |
| 1352 | # expert enable schedule cordon |
| 1353 | self.redundant_expert_enable_schedule_cordon: bool = True |
| 1354 | # model use safetensors |
| 1355 | self.model_use_safetensors: bool = True |
| 1356 | # model use offline quant |
| 1357 | self.model_use_offline_quant: bool = True |
| 1358 | # moe quant type |
| 1359 | self.moe_quant_type: str = "w4a8" |
| 1360 | for key, value in args.items(): |
| 1361 | if hasattr(self, key): |
| 1362 | setattr(self, key, value) |
| 1363 | |
| 1364 | def to_json_string(self): |
| 1365 | """ |
| 1366 | Convert eplb_config to json string. |
| 1367 | """ |
| 1368 | return json.dumps({key: value for key, value in self.__dict__.items() if value is not None}) |
| 1369 | |
| 1370 | def print(self): |
| 1371 | """ |
| 1372 | Print all configuration information. |
| 1373 | """ |
| 1374 | logger.info("EPLB Configuration Information :") |
| 1375 | for k, v in self.__dict__.items(): |
| 1376 | logger.info("{:<20}:{:<6}{}".format(k, "", v)) |
| 1377 | logger.info("=============================================================") |
| 1378 | |
| 1379 | |
| 1380 | class CacheConfig: |
no outgoing calls