(cache_dtype: str)
| 1480 | |
| 1481 | @staticmethod |
| 1482 | def get_cache_bytes(cache_dtype: str): |
| 1483 | if any(t in cache_dtype.lower() for t in ["float32", "fp32"]): |
| 1484 | return 4 |
| 1485 | elif any(t in cache_dtype.lower() for t in ["float16", "bf16", "fp16"]): |
| 1486 | return 2 |
| 1487 | elif any(t in cache_dtype.lower() for t in ["uint8", "int8", "float8", "fp8"]): |
| 1488 | return 1 |
| 1489 | elif any(t in cache_dtype.lower() for t in ["int4", "float4"]): |
| 1490 | return 0.5 |
| 1491 | else: |
| 1492 | raise ValueError(f"Unsupported cache dtype: {cache_dtype}") |
| 1493 | |
| 1494 | def metrics_info(self): |
| 1495 | """Convert cache_config to dict(key: str, value: str) for prometheus metrics info.""" |
no outgoing calls