LLMAnalysis constructor. Args: model_config (ModelConfig): model configuration gpu_config (GPUConfig): GPU configuration dtype_config (DtypeConfig, optional): data type configuration. Defaults to DtypeConfig(). parallelism_config (ParallelismC
(
self,
model_config: ModelConfig,
gpu_config: GPUConfig,
dtype_config: DtypeConfig = DtypeConfig(),
parallelism_config: ParallelismConfig = ParallelismConfig(),
achieved_tflops: float = None,
achieved_memory_bandwidth_GBs: float = None,
flops_efficiency: float = None,
hbm_memory_efficiency: float = None,
intra_node_memory_efficiency: float = INTRA_NODE_MEMORY_EFFICIENCY,
inter_node_memory_efficiency: float = INTER_NODE_MEMORY_EFFICIENCY,
)
| 84 | """ |
| 85 | |
| 86 | def __init__( |
| 87 | self, |
| 88 | model_config: ModelConfig, |
| 89 | gpu_config: GPUConfig, |
| 90 | dtype_config: DtypeConfig = DtypeConfig(), |
| 91 | parallelism_config: ParallelismConfig = ParallelismConfig(), |
| 92 | achieved_tflops: float = None, |
| 93 | achieved_memory_bandwidth_GBs: float = None, |
| 94 | flops_efficiency: float = None, |
| 95 | hbm_memory_efficiency: float = None, |
| 96 | intra_node_memory_efficiency: float = INTRA_NODE_MEMORY_EFFICIENCY, |
| 97 | inter_node_memory_efficiency: float = INTER_NODE_MEMORY_EFFICIENCY, |
| 98 | ) -> None: |
| 99 | """LLMAnalysis constructor. |
| 100 | |
| 101 | Args: |
| 102 | model_config (ModelConfig): model configuration |
| 103 | gpu_config (GPUConfig): GPU configuration |
| 104 | dtype_config (DtypeConfig, optional): data type configuration. Defaults to DtypeConfig(). |
| 105 | parallelism_config (ParallelismConfig, optional): parallelism configuration. Defaults to ParallelismConfig(). |
| 106 | achieved_tflops (float, optional): achieved TFLOPS per GPU. If specified, will override the flops_efficiency passed in. Defaults to None. |
| 107 | achieved_memory_bandwidth_GBs (float, optional): achieved GPU memory bandwidth in GB/s. If specified, will override the hbm_memory_efficiency passed in. Defaults to None. |
| 108 | flops_efficiency (float, optional): flops efficiency, ranging from 0 to 1. Defaults to None. |
| 109 | hbm_memory_efficiency (float, optional): GPU HBM memory efficiency, ranging from 0 to 1. Defaults to None. |
| 110 | intra_node_memory_efficiency (float, optional): intra-node memory efficiency, ranging from 0 to 1. Defaults to INTRA_NODE_MEMORY_EFFICIENCY. |
| 111 | inter_node_memory_efficiency (float, optional): inter-node memory efficiency, ranging from 0 to 1. Defaults to INTER_NODE_MEMORY_EFFICIENCY. |
| 112 | """ |
| 113 | self.model_config = model_config |
| 114 | self.gpu_config = gpu_config |
| 115 | self.parallelism_config = parallelism_config |
| 116 | self.dtype_config = dtype_config |
| 117 | self.intra_node_memory_efficiency = intra_node_memory_efficiency |
| 118 | self.inter_node_memory_efficiency = inter_node_memory_efficiency |
| 119 | |
| 120 | if achieved_memory_bandwidth_GBs and hbm_memory_efficiency: |
| 121 | logger.info( |
| 122 | "both achieved_memory_bandwidth_GBs and hbm_memory_efficiency are set, using achieved_memory_bandwidth_GBs({achieved_memory_bandwidth_GBs} GB/s) to calculate hbm_memory_efficiency" |
| 123 | ) |
| 124 | self.hbm_memory_efficiency = ( |
| 125 | achieved_memory_bandwidth_GBs / |
| 126 | gpu_config.hbm_bandwidth_in_GB_per_sec) |
| 127 | elif hbm_memory_efficiency: |
| 128 | self.hbm_memory_efficiency = hbm_memory_efficiency |
| 129 | elif achieved_memory_bandwidth_GBs: |
| 130 | self.hbm_memory_efficiency = ( |
| 131 | achieved_memory_bandwidth_GBs / |
| 132 | gpu_config.hbm_bandwidth_in_GB_per_sec) |
| 133 | else: |
| 134 | self.hbm_memory_efficiency = HBM_MEMORY_EFFICIENCY |
| 135 | |
| 136 | assert self.hbm_memory_efficiency > 0 and self.hbm_memory_efficiency <= 1, ( |
| 137 | "hbm_memory_efficiency must be in (0, 1], check the achieved_memory_bandwidth_GBs and hbm_memory_efficiency passed in" |
| 138 | ) |
| 139 | logger.info(f"hbm_memory_efficiency: {self.hbm_memory_efficiency}") |
| 140 | |
| 141 | if achieved_tflops and flops_efficiency: |
| 142 | logger.info( |
| 143 | "both achieved_tflops and flops_efficiency are set, using achieved_tflops({achieved_tflops} TFLOPS) to calculate flops_efficiency" |
nothing calls this directly
no test coverage detected