| 143 | } |
| 144 | |
| 145 | void printMetricExplanations(std::ostream& os) |
| 146 | { |
| 147 | os << std::endl; |
| 148 | os << "=== Explanations of the performance metrics ===" << std::endl; |
| 149 | os << "Total Host Walltime: the host walltime from when the first query (after warmups) is enqueued to when the " |
| 150 | "last query is completed." |
| 151 | << std::endl; |
| 152 | os << "GPU Compute Time: the GPU latency to execute the kernels for a query." << std::endl; |
| 153 | os << "Total GPU Compute Time: the summation of the GPU Compute Time of all the queries. If this is significantly " |
| 154 | "shorter than Total Host Walltime, the GPU may be under-utilized because of host-side overheads or data " |
| 155 | "transfers." |
| 156 | << std::endl; |
| 157 | os << "Throughput: the observed throughput computed by dividing the number of queries by the Total Host Walltime. " |
| 158 | "If this is significantly lower than the reciprocal of GPU Compute Time, the GPU may be under-utilized " |
| 159 | "because of host-side overheads or data transfers." |
| 160 | << std::endl; |
| 161 | os << "Enqueue Time: the host latency to enqueue a query. If this is longer than GPU Compute Time, the GPU may be " |
| 162 | "under-utilized." |
| 163 | << std::endl; |
| 164 | os << "H2D Latency: the latency for host-to-device data transfers for input tensors of a single query." |
| 165 | << std::endl; |
| 166 | os << "D2H Latency: the latency for device-to-host data transfers for output tensors of a single query." |
| 167 | << std::endl; |
| 168 | os << "Latency: the summation of H2D Latency, GPU Compute Time, and D2H Latency. This is the latency to infer a " |
| 169 | "single query." |
| 170 | << std::endl; |
| 171 | } |
| 172 | |
| 173 | PerformanceResult getPerformanceResult(std::vector<InferenceTime> const& timings, |
| 174 | std::function<float(InferenceTime const&)> metricGetter, std::vector<float> const& percentiles) |