| 81 | } |
| 82 | |
| 83 | void FlopCounter::printPerformanceUpdate(double wallTime) { |
| 84 | const int rank = seissol::MPI::mpi.rank(); |
| 85 | const auto worldSize = static_cast<size_t>(seissol::MPI::mpi.size()); |
| 86 | |
| 87 | const long long newTotalFlops = hardwareFlopsLocal + hardwareFlopsNeighbor + hardwareFlopsOther + |
| 88 | hardwareFlopsDynamicRupture + hardwareFlopsPlasticity; |
| 89 | const long long diffFlops = newTotalFlops - previousTotalFlops; |
| 90 | previousTotalFlops = newTotalFlops; |
| 91 | |
| 92 | const double diffTime = wallTime - previousWallTime; |
| 93 | previousWallTime = wallTime; |
| 94 | |
| 95 | const double accumulatedGflopsPerSecond = newTotalFlops * 1.e-9 / wallTime; |
| 96 | const double previousGflopsPerSecond = diffFlops * 1.e-9 / diffTime; |
| 97 | |
| 98 | auto accumulatedGflopsPerSecondOnRanks = seissol::MPI::mpi.collect(accumulatedGflopsPerSecond); |
| 99 | auto previousGflopsPerSecondOnRanks = seissol::MPI::mpi.collect(previousGflopsPerSecond); |
| 100 | |
| 101 | if (rank == 0) { |
| 102 | double accumulatedGflopsSum = 0; |
| 103 | double previousGflopsSum = 0; |
| 104 | #ifdef _OPENMP |
| 105 | #pragma omp simd reduction(+ : accumulatedGflopsSum, previousGflopsSum) |
| 106 | #endif |
| 107 | for (size_t i = 0; i < worldSize; i++) { |
| 108 | accumulatedGflopsSum += accumulatedGflopsPerSecondOnRanks[i]; |
| 109 | previousGflopsSum += previousGflopsPerSecondOnRanks[i]; |
| 110 | } |
| 111 | const auto accumulatedGflopsPerRank = accumulatedGflopsSum / seissol::MPI::mpi.size(); |
| 112 | const auto previousGflopsPerRank = previousGflopsSum / seissol::MPI::mpi.size(); |
| 113 | |
| 114 | // for now, we calculate everything in GFLOP/s, and switch back to FLOP/s for output only |
| 115 | logInfo(rank) << "Performance since the start:" |
| 116 | << UnitFlopPerS.formatPrefix(accumulatedGflopsSum * 1e9).c_str() << "(rank 0:" |
| 117 | << UnitFlopPerS.formatPrefix(accumulatedGflopsPerSecond * 1e9).c_str() |
| 118 | << ", average over ranks:" |
| 119 | << UnitFlopPerS.formatPrefix(accumulatedGflopsPerRank * 1e9).c_str() << ")"; |
| 120 | logInfo(rank) << "Performance since last sync point:" |
| 121 | << UnitFlopPerS.formatPrefix(previousGflopsSum * 1e9).c_str() |
| 122 | << "(rank 0:" << UnitFlopPerS.formatPrefix(previousGflopsPerSecond * 1e9).c_str() |
| 123 | << ", average over ranks:" |
| 124 | << UnitFlopPerS.formatPrefix(previousGflopsPerRank * 1e9).c_str() << ")"; |
| 125 | |
| 126 | out << wallTime << ","; |
| 127 | for (size_t i = 0; i < worldSize - 1; i++) { |
| 128 | out << accumulatedGflopsPerSecondOnRanks[i] << ","; |
| 129 | out << previousGflopsPerSecondOnRanks[i] << ","; |
| 130 | } |
| 131 | out << accumulatedGflopsPerSecondOnRanks[worldSize - 1] << "," |
| 132 | << previousGflopsPerSecondOnRanks[worldSize - 1] << std::endl; |
| 133 | } |
| 134 | } |
| 135 | |
| 136 | /** |
| 137 | * Prints the measured FLOP/s. |