| 58 | |
| 59 | template<typename T> |
| 60 | void CheckRedzones(const se::cuda::RedzoneAllocator& rz_allocator, |
| 61 | T* autotune_result) { |
| 62 | if (RedzoneCheckDisabled()) { |
| 63 | return; |
| 64 | } |
| 65 | se::port::StatusOr<se::cuda::RedzoneAllocator::RedzoneCheckStatus> rz_status = |
| 66 | rz_allocator.CheckRedzones(); |
| 67 | if (!rz_status.ok()) { |
| 68 | static std::once_flag failure_logged; |
| 69 | std::call_once(failure_logged, [&]() { |
| 70 | LOG(WARNING) << "Failed to check cudnn convolutions for out-of-bounds " |
| 71 | << "reads and writes with an error message: '" |
| 72 | << rz_status.status().error_message() |
| 73 | << "'; skipping this check. This only means that we won't " |
| 74 | << "check cudnn for out-of-bounds reads and writes. This " |
| 75 | << "message will only be printed once."; |
| 76 | }); |
| 77 | return; |
| 78 | } |
| 79 | auto rz_check_status = rz_status.ValueOrDie(); |
| 80 | if (!rz_check_status.ok()) { |
| 81 | auto* fail = autotune_result->mutable_failure(); |
| 82 | fail->set_msg(rz_check_status.RedzoneFailureMsg()); |
| 83 | fail->set_kind(T::REDZONE_MODIFIED); |
| 84 | fail->set_buffer_address( |
| 85 | reinterpret_cast<uint64>(rz_check_status.user_buffer_address)); |
| 86 | LOG(ERROR) |
| 87 | << "Detected cudnn out-of-bounds write in convolution buffer! This is " |
| 88 | "likely a cudnn bug. We will skip this algorithm in the future, but " |
| 89 | "your GPU state may already be corrupted, leading to incorrect " |
| 90 | "results. Within Google, no action is needed on your part. Outside " |
| 91 | "of Google, please ensure you're running the latest version of " |
| 92 | "cudnn. If that doesn't fix the problem, please file a bug with " |
| 93 | "this full error message and we'll contact nvidia."; |
| 94 | LOG(ERROR) << rz_check_status.RedzoneFailureMsg(); |
| 95 | } |
| 96 | } |
| 97 | |
| 98 | // explicit instantiation |
| 99 | template void CheckRedzones<tensorflow::AutotuneResult>( |
no test coverage detected