| 86 | // ---------------------------------------------------------------------------------------- |
| 87 | |
| 88 | void synchronize_stream(cudaStream_t stream) |
| 89 | { |
| 90 | #if !defined CUDA_VERSION |
| 91 | #error CUDA_VERSION not defined |
| 92 | #elif CUDA_VERSION >= 9020 && CUDA_VERSION < 11000 |
| 93 | // We will stop using this alternative version with cuda V11, hopefully the bug in |
| 94 | // cudaStreamSynchronize is fixed by then. |
| 95 | // |
| 96 | // This should be pretty much the same as cudaStreamSynchronize, which for some |
| 97 | // reason makes training freeze in some cases. |
| 98 | // (see https://github.com/davisking/dlib/issues/1513) |
| 99 | while (true) |
| 100 | { |
| 101 | cudaError_t err = cudaStreamQuery(stream); |
| 102 | switch (err) |
| 103 | { |
| 104 | case cudaSuccess: return; // now we are synchronized |
| 105 | case cudaErrorNotReady: break; // continue waiting |
| 106 | default: CHECK_CUDA(err); // unexpected error: throw |
| 107 | } |
| 108 | } |
| 109 | #else // CUDA_VERSION |
| 110 | CHECK_CUDA(cudaStreamSynchronize(stream)); |
| 111 | #endif // CUDA_VERSION |
| 112 | } |
| 113 | |
| 114 | void gpu_data:: |
| 115 | wait_for_transfer_to_finish() const |
no outgoing calls
no test coverage detected