| 45 | } |
| 46 | |
| 47 | phi::Allocation* CUDAAllocator::AllocateImpl(size_t size) { |
| 48 | std::call_once(once_flag_, [this] { platform::SetDeviceId(place_.device); }); |
| 49 | |
| 50 | void* ptr; |
| 51 | auto result = platform::RecordedGpuMalloc(&ptr, size, place_.device); |
| 52 | if (LIKELY(result == gpuSuccess)) { |
| 53 | return new Allocation(ptr, size, Place(place_)); |
| 54 | } |
| 55 | |
| 56 | size_t avail, total, actual_avail, actual_total; |
| 57 | bool is_limited = platform::RecordedGpuMemGetInfo( |
| 58 | &avail, &total, &actual_avail, &actual_total, place_.device); |
| 59 | size_t allocated = total - avail; |
| 60 | |
| 61 | std::string err_msg; |
| 62 | if (is_limited) { |
| 63 | auto limit_size = (total >> 20); |
| 64 | err_msg = string::Sprintf( |
| 65 | "Or set environment variable `FLAGS_gpu_memory_limit_mb` to a larger " |
| 66 | "value. Currently `FLAGS_gpu_memory_limit_mb` is %d, so the maximum " |
| 67 | "GPU memory usage is limited to %d MB.\n" |
| 68 | " The command is `export FLAGS_gpu_memory_limit_mb=xxx`.", |
| 69 | limit_size, |
| 70 | limit_size); |
| 71 | } |
| 72 | |
| 73 | size_t actual_allocated_memory = |
| 74 | paddle::memory::DeviceMemoryStatCurrentValue("Allocated", place_.device); |
| 75 | |
| 76 | PADDLE_THROW_BAD_ALLOC(common::errors::ResourceExhausted( |
| 77 | "\n\nOut of memory error on GPU %d. " |
| 78 | "Cannot allocate %s memory on GPU %d, %s memory has been " |
| 79 | "allocated(actual using allocated memory %s) and " |
| 80 | "available memory is only %s.\n\n" |
| 81 | "Please check whether there is any other process using GPU %d.\n" |
| 82 | "1. If yes, please stop them, or start PaddlePaddle on another GPU.\n" |
| 83 | "2. If no, please decrease the batch size of your model. %s\n", |
| 84 | place_.device, |
| 85 | string::HumanReadableSize(size), |
| 86 | place_.device, |
| 87 | string::HumanReadableSize(allocated), |
| 88 | string::HumanReadableSize(actual_allocated_memory), |
| 89 | string::HumanReadableSize(avail), |
| 90 | place_.device, |
| 91 | err_msg)); |
| 92 | } |
| 93 | |
| 94 | } // namespace paddle::memory::allocation |
nothing calls this directly
no test coverage detected