| 72 | |
| 73 | |
| 74 | void validateAndInitCustomQuESTEnv(int useDistrib, int useGpuAccel, int useMultithread, const char* caller) { |
| 75 | |
| 76 | // ensure that we are never re-initialising QuEST (even after finalize) because |
| 77 | // this leads to undefined behaviour in distributed mode, as per the MPI |
| 78 | validate_envNeverInit(globalEnvPtr != nullptr, hasEnvBeenFinalized, caller); |
| 79 | |
| 80 | envvars_validateAndLoadEnvVars(caller); |
| 81 | validateconfig_setEpsilonToDefault(); |
| 82 | |
| 83 | // ensure the chosen deployment is compiled and supported by hardware. |
| 84 | // note that these error messages will be printed by every node because |
| 85 | // validation occurs before comm_init() below, so all processes spawned |
| 86 | // by mpirun believe they are each the main rank. This seems unavoidable. |
| 87 | validate_newEnvDeploymentMode(useDistrib, useGpuAccel, useMultithread, caller); |
| 88 | |
| 89 | // overwrite deployments left as modeflag::USE_AUTO |
| 90 | autodep_chooseQuESTEnvDeployment(useDistrib, useGpuAccel, useMultithread); |
| 91 | |
| 92 | // optionally initialise MPI; necessary before completing validation, |
| 93 | // and before any GPU initialisation and validation, since we will |
| 94 | // perform that specifically upon the MPI-process-bound GPU(s). Further, |
| 95 | // we can make sure validation errors are reported only by the root node. |
| 96 | if (useDistrib) |
| 97 | comm_init(); |
| 98 | |
| 99 | validate_newEnvDistributedBetweenPower2Nodes(caller); |
| 100 | |
| 101 | /// @todo |
| 102 | /// consider immediately disabling MPI here if comm_numNodes() == 1 |
| 103 | /// (also overwriting useDistrib = 0) |
| 104 | |
| 105 | // bind MPI nodes to unique GPUs; even when not distributed, |
| 106 | // and before we have validated local GPUs are compatible |
| 107 | if (useGpuAccel) |
| 108 | gpu_bindLocalGPUsToNodes(); |
| 109 | |
| 110 | // consult environment variable to decide whether to allow GPU sharing |
| 111 | // (default = false) which informs whether below validation is triggered |
| 112 | bool permitGpuSharing = envvars_getWhetherGpuSharingIsPermitted(); |
| 113 | |
| 114 | // each MPI process should ordinarily use a unique GPU. This is |
| 115 | // critical when initializing cuQuantum so that we don't re-init |
| 116 | // cuStateVec on any paticular GPU (which can apparently cause a |
| 117 | // so-far-unwitnessed runtime error), but is otherwise essential |
| 118 | // for good performance. GPU sharing is useful for unit testing |
| 119 | // however permitting a single GPU to test CUDA+MPI deployment |
| 120 | if (useGpuAccel && useDistrib && ! permitGpuSharing) |
| 121 | validate_newEnvNodesEachHaveUniqueGpu(caller); |
| 122 | |
| 123 | /// @todo |
| 124 | /// should we warn here if each machine contains |
| 125 | /// more GPUs than deployed MPI-processes (some GPUs idle)? |
| 126 | |
| 127 | // cuQuantum is always used in GPU-accelerated envs when available |
| 128 | bool useCuQuantum = useGpuAccel && gpu_isCuQuantumCompiled(); |
| 129 | if (useCuQuantum) { |
| 130 | validate_gpuIsCuQuantumCompatible(caller); // assesses above bound GPU |
| 131 | gpu_initCuQuantum(); |
no test coverage detected