Method Replay

GPU-Virtual-Service/gpu-remoting/src/client/clientCkpt.cc:46–125

}

void ClientEndpoint::Replay() {
    auto start_replay = std::chrono::high_resolution_clock::now();
    tool::Logging(LOG_INFO, myName_, "start replaying %zu requests from registerIOVList\n", regIOVList.size());
    SendRegisterRequest(this, true);

    // std::ofstream outFile2("api_records_replay.log", std::ios::out);     
    tool::Logging(LOG_INFO, myName_, "start replaying %zu requests from requestIOVList\n", reqIOVList.size());
    for (auto it = reqIOVList.begin(); it != reqIOVList.end(); ++it) {
        RequestIOV* tmpReqIOV = &(*it);
        // outFile2 << tmpReqIOV->GetRequestType() << std::endl;

        if (tmpReqIOV->GetRequestType() == NCCL_GET_UNIQUE_ID) {
            ncclUniqueId uniqueId;
            RequestIOV resBuf = RequestIOV();
            resBuf.Push(uniqueId);
            SendRequestRecvResponse(tmpReqIOV, &resBuf, true, false);
#ifdef GV_GPUMAP
            gpuIdMap->UpdateUniqueID((uint8_t*)&uniqueId, sizeof(ncclUniqueId));
            tool::Logging(LOG_INFO, myName_, "[pid:%d, tid:%d] updated uniqueId\n", _processID, _threadID);
#else
            tool::Logging(LOG_ERROR, myName_, "Replaying for ncclGetUniqueId is not supported in non-GV_GPUMAP mode\n");
            exit(EXIT_FAILURE);
#endif // GV_GPUMAP
            continue;
        } 
        else if (tmpReqIOV->GetRequestType() == NCCL_COMM_INIT_RANK
              || tmpReqIOV->GetRequestType() == NCCL_COMM_INIT_RANK_CONFIG) {
            ncclUniqueId uniqueId;
#ifdef GV_GPUMAP
            gpuIdMap->RequestUniqueID((uint8_t*)&uniqueId, sizeof(ncclUniqueId));
            tmpReqIOV->SetElement(2, &uniqueId, sizeof(ncclUniqueId));
            SendRequest(tmpReqIOV, true, false);
            tool::Logging(LOG_INFO, myName_, "[pid:%d, tid:%d] requested uniqueId\n", _processID, _threadID);
#else
            tool::Logging(LOG_ERROR, myName_, "Replaying for ncclCommInitRank is not supported in non-GV_GPUMAP mode\n");
            exit(EXIT_FAILURE);
#endif // GV_GPUMAP
        }
        else if (tmpReqIOV->GetRequestType() == CUDA_MEMCPY_ASYNC_H2D) {
            SendRequestH2D(tmpReqIOV, (uint8_t*)tmpReqIOV->GetHeaders(), sizeof(cudaMemcpyKind)+sizeof(size_t)+sizeof(uint64_t)+sizeof(uint8_t)+sizeof(uint64_t), false, false);
        }
        else if (tmpReqIOV->GetRequestType() == CUDA_STREAM_SYNCHRONIZE || tmpReqIOV->GetRequestType() == NCCL_COMM_GET_ASYNC_ERROR) {
            int result = 0;
            RequestIOV resBuf = RequestIOV();
            resBuf.Push(&result);
            SendRequestRecvResponse(tmpReqIOV, &resBuf, true, false);
        }
        else {
            SendRequest(tmpReqIOV, true, false);
        }
    }
    // outFile2.close();

    // eval for replay time
    size_t free = 0;
    size_t total = 0;
    int replayFinished = 1;
    RequestIOV reqBuf = RequestIOV();