| 930 | } |
| 931 | |
| 932 | static void ValidateChecksum(uint64_t expected, uint64_t actual, |
| 933 | const Tensor& in, uint32_t request_index, |
| 934 | const std::string& key, const std::string& msg) { |
| 935 | RDMA_LOG(2) << "Request #" << request_index << ": " << key |
| 936 | << ": Checksum: " << std::hex << " Expected = 0x" << expected |
| 937 | << ". Actual = 0x" << actual << "."; |
| 938 | |
| 939 | if (expected != actual) { |
| 940 | // Checksum failed. There is one case where this is allowed - if the |
| 941 | // tensor is an AssignAdd of the global step. Since the data-validation |
| 942 | // always postpones the Tensor response in order to send a checksum message, |
| 943 | // it is possible that the global-step was updated while the response was |
| 944 | // still in queue. |
| 945 | if ((in.TotalBytes() == 8) && (in.dtype() == DT_INT64)) { |
| 946 | int64_t prev_val = *(int64_t*)DMAHelper::base(&in) - 1; |
| 947 | actual = Hash64((const char*)&prev_val, 8, 0); |
| 948 | } |
| 949 | if (expected != actual) { |
| 950 | LOG(FATAL) << "[" << msg << "]: Checksum validation failed for request #" |
| 951 | << request_index << ": " << key << std::hex << " " |
| 952 | << DataTypeString(in.dtype()) << " " |
| 953 | << in.shape().DebugString() << " (0x" << in.TotalBytes() |
| 954 | << " bytes): " |
| 955 | << " Expected 0x" << expected << ". Got 0x" << actual << "."; |
| 956 | } |
| 957 | } |
| 958 | } |
| 959 | #endif // RDMA_DATA_VALIDATION |
| 960 | |
| 961 | #if GOOGLE_CUDA |
no test coverage detected