| 1049 | } |
| 1050 | |
| 1051 | void TableImpl::TaskTimeout(SdkTask* task) { |
| 1052 | perf_counter_.GetTimeoutCnt(task).Inc(); |
| 1053 | CHECK_NOTNULL(task); |
| 1054 | |
| 1055 | task->ExcludeOtherRef(); |
| 1056 | |
| 1057 | StatusCode err = task->GetInternalError(); |
| 1058 | if (err == kKeyNotInRange || err == kConnectError) { |
| 1059 | ScheduleUpdateMeta(task->InternalRowKey(), task->GetMetaTimeStamp()); |
| 1060 | } else if (err == kRPCTimeout && !task->GetServerAddr().empty()) { |
| 1061 | MutexLock lock(&rpc_timeout_duration_mutex_); |
| 1062 | if (rpc_timeout_duration_.find(task->GetServerAddr()) == rpc_timeout_duration_.end()) { |
| 1063 | rpc_timeout_duration_[task->GetServerAddr()] = get_millis(); |
| 1064 | } else if (get_millis() - rpc_timeout_duration_[task->GetServerAddr()] >= |
| 1065 | FLAGS_tera_sdk_update_meta_rpc_timeout_max_ms) { |
| 1066 | LOG(WARNING) << "requests on server<" << task->GetServerAddr() << "> continuous " |
| 1067 | << "over " << FLAGS_tera_sdk_update_meta_rpc_timeout_max_ms / 1000 |
| 1068 | << "s with the response of kRpcTimeout, last succ request time(" |
| 1069 | << rpc_timeout_duration_[task->GetServerAddr()] / 1000 |
| 1070 | << "), may be the tabletserver is zombie, try to update meta."; |
| 1071 | rpc_timeout_duration_[task->GetServerAddr()] = get_millis(); |
| 1072 | rpc_timeout_duration_mutex_.Unlock(); |
| 1073 | ScheduleUpdateMeta(task->InternalRowKey(), task->GetMetaTimeStamp()); |
| 1074 | rpc_timeout_duration_mutex_.Lock(); |
| 1075 | } |
| 1076 | } |
| 1077 | |
| 1078 | std::string err_reason; |
| 1079 | if (task->RetryTimes() == 0) { |
| 1080 | perf_counter_.GetQueueTimeoutCnt(task).Inc(); |
| 1081 | err_reason = StringFormat("commit lld times, retry 0 times, in %u ms.", task->GetCommitTimes(), |
| 1082 | task->Type() == SdkTask::READ ? read_timeout_ : write_timeout_); |
| 1083 | } else { |
| 1084 | err_reason = StringFormat("commit %lld times, retry %u times, in %u ms. last error: %s", |
| 1085 | task->GetCommitTimes(), task->RetryTimes(), |
| 1086 | task->Type() == SdkTask::READ ? read_timeout_ : write_timeout_, |
| 1087 | StatusCodeToString(err).c_str()); |
| 1088 | } |
| 1089 | switch (task->Type()) { |
| 1090 | case SdkTask::READ: { |
| 1091 | RowReaderImpl* row_reader = (RowReaderImpl*)task; |
| 1092 | row_reader->SetError(ErrorCode::kTimeout, err_reason); |
| 1093 | cur_reader_pending_counter_.Dec(); |
| 1094 | } break; |
| 1095 | case SdkTask::BATCH_MUTATION: { |
| 1096 | BatchMutationImpl* batch_mutation = (BatchMutationImpl*)task; |
| 1097 | batch_mutation->SetError(ErrorCode::kTimeout, err_reason); |
| 1098 | cur_commit_pending_counter_.Dec(); |
| 1099 | } break; |
| 1100 | case SdkTask::MUTATION: { |
| 1101 | RowMutationImpl* row_mutation = (RowMutationImpl*)task; |
| 1102 | row_mutation->SetError(ErrorCode::kTimeout, err_reason); |
| 1103 | cur_commit_pending_counter_.Dec(); |
| 1104 | } break; |
| 1105 | default: |
| 1106 | abort(); |
| 1107 | } |
| 1108 | int64_t perf_time = get_micros(); |
nothing calls this directly
no test coverage detected