| 176 | } |
| 177 | |
| 178 | void FeedSlaveThread::loop() { |
| 179 | // is_first_repl_batch was used to fix that replication may be stuck in a dead loop |
| 180 | // when some seqs might be lost in the middle of the WAL log, so forced to replicate |
| 181 | // first batch here to work around this issue instead of waiting for enough batch size. |
| 182 | bool is_first_repl_batch = true; |
| 183 | uint32_t yield_microseconds = 2 * 1000; |
| 184 | std::string batches_bulk; |
| 185 | size_t updates_in_batches = 0; |
| 186 | while (!IsStopped()) { |
| 187 | auto curr_seq = next_repl_seq_.load(); |
| 188 | |
| 189 | // Check replication lag - disconnect slow consumers before WAL is exhausted |
| 190 | // Skip check if max_replication_lag_ is 0 (feature disabled) |
| 191 | if (max_replication_lag_ > 0) { |
| 192 | auto latest_seq = srv_->storage->LatestSeqNumber(); |
| 193 | if (latest_seq > curr_seq) { |
| 194 | auto lag = static_cast<int64_t>(latest_seq - curr_seq); |
| 195 | if (lag > max_replication_lag_) { |
| 196 | ERROR("Replication lag {} exceeds max allowed {} for slave {}:{}, disconnecting to prevent WAL exhaustion", |
| 197 | lag, max_replication_lag_, conn_->GetAnnounceIP(), conn_->GetListeningPort()); |
| 198 | Stop(); |
| 199 | return; |
| 200 | } |
| 201 | } |
| 202 | } |
| 203 | |
| 204 | if (!iter_ || !iter_->Valid()) { |
| 205 | if (iter_) INFO("WAL was rotated, would reopen again"); |
| 206 | if (!srv_->storage->WALHasNewData(curr_seq) || !srv_->storage->GetWALIter(curr_seq, &iter_).IsOK()) { |
| 207 | iter_ = nullptr; |
| 208 | usleep(yield_microseconds); |
| 209 | checkLivenessIfNeed(); |
| 210 | continue; |
| 211 | } |
| 212 | } |
| 213 | // iter_ would be always valid here |
| 214 | auto batch = iter_->GetBatch(); |
| 215 | if (batch.sequence != curr_seq) { |
| 216 | ERROR( |
| 217 | "Fatal error encountered, WAL iterator is discrete, some seq might be lost, sequence {} expected, but got {}", |
| 218 | curr_seq, batch.sequence); |
| 219 | Stop(); |
| 220 | return; |
| 221 | } |
| 222 | updates_in_batches += batch.writeBatchPtr->Count(); |
| 223 | batches_bulk += redis::BulkString(batch.writeBatchPtr->Data()); |
| 224 | // 1. We must send the first replication batch, as said above. |
| 225 | // 2. To avoid frequently calling 'write' system call to send replication stream, |
| 226 | // we pack multiple batches into one big bulk if possible, and only send once. |
| 227 | // But we should send the bulk of batches if its size exceed kMaxDelayBytes, |
| 228 | // 16Kb by default. Moreover, we also send if updates count in all bathes is |
| 229 | // more that kMaxDelayUpdates, to void too many delayed updates. |
| 230 | // 3. To avoid master don't send replication stream to slave since of packing |
| 231 | // batches strategy, we still send batches if current batch sequence is less |
| 232 | // kMaxDelayUpdates than latest sequence. |
| 233 | if (is_first_repl_batch || batches_bulk.size() >= max_delay_bytes_ || updates_in_batches >= max_delay_updates_ || |
| 234 | srv_->storage->LatestSeqNumber() - batch.sequence <= max_delay_updates_) { |
| 235 | // get the last sequence number of the batch, because WAIT uses |
no test coverage detected