| 376 | } |
| 377 | |
| 378 | Future<Nothing> _recover(const Option<RecoverResponse>& response) |
| 379 | { |
| 380 | if (response.isNone()) { |
| 381 | return Failure("Failed to recover begin and end positions of the log"); |
| 382 | } |
| 383 | |
| 384 | if (response->status() != Metadata::RECOVERING) { |
| 385 | return Failure("Unexpected status returned from the recover protocol"); |
| 386 | } |
| 387 | |
| 388 | CHECK(response->has_begin() && response->has_end()); |
| 389 | |
| 390 | if (response->begin() == response->end()) { |
| 391 | // This may happen if all replicas know only about position |
| 392 | // 0 (just initialized). |
| 393 | return Failure("Recovered only 1 position, cannot catch-up"); |
| 394 | } |
| 395 | |
| 396 | // We do not catchup the last recovered position in order to |
| 397 | // prevent coordinator demotion. |
| 398 | end = response->end() - 1; |
| 399 | |
| 400 | return replica->beginning() |
| 401 | .then(defer(self(), [this, response](uint64_t begin) { |
| 402 | // Ideally we would only need to catch-up positions from |
| 403 | // the recovered range and then adjust local replica's |
| 404 | // 'begin'. However, the replica needs to persist the |
| 405 | // truncation indicator (TRUNCATE or a tombstone NOP) to |
| 406 | // be able to recover the same 'begin' after a restart. If |
| 407 | // we only catch-up positions from the recovered range, it |
| 408 | // is possible that the replica sees neither TRUNCATE |
| 409 | // (e.g. it is the last position in the log, which we |
| 410 | // don't catch-up), nor a tombstone. We catch-up positions |
| 411 | // starting with the lowest known begin position so that |
| 412 | // the replica either retains the same 'begin', or sees a |
| 413 | // truncation indicator. |
| 414 | begin = std::min(begin, response->begin()); |
| 415 | return catchup(begin, end); |
| 416 | })); |
| 417 | } |
| 418 | |
| 419 | Future<Nothing> catchup(uint64_t begin, uint64_t end) |
| 420 | { |