(
ex: &MigrationExecutor,
state: &mut MigrationState,
group_id: u64,
req: &MigrationRequest,
migration_id: MigrationId,
)
| 94 | } |
| 95 | |
| 96 | pub(super) async fn phase2_wal_catchup( |
| 97 | ex: &MigrationExecutor, |
| 98 | state: &mut MigrationState, |
| 99 | group_id: u64, |
| 100 | req: &MigrationRequest, |
| 101 | migration_id: MigrationId, |
| 102 | ) -> Result<()> { |
| 103 | let leader_commit = { |
| 104 | let mr = ex.multi_raft.lock().unwrap_or_else(|p| p.into_inner()); |
| 105 | mr.group_statuses() |
| 106 | .iter() |
| 107 | .find(|s| s.group_id == group_id) |
| 108 | .map(|s| s.commit_index) |
| 109 | .unwrap_or(0) |
| 110 | }; |
| 111 | state.start_wal_catchup(leader_commit, leader_commit); |
| 112 | |
| 113 | info!( |
| 114 | vshard = req.vshard_id, |
| 115 | leader_commit, "phase 2: monitoring replication lag" |
| 116 | ); |
| 117 | |
| 118 | let initial_stable_id = ex.transport.peer_connection_stable_id(req.target_node); |
| 119 | let initial_target_addr = { |
| 120 | let topo = ex.topology.read().unwrap_or_else(|p| p.into_inner()); |
| 121 | topo.get_node(req.target_node).map(|n| n.addr.clone()) |
| 122 | }; |
| 123 | |
| 124 | let poll_interval = Duration::from_millis(100); |
| 125 | let timeout = Duration::from_secs(60); |
| 126 | let deadline = std::time::Instant::now() + timeout; |
| 127 | |
| 128 | loop { |
| 129 | tokio::time::sleep(poll_interval).await; |
| 130 | |
| 131 | if let Some(initial_id) = initial_stable_id { |
| 132 | match ex.transport.peer_connection_stable_id(req.target_node) { |
| 133 | Some(current_id) if current_id != initial_id => { |
| 134 | let reason = format!( |
| 135 | "peer identity changed mid-migration: stable_id {} -> {} for node {}", |
| 136 | initial_id, current_id, req.target_node |
| 137 | ); |
| 138 | state.fail(reason.clone()); |
| 139 | return Err(ClusterError::Transport { detail: reason }); |
| 140 | } |
| 141 | None => { |
| 142 | let reason = format!( |
| 143 | "connection to target node {} lost during migration", |
| 144 | req.target_node |
| 145 | ); |
| 146 | state.fail(reason.clone()); |
| 147 | return Err(ClusterError::Transport { detail: reason }); |
| 148 | } |
| 149 | _ => {} |
| 150 | } |
| 151 | } |
| 152 | |
| 153 | { |
no test coverage detected