| 2115 | } |
| 2116 | |
| 2117 | void MasterImpl::DeleteTabletNode(const std::string &tabletnode_addr, const std::string &uuid) { |
| 2118 | abnormal_node_mgr_->RecordNodeDelete(tabletnode_addr, get_micros() / 1000000); |
| 2119 | TabletNodePtr node = tabletnode_manager_->DelTabletNode(tabletnode_addr); |
| 2120 | if (!node || node->uuid_ != uuid) { |
| 2121 | LOG(INFO) << "invalid node and uuid: addr: " << tabletnode_addr << ", uuid: " << uuid; |
| 2122 | return; |
| 2123 | } |
| 2124 | |
| 2125 | // possible status: running, readonly, wait. |
| 2126 | if (GetMasterStatus() == kOnWait) { |
| 2127 | return; |
| 2128 | } |
| 2129 | |
| 2130 | std::vector<TabletPtr> tablet_list; |
| 2131 | tablet_manager_->FindTablet(tabletnode_addr, &tablet_list, false); |
| 2132 | |
| 2133 | if (meta_tablet_->GetTabletNode() && meta_tablet_->GetTabletNode()->uuid_ == uuid) { |
| 2134 | LOG(INFO) << " try move meta tablet immediately: "; |
| 2135 | TryMoveTablet(meta_tablet_); |
| 2136 | auto pend = std::remove(tablet_list.begin(), tablet_list.end(), |
| 2137 | std::dynamic_pointer_cast<Tablet>(meta_tablet_)); |
| 2138 | tablet_list.erase(pend, tablet_list.end()); |
| 2139 | } |
| 2140 | |
| 2141 | bool in_safemode = TryEnterSafeMode(); |
| 2142 | TabletEvent event = in_safemode ? TabletEvent::kTsOffline : TabletEvent::kTsDelayOffline; |
| 2143 | for (auto it = tablet_list.begin(); it != tablet_list.end(); ++it) { |
| 2144 | TabletPtr tablet = *it; |
| 2145 | if ((tablet->GetTabletNode() && tablet->GetTabletNode()->uuid_ == uuid) && |
| 2146 | tablet->LockTransition()) { |
| 2147 | tablet->DoStateTransition(event); |
| 2148 | tablet->UnlockTransition(); |
| 2149 | } |
| 2150 | } |
| 2151 | if (in_safemode) { |
| 2152 | LOG(WARNING) << "master is in safemode, will not recover user tablet at ts: " |
| 2153 | << tabletnode_addr; |
| 2154 | return; |
| 2155 | } |
| 2156 | int64_t wait_time = FLAGS_tera_master_tabletnode_timeout; |
| 2157 | wait_time = wait_time ? wait_time : 3 * FLAGS_tera_master_query_tabletnode_period; |
| 2158 | ThreadPool::Task task = |
| 2159 | std::bind(&MasterImpl::MoveTabletOnDeadTabletNode, this, tablet_list, node); |
| 2160 | int64_t reconnect_timeout_task_id = thread_pool_->DelayTask(wait_time, task); |
| 2161 | tabletnode_manager_->WaitTabletNodeReconnect(tabletnode_addr, uuid, reconnect_timeout_task_id); |
| 2162 | } |
| 2163 | |
| 2164 | void MasterImpl::MoveTabletOnDeadTabletNode(const std::vector<TabletPtr> &tablet_list, |
| 2165 | TabletNodePtr dead_node) { |