(client *topom.ApiClient, maxdown int, auth string)
| 246 | } |
| 247 | |
| 248 | func (hc *HealthyChecker) Maintains(client *topom.ApiClient, maxdown int, auth string) { |
| 249 | // remove proxy at state error from codis |
| 250 | for _, p := range hc.Proxy.Models { |
| 251 | switch hc.pstatus[p.Token] { |
| 252 | case CodeError, CodeTimeout, CodeMissing: |
| 253 | log.Warnf("try to remove proxy-[%s]", p.AdminAddr) |
| 254 | if err := client.RemoveProxy(p.Token, true); err != nil { |
| 255 | log.ErrorErrorf(err, "call rpc remove-proxy to dashboard %s failed", p.AdminAddr) |
| 256 | return |
| 257 | } |
| 258 | log.Warnf("try to remove proxy done.") |
| 259 | return |
| 260 | default: |
| 261 | continue |
| 262 | } |
| 263 | } |
| 264 | |
| 265 | // remove server at state error from codis |
| 266 | Groups: |
| 267 | for _, g := range hc.Group.Models { |
| 268 | for i, x := range g.Servers { |
| 269 | // if master state is not right, promote slave to master first(if have slave) |
| 270 | if i == 0 { |
| 271 | if len(g.Servers) > 1 { |
| 272 | switch hc.sstatus[g.Servers[0].Addr] { |
| 273 | case CodeError, CodeMissing, CodeTimeout, CodeSyncError: |
| 274 | log.Warnf("codis-server (master) %s state error", x.Addr) |
| 275 | break Groups |
| 276 | default: |
| 277 | continue |
| 278 | } |
| 279 | } else { |
| 280 | continue |
| 281 | } |
| 282 | } |
| 283 | // remove codis server(slave and only one master) which state is not right |
| 284 | switch hc.sstatus[x.Addr] { |
| 285 | case CodeError, CodeMissing, CodeTimeout, CodeSyncError: |
| 286 | log.Warnf("try to group-del-server to dashboard %s", x.Addr) |
| 287 | if err := client.GroupDelServer(g.Id, x.Addr); err != nil { |
| 288 | log.ErrorErrorf(err, "call rpc group-del-server to dashboard %s failed", x.Addr) |
| 289 | return |
| 290 | } |
| 291 | log.Debugf("call rpc group-del-server OK") |
| 292 | |
| 293 | // trt to shutdown codis-server as slave in error state |
| 294 | log.Warnf("try to shutdown codis-server(slave) %s", x.Addr) |
| 295 | c, err := redis.NewClient(x.Addr, auth, time.Minute*30) |
| 296 | if err != nil { |
| 297 | log.WarnErrorf(err, "connect to codis-server(slave) %s failed", x.Addr) |
| 298 | return |
| 299 | } |
| 300 | defer c.Close() |
| 301 | if err := c.Shutdown(); err != nil { |
| 302 | log.WarnErrorf(err, "try to shutdown codis-server %s failed", x.Addr) |
| 303 | return |
| 304 | } |
| 305 | return |
no test coverage detected