Failure marks a failure and starts backing off if needed. The next call to BackoffIfRequired will do the right thing after this. It will return the time that the current failure will result in backoff waiting until, and a bool signalling whether we have blacklisted and therefore to give up.
()
| 109 | // will result in backoff waiting until, and a bool signalling |
| 110 | // whether we have blacklisted and therefore to give up. |
| 111 | func (s *ServerStatistics) Failure() (time.Time, bool) { |
| 112 | // If we aren't already backing off, this call will start |
| 113 | // a new backoff period. Increase the failure counter and |
| 114 | // start a goroutine which will wait out the backoff and |
| 115 | // unset the backoffStarted flag when done. |
| 116 | if s.backoffStarted.CAS(false, true) { |
| 117 | if s.backoffCount.Inc() >= s.statistics.FailuresUntilBlacklist { |
| 118 | s.blacklisted.Store(true) |
| 119 | if s.statistics.DB != nil { |
| 120 | if err := s.statistics.DB.AddServerToBlacklist(s.serverName); err != nil { |
| 121 | logrus.WithError(err).Errorf("Failed to add %q to blacklist", s.serverName) |
| 122 | } |
| 123 | } |
| 124 | return time.Time{}, true |
| 125 | } |
| 126 | |
| 127 | go func() { |
| 128 | until, ok := s.backoffUntil.Load().(time.Time) |
| 129 | if ok { |
| 130 | select { |
| 131 | case <-time.After(time.Until(until)): |
| 132 | case <-s.interrupt: |
| 133 | } |
| 134 | } |
| 135 | s.backoffStarted.Store(false) |
| 136 | }() |
| 137 | } |
| 138 | |
| 139 | // Check if we have blacklisted this node. |
| 140 | if s.blacklisted.Load() { |
| 141 | return time.Now(), true |
| 142 | } |
| 143 | |
| 144 | // If we're already backing off and we haven't yet surpassed |
| 145 | // the deadline then return that. Repeated calls to Failure |
| 146 | // within a single backoff interval will have no side effects. |
| 147 | if until, ok := s.backoffUntil.Load().(time.Time); ok && !time.Now().After(until) { |
| 148 | return until, false |
| 149 | } |
| 150 | |
| 151 | // We're either backing off and have passed the deadline, or |
| 152 | // we aren't backing off, so work out what the next interval |
| 153 | // will be. |
| 154 | count := s.backoffCount.Load() |
| 155 | until := time.Now().Add(s.duration(count)) |
| 156 | s.backoffUntil.Store(until) |
| 157 | return until, false |
| 158 | } |
| 159 | |
| 160 | // BackoffInfo returns information about the current or previous backoff. |
| 161 | // Returns the last backoffUntil time and whether the server is currently blacklisted or not. |