| 192 | } |
| 193 | |
| 194 | func executeWithRetry( |
| 195 | ctx context.Context, |
| 196 | s *scope, |
| 197 | maxRetry int, |
| 198 | rp func(http.ResponseWriter, *http.Request), |
| 199 | rw ResponseWriterWithCode, |
| 200 | srw StatResponseWriter, |
| 201 | req *http.Request, |
| 202 | monitorDuration func(float64), |
| 203 | monitorRetryRequestInc func(prometheus.Labels), |
| 204 | ) (float64, error) { |
| 205 | startTime := time.Now() |
| 206 | var since float64 |
| 207 | |
| 208 | var body []byte |
| 209 | var err error |
| 210 | // Use readAndRestoreRequestBody to read the entire request body into a byte slice, |
| 211 | // and to restore req.Body so that it can be reused later in the code. |
| 212 | if maxRetry > 0 { |
| 213 | body, err = readAndRestoreRequestBody(req) |
| 214 | if err != nil { |
| 215 | since := time.Since(startTime).Seconds() |
| 216 | return since, err |
| 217 | } |
| 218 | } |
| 219 | |
| 220 | numRetry := 0 |
| 221 | for { |
| 222 | rp(rw, req) |
| 223 | |
| 224 | err := ctx.Err() |
| 225 | if err != nil { |
| 226 | since = time.Since(startTime).Seconds() |
| 227 | |
| 228 | return since, err |
| 229 | } |
| 230 | // The request has been successfully proxied. |
| 231 | |
| 232 | srw.SetStatusCode(rw.StatusCode()) |
| 233 | // StatusBadGateway response is returned by http.ReverseProxy when |
| 234 | // it cannot establish connection to remote host. |
| 235 | if rw.StatusCode() == http.StatusBadGateway { |
| 236 | log.Debugf("the invalid host is: %s", s.host) |
| 237 | s.host.Penalize() |
| 238 | // comment s.host.dec() line to avoid double increment; issue #322 |
| 239 | // s.host.dec() |
| 240 | s.host.SetIsActive(false) |
| 241 | nextHost := s.cluster.getHost() |
| 242 | // The query could be retried if it has no stickiness to a certain server |
| 243 | if numRetry < maxRetry && nextHost.IsActive() && s.sessionId == "" { |
| 244 | // the query execution has been failed |
| 245 | monitorRetryRequestInc(s.labels) |
| 246 | currentHost := s.host |
| 247 | |
| 248 | // decrement the current failed host counter and increment the new host |
| 249 | // as for the end of the requests we will close the scope and in that closed scope |
| 250 | // decrement the new host PR - https://github.com/ContentSquare/chproxy/pull/357 |
| 251 | if currentHost != nextHost { |