| 247 | } |
| 248 | |
| 249 | func metricsQueryFunc(qf rules.QueryFunc, queries, failedQueries prometheus.Counter) rules.QueryFunc { |
| 250 | return func(ctx context.Context, qs string, t time.Time) (promql.Vector, error) { |
| 251 | queries.Inc() |
| 252 | result, err := qf(ctx, qs, t) |
| 253 | |
| 254 | // We only care about errors returned by underlying Queryable. Errors returned by PromQL engine are "user-errors", |
| 255 | // and not interesting here. |
| 256 | qerr := QueryableError{} |
| 257 | if err != nil && errors.As(err, &qerr) { |
| 258 | origErr := qerr.Unwrap() |
| 259 | |
| 260 | // Not all errors returned by Queryable are interesting, only those that would result in 500 status code. |
| 261 | // |
| 262 | // We rely on TranslateToPromqlApiError to do its job here... it returns nil, if err is nil. |
| 263 | // It returns promql.ErrStorage, if error should be reported back as 500. |
| 264 | // Other errors it returns are either for canceled or timed-out queriers (we're not reporting those as failures), |
| 265 | // or various user-errors (limits, duplicate samples, etc. ... also not failures). |
| 266 | // |
| 267 | // All errors will still be counted towards "evaluation failures" metrics and logged by Prometheus Ruler, |
| 268 | // but we only want internal errors here. |
| 269 | if _, ok := querier.TranslateToPromqlAPIError(origErr).(promql.ErrStorage); ok { |
| 270 | failedQueries.Inc() |
| 271 | } |
| 272 | |
| 273 | // Return unwrapped error. |
| 274 | return result, origErr |
| 275 | } |
| 276 | |
| 277 | return result, err |
| 278 | } |
| 279 | } |
| 280 | |
| 281 | func recordAndReportRuleQueryMetrics(qf rules.QueryFunc, userID string, evalMetrics *RuleEvalMetrics, logger log.Logger) rules.QueryFunc { |
| 282 | queryTime := evalMetrics.RulerQuerySeconds.WithLabelValues(userID) |