| 279 | } |
| 280 | |
| 281 | func recordAndReportRuleQueryMetrics(qf rules.QueryFunc, userID string, evalMetrics *RuleEvalMetrics, logger log.Logger) rules.QueryFunc { |
| 282 | queryTime := evalMetrics.RulerQuerySeconds.WithLabelValues(userID) |
| 283 | querySeries := evalMetrics.RulerQuerySeries.WithLabelValues(userID) |
| 284 | querySample := evalMetrics.RulerQuerySamples.WithLabelValues(userID) |
| 285 | queryChunkBytes := evalMetrics.RulerQueryChunkBytes.WithLabelValues(userID) |
| 286 | queryDataBytes := evalMetrics.RulerQueryDataBytes.WithLabelValues(userID) |
| 287 | |
| 288 | return func(ctx context.Context, qs string, t time.Time) (promql.Vector, error) { |
| 289 | queryStats, ctx := stats.ContextWithEmptyStats(ctx) |
| 290 | // If we've been passed a counter we want to record the wall time spent executing this request. |
| 291 | timer := prometheus.NewTimer(nil) |
| 292 | |
| 293 | // Add request ID before logging so that it can be used in query stats logs. |
| 294 | ctx = requestmeta.ContextWithRequestId(ctx, uuid.NewString()) |
| 295 | |
| 296 | defer func() { |
| 297 | querySeconds := timer.ObserveDuration().Seconds() |
| 298 | queryTime.Add(querySeconds) |
| 299 | querySeries.Add(float64(queryStats.FetchedSeriesCount)) |
| 300 | querySample.Add(float64(queryStats.FetchedSamplesCount)) |
| 301 | queryChunkBytes.Add(float64(queryStats.FetchedChunkBytes)) |
| 302 | queryDataBytes.Add(float64(queryStats.FetchedDataBytes)) |
| 303 | // Log ruler query stats. |
| 304 | logMessage := []any{ |
| 305 | "msg", "query stats", |
| 306 | "component", "ruler", |
| 307 | } |
| 308 | if origin := ctx.Value(promql.QueryOrigin{}); origin != nil { |
| 309 | queryLabels := origin.(map[string]any) |
| 310 | rgMap := queryLabels["ruleGroup"].(map[string]string) |
| 311 | logMessage = append(logMessage, |
| 312 | "rule_group", rgMap["name"], |
| 313 | "namespace", rgMap["file"], |
| 314 | ) |
| 315 | } |
| 316 | ruleDetail := rules.FromOriginContext(ctx) |
| 317 | logMessage = append(logMessage, |
| 318 | "rule", ruleDetail.Name, |
| 319 | "rule_kind", ruleDetail.Kind, |
| 320 | "query", qs, |
| 321 | "cortex_ruler_query_seconds_total", querySeconds, |
| 322 | "query_wall_time_seconds", queryStats.WallTime, |
| 323 | "query_storage_wall_time_seconds", queryStats.QueryStorageWallTime, |
| 324 | "fetched_series_count", queryStats.FetchedSeriesCount, |
| 325 | "fetched_chunks_count", queryStats.FetchedChunksCount, |
| 326 | "fetched_samples_count", queryStats.FetchedSamplesCount, |
| 327 | "fetched_chunks_bytes", queryStats.FetchedChunkBytes, |
| 328 | "fetched_data_bytes", queryStats.FetchedDataBytes, |
| 329 | ) |
| 330 | logMessage = append(logMessage, queryStats.LoadExtraFields()...) |
| 331 | level.Info(util_log.WithContext(ctx, logger)).Log(logMessage...) |
| 332 | }() |
| 333 | |
| 334 | result, err := qf(ctx, qs, t) |
| 335 | return result, err |
| 336 | } |
| 337 | } |
| 338 | |