Compacts all compactable blocks. Force flag will force compaction even if head is not compactable yet.
(ctx context.Context, force bool, allowed *users.AllowedTenants)
| 3287 | |
| 3288 | // Compacts all compactable blocks. Force flag will force compaction even if head is not compactable yet. |
| 3289 | func (i *Ingester) compactBlocks(ctx context.Context, force bool, allowed *users.AllowedTenants) { |
| 3290 | // Don't compact TSDB blocks while JOINING as there may be ongoing blocks transfers. |
| 3291 | // Compaction loop is not running in LEAVING state, so if we get here in LEAVING state, we're flushing blocks. |
| 3292 | if i.lifecycler != nil { |
| 3293 | if ingesterState := i.lifecycler.GetState(); ingesterState == ring.JOINING { |
| 3294 | level.Info(logutil.WithContext(ctx, i.logger)).Log("msg", "TSDB blocks compaction has been skipped because of the current ingester state", "state", ingesterState) |
| 3295 | return |
| 3296 | } |
| 3297 | } |
| 3298 | |
| 3299 | _ = concurrency.ForEachUser(ctx, i.getTSDBUsers(), i.cfg.BlocksStorageConfig.TSDB.HeadCompactionConcurrency, func(ctx context.Context, userID string) error { |
| 3300 | if !allowed.IsAllowed(userID) { |
| 3301 | return nil |
| 3302 | } |
| 3303 | |
| 3304 | userDB, err := i.getTSDB(userID) |
| 3305 | if err != nil || userDB == nil { |
| 3306 | return nil |
| 3307 | } |
| 3308 | |
| 3309 | // Don't do anything, if there is nothing to compact. |
| 3310 | h := userDB.Head() |
| 3311 | if h.NumSeries() == 0 { |
| 3312 | return nil |
| 3313 | } |
| 3314 | |
| 3315 | i.TSDBState.compactionsTriggered.Inc() |
| 3316 | |
| 3317 | reason := "" |
| 3318 | switch { |
| 3319 | case force: |
| 3320 | reason = "forced" |
| 3321 | err = userDB.compactHead(ctx, i.cfg.BlocksStorageConfig.TSDB.BlockRanges[0].Milliseconds()) |
| 3322 | |
| 3323 | case i.TSDBState.compactionIdleTimeout > 0 && userDB.isIdle(time.Now(), i.TSDBState.compactionIdleTimeout): |
| 3324 | reason = "idle" |
| 3325 | level.Info(logutil.WithContext(ctx, i.logger)).Log("msg", "TSDB is idle, forcing compaction", "user", userID) |
| 3326 | err = userDB.compactHead(ctx, i.cfg.BlocksStorageConfig.TSDB.BlockRanges[0].Milliseconds()) |
| 3327 | |
| 3328 | default: |
| 3329 | reason = "regular" |
| 3330 | err = userDB.Compact(ctx) |
| 3331 | } |
| 3332 | |
| 3333 | if err != nil { |
| 3334 | i.TSDBState.compactionsFailed.Inc() |
| 3335 | level.Warn(logutil.WithContext(ctx, i.logger)).Log("msg", "TSDB blocks compaction for user has failed", "user", userID, "err", err, "compactReason", reason) |
| 3336 | } else { |
| 3337 | level.Debug(logutil.WithContext(ctx, i.logger)).Log("msg", "TSDB blocks compaction completed successfully", "user", userID, "compactReason", reason) |
| 3338 | } |
| 3339 | |
| 3340 | return nil |
| 3341 | }) |
| 3342 | } |
| 3343 | |
| 3344 | func (i *Ingester) closeAndDeleteIdleUserTSDBs(ctx context.Context) error { |
| 3345 | for _, userID := range i.getTSDBUsers() { |