(t *testing.T)
| 2122 | } |
| 2123 | |
| 2124 | func TestCompactor_FailedWithHaltError(t *testing.T) { |
| 2125 | t.Parallel() |
| 2126 | |
| 2127 | ss := bucketindex.Status{Status: bucketindex.Ok, Version: bucketindex.SyncStatusFileVersion} |
| 2128 | content, err := json.Marshal(ss) |
| 2129 | require.NoError(t, err) |
| 2130 | |
| 2131 | bucketClient := &bucket.ClientMock{} |
| 2132 | bucketClient.MockGet(users.UserIndexCompressedFilename, "", nil) |
| 2133 | bucketClient.MockUpload(users.UserIndexCompressedFilename, nil) |
| 2134 | bucketClient.MockIter("__markers__", []string{}, nil) |
| 2135 | bucketClient.MockIter("", []string{"user-1"}, nil) |
| 2136 | bucketClient.MockIter("user-1/", []string{"user-1/01DTVP434PA9VFXSW2JKB3392D", "user-1/01DTW0ZCPDDNV4BV83Q2SV4QAZ", "user-1/01DTVP434PA9VFXSW2JKB3392D/meta.json", "user-1/01DTW0ZCPDDNV4BV83Q2SV4QAZ/meta.json"}, nil) |
| 2137 | bucketClient.MockIter("user-1/markers/", nil, nil) |
| 2138 | bucketClient.MockGet("user-1/markers/cleaner-visit-marker.json", "", nil) |
| 2139 | bucketClient.MockUpload("user-1/markers/cleaner-visit-marker.json", nil) |
| 2140 | bucketClient.MockDelete("user-1/markers/cleaner-visit-marker.json", nil) |
| 2141 | bucketClient.MockExists(users.GetGlobalDeletionMarkPath("user-1"), false, nil) |
| 2142 | bucketClient.MockExists(users.GetLocalDeletionMarkPath("user-1"), false, nil) |
| 2143 | bucketClient.MockIter("user-1/01DTVP434PA9VFXSW2JKB3392D", nil, compact.HaltError{}) |
| 2144 | bucketClient.MockGet("user-1/01DTVP434PA9VFXSW2JKB3392D/meta.json", mockBlockMetaJSON("01DTVP434PA9VFXSW2JKB3392D"), nil) |
| 2145 | bucketClient.MockGet("user-1/01DTVP434PA9VFXSW2JKB3392D/deletion-mark.json", "", nil) |
| 2146 | bucketClient.MockGet("user-1/01DTVP434PA9VFXSW2JKB3392D/no-compact-mark.json", "", nil) |
| 2147 | bucketClient.MockIter("user-1/01DTW0ZCPDDNV4BV83Q2SV4QAZ", nil, compact.HaltError{}) |
| 2148 | bucketClient.MockGet("user-1/01DTW0ZCPDDNV4BV83Q2SV4QAZ/meta.json", mockBlockMetaJSON("01DTW0ZCPDDNV4BV83Q2SV4QAZ"), nil) |
| 2149 | bucketClient.MockGet("user-1/01DTW0ZCPDDNV4BV83Q2SV4QAZ/deletion-mark.json", "", nil) |
| 2150 | bucketClient.MockGet("user-1/01DTW0ZCPDDNV4BV83Q2SV4QAZ/no-compact-mark.json", "", nil) |
| 2151 | bucketClient.MockGet("user-1/bucket-index.json.gz", "", nil) |
| 2152 | bucketClient.MockGet("user-1/bucket-index-sync-status.json", string(content), nil) |
| 2153 | bucketClient.MockUpload("user-1/bucket-index.json.gz", nil) |
| 2154 | bucketClient.MockUpload("user-1/bucket-index-sync-status.json", nil) |
| 2155 | |
| 2156 | cfg := prepareConfig() |
| 2157 | cfg.CompactionRetries = 2 |
| 2158 | |
| 2159 | c, _, tsdbPlanner, _, registry := prepare(t, cfg, bucketClient, nil) |
| 2160 | tsdbPlanner.On("Plan", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return([]*metadata.Meta{{BlockMeta: mockBlockMeta("01DTVP434PA9VFXSW2JKB3392D")}, {BlockMeta: mockBlockMeta("01DTW0ZCPDDNV4BV83Q2SV4QAZ")}}, nil) |
| 2161 | |
| 2162 | require.NoError(t, services.StartAndAwaitRunning(context.Background(), c)) |
| 2163 | |
| 2164 | cortex_testutil.Poll(t, 1*time.Second, 1.0, func() any { |
| 2165 | return prom_testutil.ToFloat64(c.compactorMetrics.compactionErrorsCount.WithLabelValues("user-1", haltError)) |
| 2166 | }) |
| 2167 | |
| 2168 | require.NoError(t, services.StopAndAwaitTerminated(context.Background(), c)) |
| 2169 | |
| 2170 | assert.NoError(t, prom_testutil.GatherAndCompare(registry, strings.NewReader(` |
| 2171 | # HELP cortex_compactor_compaction_error_total Total number of errors from compactions. |
| 2172 | # TYPE cortex_compactor_compaction_error_total counter |
| 2173 | cortex_compactor_compaction_error_total{type="halt", user="user-1"} 1 |
| 2174 | `), |
| 2175 | "cortex_compactor_compaction_error_total", |
| 2176 | )) |
| 2177 | } |
| 2178 | |
| 2179 | func TestCompactor_RingLifecyclerShouldAutoForgetUnhealthyInstances(t *testing.T) { |
| 2180 | // Setup user IDs |
nothing calls this directly
no test coverage detected