(t *testing.T)
| 1240 | } |
| 1241 | |
| 1242 | func TestRulerMetricsWhenIngesterFails(t *testing.T) { |
| 1243 | s, err := e2e.NewScenario(networkName) |
| 1244 | require.NoError(t, err) |
| 1245 | defer s.Close() |
| 1246 | |
| 1247 | // Start dependencies. |
| 1248 | consul := e2edb.NewConsul() |
| 1249 | minio := e2edb.NewMinio(9000, bucketName, rulestoreBucketName) |
| 1250 | require.NoError(t, s.StartAndWaitReady(consul, minio)) |
| 1251 | |
| 1252 | const blockRangePeriod = 2 * time.Second |
| 1253 | // Configure the ruler. |
| 1254 | flags := mergeFlags( |
| 1255 | BlocksStorageFlags(), |
| 1256 | RulerFlags(), |
| 1257 | map[string]string{ |
| 1258 | "-blocks-storage.tsdb.block-ranges-period": blockRangePeriod.String(), |
| 1259 | "-blocks-storage.tsdb.ship-interval": "1s", |
| 1260 | "-blocks-storage.bucket-store.sync-interval": "1s", |
| 1261 | "-blocks-storage.bucket-store.index-cache.backend": tsdb.IndexCacheBackendInMemory, |
| 1262 | "-blocks-storage.tsdb.retention-period": ((blockRangePeriod * 2) - 1).String(), |
| 1263 | |
| 1264 | // Enable the bucket index so we can skip the initial bucket scan. |
| 1265 | "-blocks-storage.bucket-store.bucket-index.enabled": "false", |
| 1266 | // Evaluate rules often, so that we don't need to wait for metrics to show up. |
| 1267 | "-ruler.evaluation-interval": "2s", |
| 1268 | "-ruler.poll-interval": "2s", |
| 1269 | // No delay |
| 1270 | "-ruler.evaluation-delay-duration": "0", |
| 1271 | |
| 1272 | // We run single ingester only, no replication. |
| 1273 | "-distributor.replication-factor": "1", |
| 1274 | |
| 1275 | // Very low limit so that ruler hits it. |
| 1276 | "-querier.max-fetched-chunks-per-query": "15", |
| 1277 | "-querier.query-store-after": (1 * time.Second).String(), |
| 1278 | "-querier.query-ingesters-within": (2 * time.Second).String(), |
| 1279 | }, |
| 1280 | ) |
| 1281 | |
| 1282 | const namespace = "test" |
| 1283 | const user = "user" |
| 1284 | |
| 1285 | storeGateway := e2ecortex.NewStoreGateway("store-gateway-1", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flags, "") |
| 1286 | |
| 1287 | flags = mergeFlags(flags, map[string]string{ |
| 1288 | "-querier.store-gateway-addresses": storeGateway.NetworkGRPCEndpoint(), |
| 1289 | }) |
| 1290 | |
| 1291 | distributor := e2ecortex.NewDistributor("distributor", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flags, "") |
| 1292 | ruler := e2ecortex.NewRuler("ruler", consul.NetworkHTTPEndpoint(), flags, "") |
| 1293 | ingester := e2ecortex.NewIngester("ingester", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flags, "") |
| 1294 | require.NoError(t, s.StartAndWaitReady(distributor, ingester, ruler, storeGateway)) |
| 1295 | |
| 1296 | // Wait until both the distributor and ruler have updated the ring. The querier will also watch |
| 1297 | // the store-gateway ring if blocks sharding is enabled. |
| 1298 | require.NoError(t, distributor.WaitSumMetrics(e2e.Equals(512), "cortex_ring_tokens_total")) |
| 1299 | require.NoError(t, ruler.WaitSumMetrics(e2e.Equals(1024), "cortex_ring_tokens_total")) |
nothing calls this directly
no test coverage detected