(t *testing.T)
| 20 | ) |
| 21 | |
| 22 | func TestZoneAwareReplication(t *testing.T) { |
| 23 | s, err := e2e.NewScenario(networkName) |
| 24 | require.NoError(t, err) |
| 25 | defer s.Close() |
| 26 | |
| 27 | flags := BlocksStorageFlags() |
| 28 | flags["-distributor.shard-by-all-labels"] = "true" |
| 29 | flags["-distributor.replication-factor"] = "3" |
| 30 | flags["-distributor.zone-awareness-enabled"] = "true" |
| 31 | |
| 32 | // Start dependencies. |
| 33 | consul := e2edb.NewConsul() |
| 34 | minio := e2edb.NewMinio(9000, flags["-blocks-storage.s3.bucket-name"]) |
| 35 | require.NoError(t, s.StartAndWaitReady(consul, minio)) |
| 36 | |
| 37 | // Start Cortex components. |
| 38 | ingesterFlags := func(zone string) map[string]string { |
| 39 | return mergeFlags(flags, map[string]string{ |
| 40 | "-ingester.availability-zone": zone, |
| 41 | }) |
| 42 | } |
| 43 | |
| 44 | ingester1 := e2ecortex.NewIngesterWithConfigFile("ingester-1", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), "", ingesterFlags("zone-a"), "") |
| 45 | ingester2 := e2ecortex.NewIngesterWithConfigFile("ingester-2", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), "", ingesterFlags("zone-a"), "") |
| 46 | ingester3 := e2ecortex.NewIngesterWithConfigFile("ingester-3", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), "", ingesterFlags("zone-b"), "") |
| 47 | ingester4 := e2ecortex.NewIngesterWithConfigFile("ingester-4", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), "", ingesterFlags("zone-b"), "") |
| 48 | ingester5 := e2ecortex.NewIngesterWithConfigFile("ingester-5", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), "", ingesterFlags("zone-c"), "") |
| 49 | ingester6 := e2ecortex.NewIngesterWithConfigFile("ingester-6", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), "", ingesterFlags("zone-c"), "") |
| 50 | require.NoError(t, s.StartAndWaitReady(ingester1, ingester2, ingester3, ingester4, ingester5, ingester6)) |
| 51 | |
| 52 | distributor := e2ecortex.NewDistributor("distributor", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flags, "") |
| 53 | querier := e2ecortex.NewQuerier("querier", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flags, "") |
| 54 | require.NoError(t, s.StartAndWaitReady(distributor, querier)) |
| 55 | |
| 56 | // Wait until distributor and querier have updated the ring. |
| 57 | require.NoError(t, distributor.WaitSumMetricsWithOptions(e2e.Equals(6), []string{"cortex_ring_members"}, e2e.WithLabelMatchers( |
| 58 | labels.MustNewMatcher(labels.MatchEqual, "name", "ingester"), |
| 59 | labels.MustNewMatcher(labels.MatchEqual, "state", "ACTIVE")))) |
| 60 | |
| 61 | require.NoError(t, querier.WaitSumMetricsWithOptions(e2e.Equals(6), []string{"cortex_ring_members"}, e2e.WithLabelMatchers( |
| 62 | labels.MustNewMatcher(labels.MatchEqual, "name", "ingester"), |
| 63 | labels.MustNewMatcher(labels.MatchEqual, "state", "ACTIVE")))) |
| 64 | |
| 65 | client, err := e2ecortex.NewClient(distributor.HTTPEndpoint(), querier.HTTPEndpoint(), "", "", userID) |
| 66 | require.NoError(t, err) |
| 67 | |
| 68 | // Push some series |
| 69 | now := time.Now() |
| 70 | numSeries := 100 |
| 71 | expectedVectors := map[string]model.Vector{} |
| 72 | |
| 73 | for i := 1; i <= numSeries; i++ { |
| 74 | metricName := fmt.Sprintf("series_%d", i) |
| 75 | series, expectedVector := generateSeries(metricName, now) |
| 76 | res, err := client.Push(series) |
| 77 | require.NoError(t, err) |
| 78 | require.Equal(t, 200, res.StatusCode) |
| 79 |
nothing calls this directly
no test coverage detected