Function TestAlertmanagerShardingScaling

integration/alertmanager_test.go:541–731

(t *testing.T)

}

func TestAlertmanagerShardingScaling(t *testing.T) {
	// Note that we run the test with the persister interval reduced in
	// order to speed up the testing. However, this could mask issues with
	// the syncing state from replicas. Therefore, we also run the tests
	// with the sync interval increased (with the caveat that we cannot
	// test the all-replica shutdown/restart).
	tests := map[string]struct {
		replicationFactor int
		withPersister     bool
	}{
		"RF = 2 with persister":    {replicationFactor: 2, withPersister: true},
		"RF = 3 with persister":    {replicationFactor: 3, withPersister: true},
		"RF = 2 without persister": {replicationFactor: 2, withPersister: false},
		"RF = 3 without persister": {replicationFactor: 3, withPersister: false},
	}

	for testName, testCfg := range tests {
		t.Run(testName, func(t *testing.T) {
			s, err := e2e.NewScenario(networkName)
			require.NoError(t, err)
			defer s.Close()

			// Start dependencies.
			consul := e2edb.NewConsul()
			minio := e2edb.NewMinio(9000, alertsBucketName)
			require.NoError(t, s.StartAndWaitReady(consul, minio))

			client, err := s3.NewBucketWithConfig(nil, s3.Config{
				Endpoint:  minio.HTTPEndpoint(),
				Insecure:  true,
				Bucket:    alertsBucketName,
				AccessKey: e2edb.MinioAccessKey,
				SecretKey: e2edb.MinioSecretKey,
			}, "alertmanager-test", nil)
			require.NoError(t, err)

			// Create and upload Alertmanager configurations.
			numUsers := 20
			for i := 1; i <= numUsers; i++ {
				user := fmt.Sprintf("user-%d", i)
				desc := alertspb.AlertConfigDesc{
					RawConfig: simpleAlertmanagerConfig,
					User:      user,
					Templates: []*alertspb.TemplateDesc{},
				}

				d, err := desc.Marshal()
				require.NoError(t, err)
				err = client.Upload(context.Background(), fmt.Sprintf("/alerts/%s", user), bytes.NewReader(d))
				require.NoError(t, err)
			}

			persistInterval := "5h"
			if testCfg.withPersister {
				persistInterval = "5s"
			}

			flags := mergeFlags(AlertmanagerFlags(),