Function TestAlertmanagerClustering

integration/alertmanager_test.go:176–227

(t *testing.T)

}

func TestAlertmanagerClustering(t *testing.T) {
	s, err := e2e.NewScenario(networkName)
	require.NoError(t, err)
	defer s.Close()

	flags := mergeFlags(AlertmanagerFlags(), AlertmanagerS3Flags())

	// Start dependencies.
	minio := e2edb.NewMinio(9000, alertsBucketName)
	require.NoError(t, s.StartAndWaitReady(minio))

	client, err := s3.NewBucketWithConfig(nil, s3.Config{
		Endpoint:  minio.HTTPEndpoint(),
		Insecure:  true,
		Bucket:    alertsBucketName,
		AccessKey: e2edb.MinioAccessKey,
		SecretKey: e2edb.MinioSecretKey,
	}, "alertmanager-test", nil)
	require.NoError(t, err)

	// Create and upload an Alertmanager configuration.
	user := "user-1"
	desc := alertspb.AlertConfigDesc{RawConfig: simpleAlertmanagerConfig, User: user, Templates: []*alertspb.TemplateDesc{}}

	d, err := desc.Marshal()
	require.NoError(t, err)
	err = client.Upload(context.Background(), fmt.Sprintf("/alerts/%s", user), bytes.NewReader(d))
	require.NoError(t, err)

	peers := strings.Join([]string{
		e2e.NetworkContainerHostPort(networkName, "alertmanager-1", e2ecortex.GossipPort),
		e2e.NetworkContainerHostPort(networkName, "alertmanager-2", e2ecortex.GossipPort),
	}, ",")
	flags = mergeFlags(flags, AlertmanagerClusterFlags(peers))

	// Wait for the Alertmanagers to start.
	alertmanager1 := e2ecortex.NewAlertmanager("alertmanager-1", flags, "")
	alertmanager2 := e2ecortex.NewAlertmanager("alertmanager-2", flags, "")

	alertmanagers := e2ecortex.NewCompositeCortexService(alertmanager1, alertmanager2)

	// Start Alertmanager instances.
	for _, am := range alertmanagers.Instances() {
		require.NoError(t, s.StartAndWaitReady(am))
	}

	for _, am := range alertmanagers.Instances() {
		require.NoError(t, am.WaitSumMetrics(e2e.Equals(float64(0)), "alertmanager_cluster_health_score")) // Lower means healthier, 0 being totally healthy.
		require.NoError(t, am.WaitSumMetrics(e2e.Equals(float64(0)), "alertmanager_cluster_failed_peers"))
		require.NoError(t, am.WaitSumMetrics(e2e.Equals(float64(2)), "alertmanager_cluster_members"))
	}
}

func TestAlertmanagerSharding(t *testing.T) {
	tests := map[string]struct {