fix(events): include alert details in all notify events

eventrecorder/events.go

@@ -133,14 +133,18 @@ func NewAlertGroup(groupKey string, groupLabels model.LabelSet, groupID, receive
 	}}
 }
 
-// NewGroupedAlert snapshots an alert and its notification-pipeline hash.
-func NewGroupedAlert(hash uint64, a *alert.Alert) GroupedAlert {
-	return GroupedAlert{message: &events.GroupedAlert{Hash: hash, Details: alertToEvents(a)}}
+// NewGroupedAlert snapshots an alert and its fingerprint.
+func NewGroupedAlert(a *alert.Alert) GroupedAlert {
+	var fingerprint model.Fingerprint
+	if a != nil {
+		fingerprint = a.Fingerprint()
+	}
+	return GroupedAlert{message: &events.GroupedAlert{Fingerprint: uint64(fingerprint), Details: alertToEvents(a)}}
 }
 
-// NewGroupedAlertReference snapshots a hash-only grouped-alert reference.
-func NewGroupedAlertReference(hash uint64) GroupedAlert {
-	return GroupedAlert{message: &events.GroupedAlert{Hash: hash}}
+// NewGroupedAlertReference snapshots a fingerprint-only grouped-alert reference.
+func NewGroupedAlertReference(fingerprint model.Fingerprint) GroupedAlert {
+	return GroupedAlert{message: &events.GroupedAlert{Fingerprint: uint64(fingerprint)}}
 }
 
 // NewAlertmanagerStartupEvent constructs startup event data.

eventrecorder/events_test.go

@@ -32,16 +32,31 @@ func TestAlertEventSnapshotsLabels(t *testing.T) {
 		Labels: model.LabelSet{"alertname": "Down", "severity": "warning"}, Annotations: model.LabelSet{"summary": "test"},
 		StartsAt: time.Now(), EndsAt: time.Now().Add(time.Hour),
 	}}
+	fingerprint := a.Fingerprint()
 	event := NewAlertCreatedEvent(a)
 
 	a.Labels["severity"] = "critical"
 	a.Annotations["summary"] = "changed"
 
 	got := event.message.GetAlertCreated().Alert
+	require.Equal(t, uint64(fingerprint), got.Fingerprint)
 	require.Equal(t, "warning", got.Labels["severity"])
 	require.Equal(t, "test", got.Annotations["summary"])
 }
 
+func TestGroupedAlertUsesAlertFingerprint(t *testing.T) {
+	a := &alert.Alert{Alert: model.Alert{Labels: model.LabelSet{"alertname": "Down", "instance": "api-1"}}}
+	fingerprint := a.Fingerprint()
+
+	grouped := NewGroupedAlert(a)
+	require.Equal(t, uint64(fingerprint), grouped.message.Fingerprint)
+	require.Equal(t, uint64(fingerprint), grouped.message.Details.Fingerprint)
+
+	reference := NewGroupedAlertReference(fingerprint)
+	require.Equal(t, uint64(fingerprint), reference.message.Fingerprint)
+	require.Nil(t, reference.message.Details)
+}
+
 func TestSilenceEventSnapshotsAnnotationsAndMatchers(t *testing.T) {
 	silence := &silencepb.Silence{
 		Annotations: map[string]string{"owner": "ops"},

notify/context.go

@@ -18,6 +18,7 @@ import (
 
 	"github.com/prometheus/common/model"
 
+	"github.com/prometheus/alertmanager/alert"
 	"github.com/prometheus/alertmanager/nflog"
 	"github.com/prometheus/alertmanager/pkg/labels"
 )
@@ -44,6 +45,7 @@ const (
 	keyFlushID
 	keyGroupMatchers
 	keyRouteLabels
+	keyMutedAlertDetails
 )
 
 // WithReceiverName populates a context with a receiver name.
@@ -200,6 +202,19 @@ func MutedAlerts(ctx context.Context) (map[uint64]struct{}, bool) {
 	return v, ok
 }
 
+func withMutedAlertDetails(ctx context.Context, alerts []*alert.Alert) context.Context {
+	existing, _ := mutedAlertDetails(ctx)
+	results := make([]*alert.Alert, 0, len(existing)+len(alerts))
+	results = append(results, existing...)
+	results = append(results, alerts...)
+	return context.WithValue(ctx, keyMutedAlertDetails, results)
+}
+
+func mutedAlertDetails(ctx context.Context) ([]*alert.Alert, bool) {
+	v, ok := ctx.Value(keyMutedAlertDetails).([]*alert.Alert)
+	return v, ok
+}
+
 // WithAggrGroupID populates a context with an aggregation group UUID.
 func WithAggrGroupID(ctx context.Context, id string) context.Context {
 	return context.WithValue(ctx, keyAggrGroupID, id)

notify/event.go

@@ -14,20 +14,15 @@
 package notify
 
 // This file contains helpers for constructing event recorder events
-// from the notification pipeline context.  It lives in the notify package
-// because it accesses unexported context keys (keyFiringAlerts, etc.).
+// from notification pipeline state that is internal to the notify package.
 
 import (
 	"context"
 
+	"github.com/prometheus/alertmanager/alert"
 	"github.com/prometheus/alertmanager/eventrecorder"
-	"github.com/prometheus/alertmanager/types"
 )
 
-func groupedAlertEvent(alert *types.Alert) eventrecorder.GroupedAlert {
-	return eventrecorder.NewGroupedAlert(hashAlert(alert), alert)
-}
-
 func extractAlertGroupInfo(ctx context.Context) eventrecorder.AlertGroup {
 	groupKey, _ := ExtractGroupKey(ctx)
 	receiverName, _ := ReceiverName(ctx)
@@ -40,26 +35,40 @@ func extractAlertGroupInfo(ctx context.Context) eventrecorder.AlertGroup {
 	)
 }
 
-func extractGroupedAlerts(ctx context.Context, key notifyKey) []eventrecorder.GroupedAlert {
-	var result []eventrecorder.GroupedAlert
-	if list, ok := ctx.Value(key).([]uint64); ok {
-		for _, hash := range list {
-			result = append(result, eventrecorder.NewGroupedAlertReference(hash))
-		}
+func alertDetailsByHash(alerts []*alert.Alert) map[uint64]*alert.Alert {
+	result := make(map[uint64]*alert.Alert, len(alerts))
+	for _, alert := range alerts {
+		result[hashAlert(alert)] = alert
 	}
 	return result
 }
 
-func extractMutedGroupedAlerts(ctx context.Context) []eventrecorder.GroupedAlert {
-	var result []eventrecorder.GroupedAlert
-	if muted, ok := MutedAlerts(ctx); ok {
-		for hash := range muted {
-			result = append(result, eventrecorder.NewGroupedAlertReference(hash))
+func alertDetailsForHashes(alerts map[uint64]*alert.Alert, hashes []uint64) []*alert.Alert {
+	result := make([]*alert.Alert, 0, len(hashes))
+	for _, hash := range hashes {
+		if alert, ok := alerts[hash]; ok {
+			result = append(result, alert)
 		}
 	}
 	return result
 }
 
+func groupedAlertsWithDetails(alerts []*alert.Alert) []eventrecorder.GroupedAlert {
+	result := make([]eventrecorder.GroupedAlert, 0, len(alerts))
+	for _, alert := range alerts {
+		result = append(result, eventrecorder.NewGroupedAlert(alert))
+	}
+	return result
+}
+
+func groupedAlertReferences(alerts []*alert.Alert) []eventrecorder.GroupedAlert {
+	result := make([]eventrecorder.GroupedAlert, 0, len(alerts))
+	for _, alert := range alerts {
+		result = append(result, eventrecorder.NewGroupedAlertReference(alert.Fingerprint()))
+	}
+	return result
+}
+
 func notifyReasonToEvent(reason NotifyReason) eventrecorder.NotificationReason {
 	switch reason {
 	case ReasonFirstNotification:
@@ -77,23 +86,23 @@ func notifyReasonToEvent(reason NotifyReason) eventrecorder.NotificationReason {
 	}
 }
 
-// NewNotificationEvent constructs notification event data from the pipeline
-// context after a successful notification delivery.
-func NewNotificationEvent(ctx context.Context, alerts []*types.Alert, integration Integration) eventrecorder.EventData {
-	groupedAlerts := make([]eventrecorder.GroupedAlert, 0, len(alerts))
-	for _, alert := range alerts {
-		groupedAlerts = append(groupedAlerts, groupedAlertEvent(alert))
-	}
-
+func newNotificationEvent(ctx context.Context, alerts []*alert.Alert, integration Integration) eventrecorder.EventData {
 	notifyReason, _ := NotificationReason(ctx)
 	repeatInterval, _ := RepeatInterval(ctx)
 	flushID, _ := FlushID(ctx)
+	firingHashes, _ := FiringAlerts(ctx)
+	resolvedHashes, _ := ResolvedAlerts(ctx)
+	details := alertDetailsByHash(alerts)
+	muted, _ := mutedAlertDetails(ctx)
+	allAlerts := make([]*alert.Alert, 0, len(alerts)+len(muted))
+	allAlerts = append(allAlerts, alerts...)
+	allAlerts = append(allAlerts, muted...)
 
 	return eventrecorder.NewNotificationEvent(eventrecorder.Notification{
-		Alerts:         groupedAlerts,
-		FiringAlerts:   extractGroupedAlerts(ctx, keyFiringAlerts),
-		ResolvedAlerts: extractGroupedAlerts(ctx, keyResolvedAlerts),
-		MutedAlerts:    extractMutedGroupedAlerts(ctx),
+		Alerts:         groupedAlertsWithDetails(allAlerts),
+		FiringAlerts:   groupedAlertReferences(alertDetailsForHashes(details, firingHashes)),
+		ResolvedAlerts: groupedAlertReferences(alertDetailsForHashes(details, resolvedHashes)),
+		MutedAlerts:    groupedAlertReferences(muted),
 		Group:          extractAlertGroupInfo(ctx),
 		RepeatInterval: repeatInterval,
 		Reason:         notifyReasonToEvent(notifyReason),
@@ -104,11 +113,11 @@ func NewNotificationEvent(ctx context.Context, alerts []*types.Alert, integratio
 }
 
 // NewAlertResolvedEvent constructs alert-resolved event data.
-func NewAlertResolvedEvent(groupInfo eventrecorder.AlertGroup, alert *types.Alert) eventrecorder.EventData {
-	return eventrecorder.NewAlertResolvedEvent(groupInfo, groupedAlertEvent(alert))
+func NewAlertResolvedEvent(groupInfo eventrecorder.AlertGroup, alert *alert.Alert) eventrecorder.EventData {
+	return eventrecorder.NewAlertResolvedEvent(groupInfo, eventrecorder.NewGroupedAlert(alert))
 }
 
 // NewAlertGroupedEvent constructs alert-grouped event data.
-func NewAlertGroupedEvent(groupInfo eventrecorder.AlertGroup, alert *types.Alert) eventrecorder.EventData {
-	return eventrecorder.NewAlertGroupedEvent(groupInfo, groupedAlertEvent(alert))
+func NewAlertGroupedEvent(groupInfo eventrecorder.AlertGroup, alert *alert.Alert) eventrecorder.EventData {
+	return eventrecorder.NewAlertGroupedEvent(groupInfo, eventrecorder.NewGroupedAlert(alert))
 }

notify/mute.go

@@ -61,7 +61,8 @@ func recordMuted(ctx context.Context, muted []*alert.Alert) context.Context {
 	for _, a := range muted {
 		hashes[hashAlert(a)] = struct{}{}
 	}
-	return WithMutedAlerts(ctx, hashes)
+	ctx = WithMutedAlerts(ctx, hashes)
+	return withMutedAlertDetails(ctx, muted)
 }
 
 // MuteStage filters alerts through a Muter.

notify/mute_test.go

@@ -91,6 +91,29 @@ func TestMuteStage(t *testing.T) {
 	}
 }
 
+func TestMuteStageAccumulatesMutedAlertDetails(t *testing.T) {
+	metrics := NewMetrics(prometheus.NewRegistry(), featurecontrol.NoopFlags{})
+	firstStage := NewMuteStage(MuteFunc(func(_ context.Context, lset model.LabelSet) bool {
+		return lset["muted_by"] == "first"
+	}), metrics)
+	secondStage := NewMuteStage(MuteFunc(func(_ context.Context, lset model.LabelSet) bool {
+		return lset["muted_by"] == "second"
+	}), metrics)
+	first := &alert.Alert{Alert: model.Alert{Labels: model.LabelSet{"alertname": "First", "muted_by": "first"}}}
+	second := &alert.Alert{Alert: model.Alert{Labels: model.LabelSet{"alertname": "Second", "muted_by": "second"}}}
+	active := &alert.Alert{Alert: model.Alert{Labels: model.LabelSet{"alertname": "Active"}}}
+
+	ctx, alerts, err := firstStage.Exec(context.Background(), promslog.NewNopLogger(), first, second, active)
+	require.NoError(t, err)
+	ctx, alerts, err = secondStage.Exec(ctx, promslog.NewNopLogger(), alerts...)
+	require.NoError(t, err)
+	require.Equal(t, []*alert.Alert{active}, alerts)
+
+	muted, ok := mutedAlertDetails(ctx)
+	require.True(t, ok)
+	require.Equal(t, []*alert.Alert{first, second}, muted)
+}
+
 func TestMuteStageWithSilences(t *testing.T) {
 	silences, err := silence.New(silence.Options{Metrics: prometheus.NewRegistry(), Retention: time.Hour})
 	if err != nil {