diff --git a/dispatch/dispatch.go b/dispatch/dispatch.go index 2e3da19970..48be401fb5 100644 --- a/dispatch/dispatch.go +++ b/dispatch/dispatch.go @@ -578,7 +578,10 @@ func (d *Dispatcher) runAG(ag *aggrGroup) { go ag.run(func(ctx context.Context, alerts ...*alert.Alert) bool { _, _, err := d.stage.Exec(ctx, d.logger, alerts...) if err != nil { - logger := d.logger.With("aggrGroup", ag.GroupKey(), "num_alerts", len(alerts), "err", err) + logger := d.logger.With("aggrGroup", ag.GroupKey(), "num_alerts", len(alerts), "receiver", ag.opts.Receiver, "err", err) + if ie, ok := errors.AsType[*notify.ErrorWithIntegration](err); ok { + logger = logger.With("integration", ie.Integration) + } if errors.Is(ctx.Err(), context.Canceled) { // It is expected for the context to be canceled on // configuration reload or shutdown. In this case, the diff --git a/notify/notify_test.go b/notify/notify_test.go index e536223375..9800993b1c 100644 --- a/notify/notify_test.go +++ b/notify/notify_test.go @@ -500,6 +500,8 @@ func TestRetryStageWithError(t *testing.T) { fail, retry := true, true sent := []*alert.Alert{} i := Integration{ + name: "slack", + idx: 0, notifier: notifierFunc(func(ctx context.Context, alerts ...*alert.Alert) (bool, error) { if fail { fail = false @@ -510,7 +512,7 @@ func TestRetryStageWithError(t *testing.T) { }), rs: sendResolved(false), } - r := NewRetryStage(i, "", NewMetrics(prometheus.NewRegistry(), featurecontrol.NoopFlags{}), eventrecorder.NopRecorder()) + r := NewRetryStage(i, "team-receiver", NewMetrics(prometheus.NewRegistry(), featurecontrol.NoopFlags{}), eventrecorder.NopRecorder()) alerts := []*alert.Alert{ { @@ -537,6 +539,12 @@ func TestRetryStageWithError(t *testing.T) { resctx, _, err = r.Exec(ctx, promslog.NewNopLogger(), alerts...) require.Error(t, err) require.NotNil(t, resctx) + + var ie *ErrorWithIntegration + require.ErrorAs(t, err, &ie) + require.Equal(t, "team-receiver", ie.Receiver) + require.Equal(t, "slack[0]", ie.Integration) + require.Contains(t, err.Error(), "fail to deliver notification") } func TestRetryStageWithErrorCode(t *testing.T) { @@ -598,7 +606,7 @@ func TestRetryStageWithContextCanceled(t *testing.T) { }), rs: sendResolved(false), } - r := NewRetryStage(i, "", NewMetrics(prometheus.NewRegistry(), featurecontrol.NoopFlags{}), eventrecorder.NopRecorder()) + r := NewRetryStage(i, "canceled-receiver", NewMetrics(prometheus.NewRegistry(), featurecontrol.NoopFlags{}), eventrecorder.NopRecorder()) alerts := []*alert.Alert{ { @@ -619,6 +627,11 @@ func TestRetryStageWithContextCanceled(t *testing.T) { require.Error(t, err) require.NotNil(t, resctx) + + var ie *ErrorWithIntegration + require.ErrorAs(t, err, &ie) + require.Equal(t, "canceled-receiver", ie.Receiver) + require.Equal(t, "test[0]", ie.Integration) } func TestRetryStageNoResolved(t *testing.T) { diff --git a/notify/retry_stage.go b/notify/retry_stage.go index 7c71f9c7d5..cf1cf9d0e2 100644 --- a/notify/retry_stage.go +++ b/notify/retry_stage.go @@ -144,7 +144,8 @@ func (r RetryStage) exec(ctx context.Context, l *slog.Logger, alerts ...*alert.A } if iErr != nil { - return ctx, nil, fmt.Errorf("%s/%s: notify retry canceled after %d attempts: %w", r.groupName, r.integration.String(), i, iErr) + return ctx, nil, NewErrorWithIntegration(r.groupName, r.integration.String(), + fmt.Errorf("%s/%s: notify retry canceled after %d attempts: %w", r.groupName, r.integration.String(), i, iErr)) } return ctx, nil, nil default: @@ -161,7 +162,8 @@ func (r RetryStage) exec(ctx context.Context, l *slog.Logger, alerts ...*alert.A if err != nil { r.metrics.numNotificationRequestsFailedTotal.WithLabelValues(r.labelValues...).Inc() if !retry { - return ctx, alerts, fmt.Errorf("%s/%s: notify retry canceled due to unrecoverable error after %d attempts: %w", r.groupName, r.integration.String(), i, err) + return ctx, alerts, NewErrorWithIntegration(r.groupName, r.integration.String(), + fmt.Errorf("%s/%s: notify retry canceled due to unrecoverable error after %d attempts: %w", r.groupName, r.integration.String(), i, err)) } if ctx.Err() == nil { if iErr == nil || err.Error() != iErr.Error() { diff --git a/notify/util.go b/notify/util.go index fe4c9ea508..4a1ae48762 100644 --- a/notify/util.go +++ b/notify/util.go @@ -281,6 +281,31 @@ func (e *ErrorWithReason) Error() string { return e.Err.Error() } +// ErrorWithIntegration annotates a notification failure with the receiver and +// integration that failed, so callers can attach them as structured log fields. +type ErrorWithIntegration struct { + Receiver string + Integration string + Err error +} + +// NewErrorWithIntegration returns an error annotated with receiver and integration. +func NewErrorWithIntegration(receiver, integration string, err error) *ErrorWithIntegration { + return &ErrorWithIntegration{ + Receiver: receiver, + Integration: integration, + Err: err, + } +} + +func (e *ErrorWithIntegration) Error() string { + return e.Err.Error() +} + +func (e *ErrorWithIntegration) Unwrap() error { + return e.Err +} + // Reason is the failure reason. type Reason int