From 0f91fb4412c3b65d289af7686da6d5a54854d782 Mon Sep 17 00:00:00 2001 From: Dean Chen <862469039@qq.com> Date: Wed, 22 Jul 2026 15:44:58 +0500 Subject: [PATCH 1/2] notify: add receiver and integration as structured log fields on failures When notification delivery fails, the dispatcher error log only exposed receiver/integration inside the free-text err string. Annotate RetryStage failures with ErrorWithIntegration and attach receiver/integration as structured fields on the dispatch failure log so operators can group by them in log aggregators without regex-parsing err. Fixes #5396 Signed-off-by: Dean Chen <862469039@qq.com> --- dispatch/dispatch.go | 6 +++++- notify/notify_test.go | 17 +++++++++++++++-- notify/retry_stage.go | 6 ++++-- notify/util.go | 25 +++++++++++++++++++++++++ 4 files changed, 49 insertions(+), 5 deletions(-) diff --git a/dispatch/dispatch.go b/dispatch/dispatch.go index 2e3da19970..b59ba92fed 100644 --- a/dispatch/dispatch.go +++ b/dispatch/dispatch.go @@ -578,7 +578,11 @@ func (d *Dispatcher) runAG(ag *aggrGroup) { go ag.run(func(ctx context.Context, alerts ...*alert.Alert) bool { _, _, err := d.stage.Exec(ctx, d.logger, alerts...) if err != nil { - logger := d.logger.With("aggrGroup", ag.GroupKey(), "num_alerts", len(alerts), "err", err) + logger := d.logger.With("aggrGroup", ag.GroupKey(), "num_alerts", len(alerts), "receiver", ag.opts.Receiver, "err", err) + var ie *notify.ErrorWithIntegration + if errors.As(err, &ie) { + logger = logger.With("integration", ie.Integration) + } if errors.Is(ctx.Err(), context.Canceled) { // It is expected for the context to be canceled on // configuration reload or shutdown. In this case, the diff --git a/notify/notify_test.go b/notify/notify_test.go index e536223375..9800993b1c 100644 --- a/notify/notify_test.go +++ b/notify/notify_test.go @@ -500,6 +500,8 @@ func TestRetryStageWithError(t *testing.T) { fail, retry := true, true sent := []*alert.Alert{} i := Integration{ + name: "slack", + idx: 0, notifier: notifierFunc(func(ctx context.Context, alerts ...*alert.Alert) (bool, error) { if fail { fail = false @@ -510,7 +512,7 @@ func TestRetryStageWithError(t *testing.T) { }), rs: sendResolved(false), } - r := NewRetryStage(i, "", NewMetrics(prometheus.NewRegistry(), featurecontrol.NoopFlags{}), eventrecorder.NopRecorder()) + r := NewRetryStage(i, "team-receiver", NewMetrics(prometheus.NewRegistry(), featurecontrol.NoopFlags{}), eventrecorder.NopRecorder()) alerts := []*alert.Alert{ { @@ -537,6 +539,12 @@ func TestRetryStageWithError(t *testing.T) { resctx, _, err = r.Exec(ctx, promslog.NewNopLogger(), alerts...) require.Error(t, err) require.NotNil(t, resctx) + + var ie *ErrorWithIntegration + require.ErrorAs(t, err, &ie) + require.Equal(t, "team-receiver", ie.Receiver) + require.Equal(t, "slack[0]", ie.Integration) + require.Contains(t, err.Error(), "fail to deliver notification") } func TestRetryStageWithErrorCode(t *testing.T) { @@ -598,7 +606,7 @@ func TestRetryStageWithContextCanceled(t *testing.T) { }), rs: sendResolved(false), } - r := NewRetryStage(i, "", NewMetrics(prometheus.NewRegistry(), featurecontrol.NoopFlags{}), eventrecorder.NopRecorder()) + r := NewRetryStage(i, "canceled-receiver", NewMetrics(prometheus.NewRegistry(), featurecontrol.NoopFlags{}), eventrecorder.NopRecorder()) alerts := []*alert.Alert{ { @@ -619,6 +627,11 @@ func TestRetryStageWithContextCanceled(t *testing.T) { require.Error(t, err) require.NotNil(t, resctx) + + var ie *ErrorWithIntegration + require.ErrorAs(t, err, &ie) + require.Equal(t, "canceled-receiver", ie.Receiver) + require.Equal(t, "test[0]", ie.Integration) } func TestRetryStageNoResolved(t *testing.T) { diff --git a/notify/retry_stage.go b/notify/retry_stage.go index 7c71f9c7d5..cf1cf9d0e2 100644 --- a/notify/retry_stage.go +++ b/notify/retry_stage.go @@ -144,7 +144,8 @@ func (r RetryStage) exec(ctx context.Context, l *slog.Logger, alerts ...*alert.A } if iErr != nil { - return ctx, nil, fmt.Errorf("%s/%s: notify retry canceled after %d attempts: %w", r.groupName, r.integration.String(), i, iErr) + return ctx, nil, NewErrorWithIntegration(r.groupName, r.integration.String(), + fmt.Errorf("%s/%s: notify retry canceled after %d attempts: %w", r.groupName, r.integration.String(), i, iErr)) } return ctx, nil, nil default: @@ -161,7 +162,8 @@ func (r RetryStage) exec(ctx context.Context, l *slog.Logger, alerts ...*alert.A if err != nil { r.metrics.numNotificationRequestsFailedTotal.WithLabelValues(r.labelValues...).Inc() if !retry { - return ctx, alerts, fmt.Errorf("%s/%s: notify retry canceled due to unrecoverable error after %d attempts: %w", r.groupName, r.integration.String(), i, err) + return ctx, alerts, NewErrorWithIntegration(r.groupName, r.integration.String(), + fmt.Errorf("%s/%s: notify retry canceled due to unrecoverable error after %d attempts: %w", r.groupName, r.integration.String(), i, err)) } if ctx.Err() == nil { if iErr == nil || err.Error() != iErr.Error() { diff --git a/notify/util.go b/notify/util.go index fe4c9ea508..4a1ae48762 100644 --- a/notify/util.go +++ b/notify/util.go @@ -281,6 +281,31 @@ func (e *ErrorWithReason) Error() string { return e.Err.Error() } +// ErrorWithIntegration annotates a notification failure with the receiver and +// integration that failed, so callers can attach them as structured log fields. +type ErrorWithIntegration struct { + Receiver string + Integration string + Err error +} + +// NewErrorWithIntegration returns an error annotated with receiver and integration. +func NewErrorWithIntegration(receiver, integration string, err error) *ErrorWithIntegration { + return &ErrorWithIntegration{ + Receiver: receiver, + Integration: integration, + Err: err, + } +} + +func (e *ErrorWithIntegration) Error() string { + return e.Err.Error() +} + +func (e *ErrorWithIntegration) Unwrap() error { + return e.Err +} + // Reason is the failure reason. type Reason int From 063b00a3438b9bd2ff533b778a26b10b782ef49d Mon Sep 17 00:00:00 2001 From: Dean Chen <862469039@qq.com> Date: Tue, 15 Sep 2026 13:29:21 +0500 Subject: [PATCH 2/2] dispatch: use errors.AsType for ErrorWithIntegration golangci-lint modernize flags errors.As here. Signed-off-by: Dean Chen <862469039@qq.com> --- dispatch/dispatch.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dispatch/dispatch.go b/dispatch/dispatch.go index b59ba92fed..48be401fb5 100644 --- a/dispatch/dispatch.go +++ b/dispatch/dispatch.go @@ -579,8 +579,7 @@ func (d *Dispatcher) runAG(ag *aggrGroup) { _, _, err := d.stage.Exec(ctx, d.logger, alerts...) if err != nil { logger := d.logger.With("aggrGroup", ag.GroupKey(), "num_alerts", len(alerts), "receiver", ag.opts.Receiver, "err", err) - var ie *notify.ErrorWithIntegration - if errors.As(err, &ie) { + if ie, ok := errors.AsType[*notify.ErrorWithIntegration](err); ok { logger = logger.With("integration", ie.Integration) } if errors.Is(ctx.Err(), context.Canceled) {