From 4a6794d4da24953db972b201211ef25009a61710 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sun, 8 Nov 2026 11:18:52 -0800 Subject: [PATCH 1/5] feat(alerts): deliver alerts by email Alerts could only reach a webhook, so someone had to run an endpoint before any alert reached a person. Rules can now email up to 10 addresses using the existing SMTP settings, and FANOUT_ALERTS_EMAIL_DEFAULT_TO gives a destination to rules that have none. A notification is queued per recipient in the same transaction as the alert transition. One worker sends with persisted backoff, so a restart resumes the queue. Delivery is at least once: only a message interrupted mid-handoff can repeat, and it keeps its Message-ID. FANOUT_ALERTS_EMAIL_HOURLY_LIMIT caps mail per address across all rules. Mail over the limit is suppressed and reported on the rule. Telemetry values are reduced to one printable line before reaching headers or text, and escaped in HTML. Webhook delivery is unchanged. A rule with no webhook reports its email result on the alert in place of the webhook's "skipped". Closes #447 --- .env.example | 10 + cmd/fanout-docgen/main.go | 6 +- cmd/fanout/main.go | 27 +- internal/alert/email.go | 462 ++++++++++++++++ internal/alert/email_test.go | 511 ++++++++++++++++++ internal/alert/engine.go | 43 +- internal/alert/store.go | 51 +- internal/alert/types.go | 12 +- internal/api/alerts.go | 41 +- internal/api/alerts_email_test.go | 157 ++++++ internal/api/auth_middleware.go | 2 +- internal/auth/alert_mail.go | 145 +++++ internal/auth/alert_mail_test.go | 298 ++++++++++ internal/auth/email.go | 35 ++ internal/auth/smtp.go | 5 +- internal/config/config.go | 23 + internal/config/config_test.go | 30 + internal/db/generated/alert_email.sql.go | 247 +++++++++ internal/db/generated/models.go | 51 +- internal/db/generated/rules.sql.go | 40 +- .../migrations/20261108120000_alert_email.sql | 27 + internal/db/migrations_test.go | 1 + internal/db/queries/alert_email.sql | 31 ++ internal/db/queries/rules.sql | 6 +- internal/ui/dist/assets/alerts-DD6_LYeu.js | 1 + internal/ui/dist/assets/alerts-DH7dglkG.js | 1 - .../{chat-C9Lgsz9A.js => chat-S7LVuufB.js} | 4 +- .../ui/dist/assets/chat._threadId-CWjhRiA-.js | 1 + .../ui/dist/assets/chat._threadId-_q5tLfzU.js | 1 - .../ui/dist/assets/chat.index-CWjhRiA-.js | 1 + .../ui/dist/assets/chat.index-_q5tLfzU.js | 1 - ...js => dashboards._dashboardId-CHorvktr.js} | 2 +- ...P8lnwh.js => dashboards.index-OaN8mvAC.js} | 2 +- ...xplore-qAMgIiPy.js => explore-By3WiS-G.js} | 2 +- .../{index-DEaVwvoN.js => index-CO28hMqa.js} | 4 +- .../{logs-B-DEpUTT.js => logs-DDLW4tx6.js} | 2 +- ...-Cnh7lvB1.js => mcp-app-frame-HVkgIOHp.js} | 2 +- ...-CbhBEf5M.js => new-dashboard-DM0vLVU_.js} | 2 +- .../{page-_NPyEsnO.js => page-CxQVwyyf.js} | 2 +- .../{page-CKTCCvtZ.js => page-DRMCV_oz.js} | 6 +- ...r-Bhy4dqnA.js => panel-editor-I63uZI5h.js} | 2 +- ...og-CXtJqeqG.js => save-dialog-DWI-UzcG.js} | 2 +- ...ge-C2_0ObTE.js => search-page-Q8y34j9f.js} | 2 +- ...tings-BuNHdI4Y.js => settings-DUGc15Wj.js} | 2 +- ...oolbar-D0a2hqiz.js => toolbar-DeZmbzhH.js} | 2 +- ...{traces-HMliS1WN.js => traces-B9ZppPb7.js} | 2 +- ...on-BbyzGftf.js => useMutation-DPafCKuL.js} | 2 +- internal/ui/dist/index.html | 2 +- site/src/content/docs/docs/guides/alerts.mdx | 87 ++- site/src/content/docs/docs/reference/api.mdx | 1 + .../docs/reference/configuration/alerts.mdx | 15 +- ui/host/src/alerts/api.ts | 9 +- ui/host/src/alerts/email.test.tsx | 136 +++++ ui/host/src/alerts/page.tsx | 13 +- ui/host/src/alerts/rule-form.tsx | 29 +- 55 files changed, 2485 insertions(+), 116 deletions(-) create mode 100644 internal/alert/email.go create mode 100644 internal/alert/email_test.go create mode 100644 internal/api/alerts_email_test.go create mode 100644 internal/auth/alert_mail.go create mode 100644 internal/auth/alert_mail_test.go create mode 100644 internal/db/generated/alert_email.sql.go create mode 100644 internal/db/migrations/20261108120000_alert_email.sql create mode 100644 internal/db/queries/alert_email.sql create mode 100644 internal/ui/dist/assets/alerts-DD6_LYeu.js delete mode 100644 internal/ui/dist/assets/alerts-DH7dglkG.js rename internal/ui/dist/assets/{chat-C9Lgsz9A.js => chat-S7LVuufB.js} (99%) create mode 100644 internal/ui/dist/assets/chat._threadId-CWjhRiA-.js delete mode 100644 internal/ui/dist/assets/chat._threadId-_q5tLfzU.js create mode 100644 internal/ui/dist/assets/chat.index-CWjhRiA-.js delete mode 100644 internal/ui/dist/assets/chat.index-_q5tLfzU.js rename internal/ui/dist/assets/{dashboards._dashboardId-DSPyJ_4u.js => dashboards._dashboardId-CHorvktr.js} (75%) rename internal/ui/dist/assets/{dashboards.index-BKP8lnwh.js => dashboards.index-OaN8mvAC.js} (70%) rename internal/ui/dist/assets/{explore-qAMgIiPy.js => explore-By3WiS-G.js} (71%) rename internal/ui/dist/assets/{index-DEaVwvoN.js => index-CO28hMqa.js} (99%) rename internal/ui/dist/assets/{logs-B-DEpUTT.js => logs-DDLW4tx6.js} (73%) rename internal/ui/dist/assets/{mcp-app-frame-Cnh7lvB1.js => mcp-app-frame-HVkgIOHp.js} (99%) rename internal/ui/dist/assets/{new-dashboard-CbhBEf5M.js => new-dashboard-DM0vLVU_.js} (95%) rename internal/ui/dist/assets/{page-_NPyEsnO.js => page-CxQVwyyf.js} (99%) rename internal/ui/dist/assets/{page-CKTCCvtZ.js => page-DRMCV_oz.js} (99%) rename internal/ui/dist/assets/{panel-editor-Bhy4dqnA.js => panel-editor-I63uZI5h.js} (96%) rename internal/ui/dist/assets/{save-dialog-CXtJqeqG.js => save-dialog-DWI-UzcG.js} (99%) rename internal/ui/dist/assets/{search-page-C2_0ObTE.js => search-page-Q8y34j9f.js} (98%) rename internal/ui/dist/assets/{settings-BuNHdI4Y.js => settings-DUGc15Wj.js} (99%) rename internal/ui/dist/assets/{toolbar-D0a2hqiz.js => toolbar-DeZmbzhH.js} (99%) rename internal/ui/dist/assets/{traces-HMliS1WN.js => traces-B9ZppPb7.js} (78%) rename internal/ui/dist/assets/{useMutation-BbyzGftf.js => useMutation-DPafCKuL.js} (97%) create mode 100644 ui/host/src/alerts/email.test.tsx diff --git a/.env.example b/.env.example index 2fbe451d8..ed695310e 100644 --- a/.env.example +++ b/.env.example @@ -34,6 +34,16 @@ # Default: (empty) # FANOUT_AI_PROVIDER= +# FANOUT_ALERTS_EMAIL_DEFAULT_TO; string +# A comma-separated list of addresses that receive alerts from rules with no webhook and no email recipients of their own; requires SMTP. +# Default: (empty) +# FANOUT_ALERTS_EMAIL_DEFAULT_TO= + +# FANOUT_ALERTS_EMAIL_HOURLY_LIMIT; integer +# Caps the alert emails sent to one address in any rolling hour, across all rules. Notifications over the limit are suppressed, not delayed, and shown on the rule; 0 suppresses all alert email. +# Default: 30 +# FANOUT_ALERTS_EMAIL_HOURLY_LIMIT= + # FANOUT_ALERTS_ENABLED; boolean # Runs the alert evaluation engine. # Default: true diff --git a/cmd/fanout-docgen/main.go b/cmd/fanout-docgen/main.go index 8154adc0c..5ca60d46c 100644 --- a/cmd/fanout-docgen/main.go +++ b/cmd/fanout-docgen/main.go @@ -500,9 +500,9 @@ var groups = map[string]groupPage{ }, "alerts": { Title: "Alert settings", - Description: "Whether the alert engine runs, how often it evaluates, and how long history is kept.", - Summary: "Settings for the alert evaluation loop.", - ReadWhen: []string{"Alerts are evaluating too slowly, or not at all."}, + Description: "Whether the alert engine runs, how often it evaluates, how long history is kept, and how alert email is addressed and limited.", + Summary: "Settings for the alert evaluation loop and alert email.", + ReadWhen: []string{"Alerts are evaluating too slowly, or not at all.", "You are setting a default alert email destination or changing the per-address limit."}, }, "metrics": { Title: "Metrics settings", diff --git a/cmd/fanout/main.go b/cmd/fanout/main.go index 41fd3fff0..e9998e0f0 100644 --- a/cmd/fanout/main.go +++ b/cmd/fanout/main.go @@ -197,6 +197,14 @@ func main() { os.Exit(1) } + smtpCfg := auth.SMTPConfig{ + Host: cfg.SMTPHost, + Port: cfg.SMTPPort, + User: cfg.SMTPUser, + Pass: cfg.SMTPPass, + From: cfg.SMTPFrom, + } + // Start alert engine var alertStore *alert.Store var alertEngine *alert.Engine @@ -207,6 +215,18 @@ func main() { cfg.AlertEvaluationInterval, cfg.AlertHistoryDays, ) + if cfg.SMTPConfigured() { + defaults, err := cfg.AlertEmailDefaults() + if err != nil { + slog.Error("alert email defaults invalid", "err", err) + os.Exit(1) + } + alertEngine.EnableEmail(alert.EmailConfig{ + Send: func(ctx context.Context, mail auth.AlertMail) error { return auth.SendAlert(ctx, smtpCfg, mail) }, + Defaults: defaults, + HourlyLimit: cfg.AlertEmailHourlyLimit, + }) + } go alertEngine.Run(ctx) slog.Info("alert engine enabled", "interval", cfg.AlertEvaluationInterval) } @@ -326,13 +346,6 @@ func main() { api.RegisterAlertRoutes(e, alertStore, alertEngine) // Auth routes (web-only setup + email-code login) - smtpCfg := auth.SMTPConfig{ - Host: cfg.SMTPHost, - Port: cfg.SMTPPort, - User: cfg.SMTPUser, - Pass: cfg.SMTPPass, - From: cfg.SMTPFrom, - } codeStore := auth.NewCodeStore(sqlite.DB, cfg.AuthCodeSecret) api.RegisterAuthRoutes(e, userStore, codeStore, setup, settingsStore, browserSessions, auditStore, smtpCfg, cfg) oidcDiscovery, err := api.RegisterOIDCRoutes(ctx, e, cfg, userStore, identityStore, browserSessions, auditStore) diff --git a/internal/alert/email.go b/internal/alert/email.go new file mode 100644 index 000000000..540effd0c --- /dev/null +++ b/internal/alert/email.go @@ -0,0 +1,462 @@ +package alert + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "log/slog" + "runtime/debug" + "strings" + "time" + + "github.com/labstack/fanout/internal/auth" + "github.com/labstack/fanout/internal/db/generated" +) + +// Rule email statuses, reported on Rule.EmailStatus. Delivery rows use sent, +// failed and suppressed as their terminal states. +const ( + emailSent = "sent" + emailRetrying = "retrying" + emailFailed = "failed" + emailSuppressed = "suppressed" + emailUnavailable = "unavailable" + + emailPending = "pending" + emailSending = "sending" +) + +// EmailUnavailableMessage explains why email cannot be chosen or delivered. +const EmailUnavailableMessage = "Email delivery is unavailable because SMTP is not configured. Set FANOUT_SMTP_HOST, FANOUT_SMTP_USERNAME, FANOUT_SMTP_PASSWORD and FANOUT_SMTP_FROM." + +const ( + emailBatchSize = 50 + emailPollInterval = 2 * time.Second + emailSendTimeout = 30 * time.Second + emailLimitWindow = time.Hour + maxEmailDetail = 300 +) + +// emailBackoff holds the wait before each attempt. Mail servers defer delivery +// for minutes at a time (greylisting, brief outages), so it runs longer than +// the webhook's in-memory 2s and 4s retries. +var emailBackoff = []time.Duration{0, time.Minute, 5 * time.Minute, 15 * time.Minute, 30 * time.Minute} + +// EmailConfig enables alert email on an Engine. +type EmailConfig struct { + Send func(context.Context, auth.AlertMail) error + Defaults []string // Addresses for rules with no destination of their own. + HourlyLimit int // Emails per address per rolling hour, across all rules. +} + +// EmailOptions describes alert email on this instance to API clients. +type EmailOptions struct { + Available bool `json:"available"` + Message string `json:"message,omitempty"` + DefaultDestination bool `json:"default_destination"` + MaxRecipients int `json:"max_recipients"` + HourlyLimit int `json:"hourly_limit,omitempty"` +} + +// EnableEmail turns on alert email. Call it before Run. +func (e *Engine) EnableEmail(cfg EmailConfig) { + e.email = &cfg +} + +// EmailOptions reports whether email can be chosen. A nil engine means alerts +// are disabled. +func (e *Engine) EmailOptions() EmailOptions { + options := EmailOptions{MaxRecipients: auth.MaxAlertEmailRecipients} + if e == nil { + options.Message = "Alert delivery is unavailable because alerts are disabled on this instance." + return options + } + if e.email == nil { + options.Message = EmailUnavailableMessage + return options + } + options.Available = true + options.DefaultDestination = len(e.email.Defaults) > 0 + options.HourlyLimit = e.email.HourlyLimit + return options +} + +// WithEmailState adjusts a rule's reported email state for what this instance +// can do now: saved recipients without SMTP are a standing failure, and a rule +// with no email destination has no email state. +func (e *Engine) WithEmailState(rule Rule) Rule { + if e == nil || e.email == nil { + rule.EmailStatus, rule.EmailStatusDetail, rule.EmailStatusAt = "", "", "" + if len(rule.EmailRecipients) > 0 { + rule.EmailStatus = emailUnavailable + rule.EmailStatusDetail = e.EmailOptions().Message + } + return rule + } + if len(e.emailRecipients(rule)) == 0 { + rule.EmailStatus, rule.EmailStatusDetail, rule.EmailStatusAt = "", "", "" + } + return rule +} + +// emailRecipients resolves where a rule's notifications are mailed. The default +// destination applies only to a rule with neither a webhook nor recipients. +func (e *Engine) emailRecipients(rule Rule) []string { + if e.email == nil { + return nil + } + if len(rule.EmailRecipients) > 0 { + return rule.EmailRecipients + } + if strings.TrimSpace(rule.WebhookURL) == "" { + return e.email.Defaults + } + return nil +} + +// saveAlert persists a transition. A non-empty event also queues its email in +// the same transaction, so a crash cannot keep one without the other. +func (e *Engine) saveAlert(rule Rule, a Alert, event string) (Alert, error) { + recipients := e.emailRecipients(rule) + if event == "" || len(recipients) == 0 { + return e.store.UpsertAlert(a) + } + now := e.clock() + return e.store.UpsertAlertWithEmail(a, recipients, emailContent{ + Rule: rule.Name, + Description: rule.Description, + Service: a.Service, + Event: event, + Value: a.Value, + Time: now, + }, now) +} + +func (e *Engine) clock() time.Time { + if e.now != nil { + return e.now() + } + return time.Now() +} + +// emailContent is what a delivery row keeps to render its message. +type emailContent struct { + Rule string `json:"rule"` + Description string `json:"description,omitempty"` + Service string `json:"service"` + Event string `json:"event"` + Value float64 `json:"value"` + Time time.Time `json:"time"` +} + +// UpsertAlertWithEmail saves an alert and queues one delivery per recipient, +// atomically. +func (s *Store) UpsertAlertWithEmail(a Alert, recipients []string, content emailContent, now time.Time) (Alert, error) { + ctx := context.Background() + payload, err := json.Marshal(content) + if err != nil { + return Alert{}, fmt.Errorf("store: encode email: %w", err) + } + notificationID, err := newID() + if err != nil { + return Alert{}, err + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return Alert{}, fmt.Errorf("store: begin: %w", err) + } + defer func() { _ = tx.Rollback() }() + q := s.q.WithTx(tx) + saved, err := upsertAlert(q, a) + if err != nil { + return Alert{}, err + } + at := formatStoredTime(now) + for _, recipient := range recipients { + id, err := newID() + if err != nil { + return Alert{}, err + } + if err := q.InsertAlertEmailDelivery(ctx, generated.InsertAlertEmailDeliveryParams{ + ID: id, + NotificationID: notificationID, + RuleID: saved.RuleID, + AlertID: saved.ID, + Recipient: recipient, + Content: string(payload), + NextAttemptAt: at, + CreatedAt: at, + }); err != nil { + return Alert{}, fmt.Errorf("store: queue email: %w", err) + } + } + if err := tx.Commit(); err != nil { + return Alert{}, fmt.Errorf("store: commit: %w", err) + } + return saved, nil +} + +func encodeRecipients(recipients []string) sql.NullString { + if len(recipients) == 0 { + return sql.NullString{} + } + encoded, err := json.Marshal(recipients) + if err != nil { + return sql.NullString{} + } + return sql.NullString{String: string(encoded), Valid: true} +} + +func decodeRecipients(ruleID string, stored sql.NullString) []string { + if !stored.Valid || stored.String == "" { + return nil + } + var recipients []string + if err := json.Unmarshal([]byte(stored.String), &recipients); err != nil { + slog.Error("alert: invalid stored email recipients", "rule", ruleID, "err", err) + return nil + } + return recipients +} + +// runEmail delivers queued mail until ctx ends. One worker sends sequentially, +// which keeps the per-recipient count exact. +func (e *Engine) runEmail(ctx context.Context) { + defer func() { + if r := recover(); r != nil { + slog.Error("alert: email worker panic", "panic", r, "stack", string(debug.Stack())) + } + }() + e.recoverEmail(context.WithoutCancel(ctx)) + ticker := time.NewTicker(emailPollInterval) + defer ticker.Stop() + for { + e.drainEmail(ctx) + select { + case <-ticker.C: + case <-ctx.Done(): + return + } + } +} + +// recoverEmail resumes after a restart. A row still marked sending may or may +// not have reached the server, so it is sent again: delivery is at least once, +// and the repeated Message-ID lets a mail system collapse the duplicate. +func (e *Engine) recoverEmail(ctx context.Context) { + rows, err := e.store.q.ListOpenAlertEmailDeliveries(ctx) + if err != nil { + slog.Error("alert: list open email", "err", err) + return + } + now := formatStoredTime(e.clock()) + for _, row := range rows { + if row.Status != emailSending { + continue + } + if int(row.Attempts) >= len(emailBackoff) { + e.finishEmail(ctx, row, emailFailed, "Interrupted during its last attempt", "", row.NextAttemptAt) + e.settleEmail(ctx, row) + continue + } + e.finishEmail(ctx, row, emailPending, row.LastError, "", now) + } +} + +// failUndeliverableEmail settles mail left queued by a run that had SMTP, so it +// shows as failed instead of waiting forever or going out long after the fact. +func (e *Engine) failUndeliverableEmail(ctx context.Context) { + rows, err := e.store.q.ListOpenAlertEmailDeliveries(ctx) + if err != nil { + slog.Error("alert: list open email", "err", err) + return + } + for _, row := range rows { + e.finishEmail(ctx, row, emailFailed, EmailUnavailableMessage, "", row.NextAttemptAt) + e.settleEmail(ctx, row) + } +} + +// drainEmail processes everything currently due, oldest first. +func (e *Engine) drainEmail(ctx context.Context) { + for ctx.Err() == nil { + rows, err := e.store.q.ListDueAlertEmailDeliveries(ctx, generated.ListDueAlertEmailDeliveriesParams{ + NextAttemptAt: formatStoredTime(e.clock()), + Limit: emailBatchSize, + }) + if err != nil { + if ctx.Err() == nil { + slog.Error("alert: list due email", "err", err) + } + return + } + if len(rows) == 0 { + return + } + for _, row := range rows { + if ctx.Err() != nil { + return + } + e.deliverEmail(ctx, row) + } + } +} + +// deliverEmail makes one attempt, or suppresses the row if its recipient has +// reached the hourly limit. Suppressed mail is dropped rather than delayed, so +// a burst cannot arrive later as a backlog. +func (e *Engine) deliverEmail(ctx context.Context, row generated.AlertEmailDelivery) { + now := e.clock() + sent, err := e.store.q.CountAlertEmailsSentSince(ctx, generated.CountAlertEmailsSentSinceParams{ + Recipient: row.Recipient, + SentAt: formatStoredTime(now.Add(-emailLimitWindow)), + }) + if err != nil { + slog.Error("alert: count sent email", "err", err) + return + } + limit := e.email.HourlyLimit + if int(sent) >= limit { + slog.Warn("alert: email suppressed by hourly limit", "rule", row.RuleID, "alert", row.AlertID, "limit", limit) + e.finishEmail(ctx, row, emailSuppressed, fmt.Sprintf("Over the limit of %d emails per hour to this address", limit), "", row.NextAttemptAt) + e.settleEmail(ctx, row) + return + } + + var content emailContent + if err := json.Unmarshal([]byte(row.Content), &content); err != nil { + e.finishEmail(ctx, row, emailFailed, "The stored message could not be read", "", row.NextAttemptAt) + e.settleEmail(ctx, row) + return + } + if err := e.store.q.ClaimAlertEmailDelivery(ctx, row.ID); err != nil { + slog.Error("alert: claim email", "err", err) + return + } + attempt := int(row.Attempts) + 1 + + mail := auth.AlertMail{ + ID: row.ID, + To: row.Recipient, + Rule: content.Rule, + Description: content.Description, + Service: content.Service, + Event: content.Event, + Value: content.Value, + Time: content.Time, + } + if int(sent) == limit-1 { + mail.LimitNotice = fmt.Sprintf("This address has reached the limit of %d alert emails per hour. Further alerts are recorded on their rules in Fanout and not emailed until the count falls.", limit) + } + sendCtx, cancel := context.WithTimeout(ctx, emailSendTimeout) + err = e.email.Send(sendCtx, mail) + cancel() + + switch { + case err == nil: + e.finishEmail(ctx, row, emailSent, "", formatStoredTime(e.clock()), row.NextAttemptAt) + e.settleEmail(ctx, row) + case ctx.Err() != nil: + // Shutting down: leave the row marked sending for recoverEmail. + case auth.IsPermanentMailError(err) || attempt >= len(emailBackoff): + slog.Warn("alert: email delivery failed", "rule", row.RuleID, "alert", row.AlertID, "attempt", attempt, "err", err) + e.finishEmail(ctx, row, emailFailed, emailErrorDetail(err), "", row.NextAttemptAt) + e.settleEmail(ctx, row) + default: + next := e.clock().Add(emailBackoff[attempt]) + slog.Warn("alert: email attempt failed, will retry", "rule", row.RuleID, "alert", row.AlertID, "attempt", attempt, "err", err) + e.finishEmail(ctx, row, emailPending, emailErrorDetail(err), "", formatStoredTime(next)) + e.setRuleEmailStatus(ctx, row.RuleID, emailRetrying, fmt.Sprintf("Attempt %d of %d failed: %s", attempt, len(emailBackoff), emailErrorDetail(err))) + } +} + +func emailErrorDetail(err error) string { + detail := []rune(strings.ToValidUTF8(err.Error(), " ")) + if len(detail) > maxEmailDetail { + detail = detail[:maxEmailDetail] + } + return string(detail) +} + +func (e *Engine) finishEmail(ctx context.Context, row generated.AlertEmailDelivery, status, detail, sentAt, nextAttemptAt string) { + if err := e.store.q.FinishAlertEmailDelivery(ctx, generated.FinishAlertEmailDeliveryParams{ + Status: status, + LastError: detail, + SentAt: sentAt, + NextAttemptAt: nextAttemptAt, + ID: row.ID, + }); err != nil { + slog.Error("alert: update email delivery", "rule", row.RuleID, "status", status, "err", err) + } +} + +// settleEmail publishes a notification's result once every recipient is +// finished: failure outranks suppression, and suppression outranks success. A +// rule without a webhook also reports it on the alert, where the webhook +// result would otherwise go. +func (e *Engine) settleEmail(ctx context.Context, row generated.AlertEmailDelivery) { + rows, err := e.store.q.ListAlertEmailDeliveriesByNotification(ctx, row.NotificationID) + if err != nil { + slog.Error("alert: read notification", "rule", row.RuleID, "err", err) + return + } + var failed, suppressed int + var cause string + for _, r := range rows { + switch r.Status { + case emailPending, emailSending: + return + case emailFailed: + failed++ + if cause == "" { + cause = r.LastError + } + case emailSuppressed: + suppressed++ + if cause == "" { + cause = r.LastError + } + } + } + + status, alertStatus, detail := emailSent, "success", "" + switch { + case failed > 0: + status, alertStatus = emailFailed, "failed" + detail = fmt.Sprintf("%d of %d recipients failed: %s", failed, len(rows), cause) + case suppressed > 0: + status, alertStatus = emailSuppressed, "suppressed" + detail = fmt.Sprintf("%d of %d recipients suppressed: %s", suppressed, len(rows), cause) + } + e.setRuleEmailStatus(ctx, row.RuleID, status, detail) + + rule, err := e.store.GetRule(row.RuleID) + if err != nil { + slog.Error("alert: read rule for email status", "rule", row.RuleID, "err", err) + return + } + if strings.TrimSpace(rule.WebhookURL) == "" { + if err := e.store.UpdateDeliveryStatus(row.AlertID, alertStatus, e.clock().Format(time.RFC3339)); err != nil { + slog.Error("alert: update delivery status", "rule", row.RuleID, "err", err) + } + } +} + +func (e *Engine) setRuleEmailStatus(ctx context.Context, ruleID, status, detail string) { + if err := e.store.q.UpdateRuleEmailStatus(ctx, generated.UpdateRuleEmailStatusParams{ + EmailStatus: status, + EmailStatusDetail: detail, + EmailStatusAt: e.clock().Format(time.RFC3339), + ID: ruleID, + }); err != nil && !errors.Is(err, sql.ErrNoRows) { + slog.Error("alert: update rule email status", "rule", ruleID, "err", err) + } +} + +// pruneEmailDeliveries removes finished rows older than the history window. +func (s *Store) pruneEmailDeliveries(days int, now time.Time) error { + _, err := s.q.PruneAlertEmailDeliveries(context.Background(), formatStoredTime(now.AddDate(0, 0, -days))) + return err +} diff --git a/internal/alert/email_test.go b/internal/alert/email_test.go new file mode 100644 index 000000000..37a1f5e13 --- /dev/null +++ b/internal/alert/email_test.go @@ -0,0 +1,511 @@ +package alert + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + mail "github.com/wneessen/go-mail" + + "github.com/labstack/fanout/internal/auth" +) + +// outbox records sends and fails them on demand. +type outbox struct { + mu sync.Mutex + mails []auth.AlertMail + fail func(auth.AlertMail) error +} + +func (o *outbox) send(_ context.Context, m auth.AlertMail) error { + o.mu.Lock() + defer o.mu.Unlock() + if o.fail != nil { + if err := o.fail(m); err != nil { + return err + } + } + o.mails = append(o.mails, m) + return nil +} + +func (o *outbox) to(address string) []auth.AlertMail { + o.mu.Lock() + defer o.mu.Unlock() + var matched []auth.AlertMail + for _, m := range o.mails { + if m.To == address { + matched = append(matched, m) + } + } + return matched +} + +type mailHarness struct { + engine *Engine + store *Store + box *outbox + now time.Time +} + +func newMailHarness(t *testing.T, limit int, defaults ...string) *mailHarness { + t.Helper() + engine, store := newTestEngine(t) + h := &mailHarness{engine: engine, store: store, box: &outbox{}, now: time.Now().UTC()} + engine.now = func() time.Time { return h.now } + engine.EnableEmail(EmailConfig{Send: h.box.send, Defaults: defaults, HourlyLimit: limit}) + return h +} + +func (h *mailHarness) rule(t *testing.T, name string, recipients ...string) Rule { + t.Helper() + rule, err := h.store.CreateRule(Rule{Name: name, Enabled: true, Service: "api", Expression: "error_rate > 0.1", RepeatIntervalS: 3600, EmailRecipients: recipients}) + if err != nil { + t.Fatal(err) + } + return rule +} + +func (h *mailHarness) fire(rule Rule) { + h.engine.transition(rule, "api", true, Env{Service: "api", ErrorRate: 0.5}, h.now) +} + +func (h *mailHarness) drain() { h.engine.drainEmail(context.Background()) } + +func (h *mailHarness) ruleStatus(t *testing.T, id string) Rule { + t.Helper() + rule, err := h.store.GetRule(id) + if err != nil { + t.Fatal(err) + } + return rule +} + +func TestEmailReachesEveryRecipientAndReportsSent(t *testing.T) { + h := newMailHarness(t, 30) + rule := h.rule(t, "errors", "a@example.com", "b@example.com") + h.fire(rule) + h.drain() + if len(h.box.to("a@example.com")) != 1 || len(h.box.to("b@example.com")) != 1 { + t.Fatalf("mails = %+v", h.box.mails) + } + got := h.box.to("a@example.com")[0] + if got.Rule != "errors" || got.Service != "api" || got.Event != "firing" || got.Value != 0.5 { + t.Fatalf("mail = %+v", got) + } + if status := h.ruleStatus(t, rule.ID).EmailStatus; status != "sent" { + t.Fatalf("rule status = %q", status) + } + alert, err := h.store.GetAlert(rule.ID, "api") + if err != nil || alert.LastDeliveryStatus != "success" { + t.Fatalf("alert = %+v, %v", alert, err) + } +} + +func TestDefaultDestinationAppliesOnlyToRulesWithoutOne(t *testing.T) { + h := newMailHarness(t, 30, "oncall@example.com") + bare := h.rule(t, "bare") + own := h.rule(t, "own", "owner@example.com") + webhook, err := h.store.CreateRule(Rule{Name: "hook", Enabled: true, Service: "api", Expression: "true", WebhookURL: "http://127.0.0.1:1/hook"}) + if err != nil { + t.Fatal(err) + } + for _, rule := range []Rule{bare, own, webhook} { + h.fire(rule) + } + h.drain() + if got := h.box.to("oncall@example.com"); len(got) != 1 || got[0].Rule != "bare" { + t.Fatalf("default destination got %+v", got) + } + if got := h.box.to("owner@example.com"); len(got) != 1 { + t.Fatalf("own recipient got %+v", got) + } +} + +// A rule row from before this feature has NULL recipients and must still deliver. +func TestOlderRuleWithoutEmailColumnsUsesDefaultDestination(t *testing.T) { + h := newMailHarness(t, 30, "oncall@example.com") + if _, err := h.store.db.Exec(`INSERT INTO alert_rules (id, name, expression, for_seconds) VALUES ('old', 'old rule', 'true', 0)`); err != nil { + t.Fatal(err) + } + rule := h.ruleStatus(t, "old") + if rule.EmailRecipients != nil || rule.EmailStatus != "" { + t.Fatalf("older rule = %+v", rule) + } + h.engine.transition(rule, "api", true, Env{Service: "api"}, h.now) + h.drain() + if got := h.box.to("oncall@example.com"); len(got) != 1 { + t.Fatalf("older rule mails = %+v", got) + } +} + +func TestEmailRetriesWithBackoffWithoutResendingToDelivered(t *testing.T) { + h := newMailHarness(t, 30) + rule := h.rule(t, "errors", "ok@example.com", "flaky@example.com") + failing := true + h.box.fail = func(m auth.AlertMail) error { + if m.To == "flaky@example.com" && failing { + return errors.New("451 try later") + } + return nil + } + h.fire(rule) + h.drain() + stored := h.ruleStatus(t, rule.ID) + if stored.EmailStatus != "retrying" || !strings.Contains(stored.EmailStatusDetail, "451") { + t.Fatalf("rule = %+v", stored) + } + + h.now = h.now.Add(30 * time.Second) + h.drain() + if len(h.box.to("flaky@example.com")) != 0 { + t.Fatal("retried before the backoff elapsed") + } + + failing = false + h.now = h.now.Add(31 * time.Second) + h.drain() + if len(h.box.to("flaky@example.com")) != 1 || len(h.box.to("ok@example.com")) != 1 { + t.Fatalf("mails = %+v", h.box.mails) + } + if status := h.ruleStatus(t, rule.ID).EmailStatus; status != "sent" { + t.Fatalf("rule status = %q", status) + } +} + +func TestExhaustedRetriesShowFailureOnRule(t *testing.T) { + h := newMailHarness(t, 30) + rule := h.rule(t, "errors", "a@example.com") + attempts := 0 + h.box.fail = func(auth.AlertMail) error { + attempts++ + return errors.New("connection refused") + } + h.fire(rule) + for range len(emailBackoff) + 2 { + h.drain() + h.now = h.now.Add(time.Hour) + } + if attempts != len(emailBackoff) { + t.Fatalf("attempts = %d, want %d", attempts, len(emailBackoff)) + } + stored := h.ruleStatus(t, rule.ID) + if stored.EmailStatus != "failed" || !strings.Contains(stored.EmailStatusDetail, "connection refused") { + t.Fatalf("rule = %+v", stored) + } + alert, err := h.store.GetAlert(rule.ID, "api") + if err != nil || alert.LastDeliveryStatus != "failed" { + t.Fatalf("alert = %+v, %v", alert, err) + } +} + +func TestPermanentRejectionIsNotRetried(t *testing.T) { + h := newMailHarness(t, 30) + rule := h.rule(t, "errors", "gone@example.com") + attempts := 0 + h.box.fail = func(auth.AlertMail) error { + attempts++ + return &mail.SendError{Reason: mail.ErrSMTPRcptTo} + } + h.fire(rule) + h.drain() + h.now = h.now.Add(time.Hour) + h.drain() + if attempts != 1 { + t.Fatalf("attempts = %d, want 1", attempts) + } + if status := h.ruleStatus(t, rule.ID).EmailStatus; status != "failed" { + t.Fatalf("rule status = %q", status) + } +} + +func TestBurstAcrossRulesIsBoundedPerRecipientAndSuppressionIsVisible(t *testing.T) { + h := newMailHarness(t, 3) + var rules []Rule + for i := range 20 { + rules = append(rules, h.rule(t, fmt.Sprintf("rule-%d", i), "pager@example.com")) + } + quiet := h.rule(t, "quiet", "other@example.com") + for _, rule := range rules { + h.fire(rule) + h.now = h.now.Add(time.Millisecond) + } + h.fire(quiet) + h.now = h.now.Add(time.Millisecond) + h.drain() + + delivered := h.box.to("pager@example.com") + if len(delivered) != 3 { + t.Fatalf("delivered = %d, want 3", len(delivered)) + } + if delivered[1].LimitNotice != "" || !strings.Contains(delivered[2].LimitNotice, "limit of 3") { + t.Fatalf("limit notices = %q, %q", delivered[1].LimitNotice, delivered[2].LimitNotice) + } + if len(h.box.to("other@example.com")) != 1 { + t.Fatal("another recipient was affected by the limit") + } + suppressed := 0 + for _, rule := range rules { + stored := h.ruleStatus(t, rule.ID) + if stored.EmailStatus == "suppressed" { + suppressed++ + if !strings.Contains(stored.EmailStatusDetail, "limit of 3") { + t.Fatalf("detail = %q", stored.EmailStatusDetail) + } + } + } + if suppressed != 17 { + t.Fatalf("suppressed rules = %d, want 17", suppressed) + } + + // Suppressed mail never arrives later, and the window reopens. + h.now = h.now.Add(2 * time.Hour) + h.drain() + if got := len(h.box.to("pager@example.com")); got != 3 { + t.Fatalf("suppressed mail delivered later: %d", got) + } + late := h.rule(t, "late", "pager@example.com") + h.fire(late) + h.drain() + if got := len(h.box.to("pager@example.com")); got != 4 { + t.Fatalf("after window delivered = %d, want 4", got) + } +} + +func TestLimitSurvivesRestart(t *testing.T) { + h := newMailHarness(t, 1) + h.fire(h.rule(t, "one", "a@example.com")) + h.drain() + + restarted := &Engine{store: h.store, programs: map[string]Program{}, stopped: map[string]bool{}, now: h.engine.now} + box := &outbox{} + restarted.EnableEmail(EmailConfig{Send: box.send, HourlyLimit: 1}) + rule := h.rule(t, "two", "a@example.com") + restarted.transition(rule, "api", true, Env{Service: "api"}, h.now) + restarted.drainEmail(context.Background()) + if len(box.mails) != 0 { + t.Fatalf("limit reset by restart: %+v", box.mails) + } +} + +func TestRestartResumesQueuedMailAndDoesNotResendFinished(t *testing.T) { + h := newMailHarness(t, 30) + done := h.rule(t, "done", "done@example.com") + h.fire(done) + h.drain() + + interrupted := h.rule(t, "interrupted", "cut@example.com") + h.fire(interrupted) + if _, err := h.store.db.Exec(`UPDATE alert_email_deliveries SET status='sending', attempts=1 WHERE recipient='cut@example.com'`); err != nil { + t.Fatal(err) + } + queued := h.rule(t, "queued", "wait@example.com") + h.fire(queued) + + restarted := &Engine{store: h.store, programs: map[string]Program{}, stopped: map[string]bool{}, now: h.engine.now} + box := &outbox{} + restarted.EnableEmail(EmailConfig{Send: box.send, HourlyLimit: 30}) + restarted.recoverEmail(context.Background()) + restarted.drainEmail(context.Background()) + + if len(box.to("done@example.com")) != 0 { + t.Fatal("finished mail was resent") + } + if len(box.to("cut@example.com")) != 1 || len(box.to("wait@example.com")) != 1 { + t.Fatalf("mails = %+v", box.mails) + } + restarted.drainEmail(context.Background()) + if len(box.mails) != 2 { + t.Fatalf("resent on a later drain: %+v", box.mails) + } +} + +func TestResentMailKeepsItsMessageIdentity(t *testing.T) { + h := newMailHarness(t, 30) + rule := h.rule(t, "errors", "a@example.com") + var ids []string + h.box.fail = func(m auth.AlertMail) error { + ids = append(ids, m.ID) + return errors.New("timeout after DATA") + } + h.fire(rule) + h.drain() + h.now = h.now.Add(2 * time.Minute) + h.drain() + if len(ids) != 2 || ids[0] != ids[1] { + t.Fatalf("ids = %v", ids) + } +} + +func TestInterruptedLastAttemptFailsInsteadOfSendingAgain(t *testing.T) { + h := newMailHarness(t, 30) + rule := h.rule(t, "errors", "a@example.com") + h.fire(rule) + if _, err := h.store.db.Exec(`UPDATE alert_email_deliveries SET status='sending', attempts=?`, len(emailBackoff)); err != nil { + t.Fatal(err) + } + h.engine.recoverEmail(context.Background()) + h.drain() + if len(h.box.mails) != 0 { + t.Fatalf("mails = %+v", h.box.mails) + } + if status := h.ruleStatus(t, rule.ID).EmailStatus; status != "failed" { + t.Fatalf("rule status = %q", status) + } +} + +func TestEmailQueuedAtomicallyWithTransition(t *testing.T) { + h := newMailHarness(t, 30) + rule := h.rule(t, "errors", "a@example.com") + if _, err := h.store.db.Exec(`CREATE TRIGGER block_mail BEFORE INSERT ON alert_email_deliveries BEGIN SELECT RAISE(ABORT, 'blocked'); END`); err != nil { + t.Fatal(err) + } + h.fire(rule) + if _, err := h.store.GetAlert(rule.ID, "api"); !errors.Is(err, ErrNotFound) { + t.Fatalf("alert saved without its email: %v", err) + } +} + +func TestRepeatsFollowTheRulesNotificationBehaviour(t *testing.T) { + h := newMailHarness(t, 30) + rule, err := h.store.CreateRule(Rule{Name: "errors", Enabled: true, Service: "api", Expression: "true", RepeatIntervalS: 600, EmailRecipients: []string{"a@example.com"}}) + if err != nil { + t.Fatal(err) + } + h.fire(rule) + h.drain() + h.now = h.now.Add(5 * time.Minute) + h.fire(rule) + h.drain() + if got := h.box.to("a@example.com"); len(got) != 1 { + t.Fatalf("repeated inside the interval: %+v", got) + } + h.now = h.now.Add(6 * time.Minute) + h.fire(rule) + h.drain() + got := h.box.to("a@example.com") + if len(got) != 2 || got[1].Event != "reminder" { + t.Fatalf("reminder mails = %+v", got) + } + + // notify_on_resolve is off, so the resolve is silent. + h.now = h.now.Add(time.Minute) + h.engine.transition(rule, "api", false, Env{Service: "api"}, h.now) + h.drain() + if got := h.box.to("a@example.com"); len(got) != 2 { + t.Fatalf("resolve mailed with notify_on_resolve off: %+v", got) + } + + rule.NotifyOnResolve = true + h.now = h.now.Add(time.Minute) + h.fire(rule) + h.now = h.now.Add(time.Minute) + h.engine.transition(rule, "api", false, Env{Service: "api"}, h.now) + h.drain() + got = h.box.to("a@example.com") + if len(got) != 4 || got[3].Event != "resolved" { + t.Fatalf("resolve mails = %+v", got) + } +} + +func TestDisablingAFiringRuleMailsItsResolve(t *testing.T) { + h := newMailHarness(t, 30) + rule, err := h.store.CreateRule(Rule{Name: "errors", Enabled: true, Service: "api", Expression: "true", NotifyOnResolve: true, EmailRecipients: []string{"a@example.com"}}) + if err != nil { + t.Fatal(err) + } + h.fire(rule) + h.engine.DisableRule(rule) + h.drain() + got := h.box.to("a@example.com") + if len(got) != 2 || got[1].Event != "resolved" { + t.Fatalf("mails = %+v", got) + } +} + +func TestWebhookResultStaysOnAlertWhenRuleAlsoEmails(t *testing.T) { + var calls sync.WaitGroup + calls.Add(1) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + defer calls.Done() + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + h := newMailHarness(t, 30) + rule, err := h.store.CreateRule(Rule{Name: "both", Enabled: true, Service: "api", Expression: "true", WebhookURL: srv.URL, EmailRecipients: []string{"a@example.com"}}) + if err != nil { + t.Fatal(err) + } + h.box.fail = func(auth.AlertMail) error { return &mail.SendError{Reason: mail.ErrSMTPRcptTo} } + h.fire(rule) + calls.Wait() + h.drain() + + deadline := time.Now().Add(5 * time.Second) + for { + alert, err := h.store.GetAlert(rule.ID, "api") + if err != nil { + t.Fatal(err) + } + if alert.LastDeliveryStatus == "success" { + break + } + if time.Now().After(deadline) { + t.Fatalf("alert status = %q, want the webhook's success", alert.LastDeliveryStatus) + } + time.Sleep(10 * time.Millisecond) + } + if status := h.ruleStatus(t, rule.ID).EmailStatus; status != "failed" { + t.Fatalf("rule email status = %q", status) + } +} + +func TestWithoutSMTPSavedRecipientsReportUnavailable(t *testing.T) { + engine, store := newTestEngine(t) + rule, err := store.CreateRule(Rule{Name: "errors", Enabled: true, Expression: "true", EmailRecipients: []string{"a@example.com"}}) + if err != nil { + t.Fatal(err) + } + got := engine.WithEmailState(rule) + if got.EmailStatus != "unavailable" || !strings.Contains(got.EmailStatusDetail, "SMTP") { + t.Fatalf("rule = %+v", got) + } + if engine.EmailOptions().Available { + t.Fatal("email reported available without SMTP") + } + plain, err := store.CreateRule(Rule{Name: "plain", Enabled: true, Expression: "true"}) + if err != nil { + t.Fatal(err) + } + if got := engine.WithEmailState(plain); got.EmailStatus != "" { + t.Fatalf("rule without email = %+v", got) + } +} + +func TestMailQueuedBeforeSMTPWasRemovedIsMarkedFailed(t *testing.T) { + h := newMailHarness(t, 30) + rule := h.rule(t, "errors", "a@example.com") + h.fire(rule) + + without := &Engine{store: h.store, programs: map[string]Program{}, stopped: map[string]bool{}} + without.failUndeliverableEmail(context.Background()) + stored := h.ruleStatus(t, rule.ID) + if stored.EmailStatus != "failed" || !strings.Contains(stored.EmailStatusDetail, "SMTP") { + t.Fatalf("rule = %+v", stored) + } +} + +func TestRuleWithoutEmailDestinationHasNoEmailState(t *testing.T) { + h := newMailHarness(t, 30) + rule := h.rule(t, "errors") + rule.EmailStatus = "failed" + if got := h.engine.WithEmailState(rule); got.EmailStatus != "" { + t.Fatalf("rule = %+v", got) + } +} diff --git a/internal/alert/engine.go b/internal/alert/engine.go index 01dce18ed..e333d3207 100644 --- a/internal/alert/engine.go +++ b/internal/alert/engine.go @@ -7,6 +7,7 @@ import ( "regexp" "runtime/debug" "sort" + "strings" "sync" "time" @@ -29,8 +30,10 @@ type Engine struct { interval time.Duration histDays int lastPrune time.Time - envOverride map[string]Env // for testing — bypasses DuckDB query - afterList func() // for testing — runs once a tick has listed its rules + envOverride map[string]Env // for testing — bypasses DuckDB query + afterList func() // for testing — runs once a tick has listed its rules + email *EmailConfig // nil without SMTP + now func() time.Time // for testing — the email clock } // NewEngine creates an Engine ready to Run. @@ -76,7 +79,7 @@ func (e *Engine) DisableRule(rule Rule) { a.State = "resolved" a.ResolvedAt = now a.LastEvaluatedAt = now - saved, err := e.store.UpsertAlert(a) + saved, err := e.saveAlert(rule, a, resolveEvent(rule)) if err != nil { slog.Error("alert: resolve alert of disabled rule", "rule", rule.ID, "service", a.Service, "err", err) continue @@ -108,6 +111,12 @@ func (e *Engine) Run(ctx context.Context) { slog.Error("alert: evaluation interval must be positive", "interval", e.interval) return } + if e.email != nil { + go e.runEmail(ctx) + } else { + e.failUndeliverableEmail(ctx) + } + ticker := time.NewTicker(e.interval) defer ticker.Stop() @@ -262,7 +271,7 @@ func (e *Engine) transition(rule Rule, svc string, triggered bool, env Env, now LastEvaluatedAt: nowStr, CreatedAt: began, } - saved, upsertErr := e.store.UpsertAlert(a) + saved, upsertErr := e.saveAlert(rule, a, "firing") if upsertErr != nil { slog.Error("alert: upsert firing", "rule", rule.ID, "service", svc, "err", upsertErr) return @@ -299,7 +308,7 @@ func (e *Engine) transition(rule Rule, svc string, triggered bool, env Env, now existing.State = "firing" existing.FiredAt = nowStr existing.LastEvaluatedAt = nowStr - saved, upsertErr := e.store.UpsertAlert(existing) + saved, upsertErr := e.saveAlert(rule, existing, "firing") if upsertErr != nil { slog.Error("alert: promote to firing", "rule", rule.ID, "service", svc, "err", upsertErr) return @@ -335,7 +344,7 @@ func (e *Engine) transition(rule Rule, svc string, triggered bool, env Env, now repeatInterval := time.Duration(rule.RepeatIntervalS) * time.Second if parseErr == nil && now.Sub(refTime) >= repeatInterval { existing.RepeatedAt = nowStr - saved, upsertErr := e.store.UpsertAlert(existing) + saved, upsertErr := e.saveAlert(rule, existing, "reminder") if upsertErr != nil { slog.Error("alert: update repeated_at", "rule", rule.ID, "service", svc, "err", upsertErr) return @@ -353,7 +362,7 @@ func (e *Engine) transition(rule Rule, svc string, triggered bool, env Env, now existing.State = "resolved" existing.ResolvedAt = nowStr existing.LastEvaluatedAt = nowStr - saved, upsertErr := e.store.UpsertAlert(existing) + saved, upsertErr := e.saveAlert(rule, existing, resolveEvent(rule)) if upsertErr != nil { slog.Error("alert: resolve alert", "rule", rule.ID, "service", svc, "err", upsertErr) return @@ -387,7 +396,7 @@ func (e *Engine) transition(rule Rule, svc string, triggered bool, env Env, now if rule.ForSeconds == 0 { a.State = "firing" a.FiredAt = nowStr - saved, upsertErr := e.store.UpsertAlert(a) + saved, upsertErr := e.saveAlert(rule, a, "firing") if upsertErr != nil { slog.Error("alert: re-fire from resolved", "rule", rule.ID, "service", svc, "err", upsertErr) return @@ -453,8 +462,22 @@ func (e *Engine) Test(ctx context.Context, rule Rule) ([]TestResult, error) { return results, nil } +// resolveEvent is the event a resolve notifies, or empty when the rule does not +// notify on resolve. +func resolveEvent(rule Rule) string { + if rule.NotifyOnResolve { + return "resolved" + } + return "" +} + // fireWebhookAsync fires the webhook for an alert in a goroutine and updates delivery status. +// A rule delivering only by email has no webhook result to record; the email +// worker reports on the alert instead. func (e *Engine) fireWebhookAsync(rule Rule, alert Alert, env Env, event string) { + if strings.TrimSpace(rule.WebhookURL) == "" && len(e.emailRecipients(rule)) > 0 { + return + } go func() { defer func() { if r := recover(); r != nil { @@ -577,6 +600,10 @@ func (e *Engine) pruneOldAlerts() { slog.Warn("alert: prune resolved", "err", err) return } + if err := e.store.pruneEmailDeliveries(days, time.Now()); err != nil { + slog.Warn("alert: prune email deliveries", "err", err) + return + } e.lastPrune = time.Now() if n > 0 { slog.Info("alert: pruned resolved alerts", "count", n) diff --git a/internal/alert/store.go b/internal/alert/store.go index 6567cd9d6..64dfc4afd 100644 --- a/internal/alert/store.go +++ b/internal/alert/store.go @@ -15,12 +15,13 @@ var ErrNotFound = errors.New("not found") // Store provides CRUD operations for alert rules and alert instances. type Store struct { - q *generated.Queries + db *sql.DB + q *generated.Queries } // NewStore creates a Store backed by the given database connection. func NewStore(db *sql.DB) *Store { - return &Store{q: generated.New(db)} + return &Store{db: db, q: generated.New(db)} } // newID returns a new UUIDv7 string. @@ -49,6 +50,7 @@ func ruleToCreateParams(r Rule) generated.CreateRuleParams { WebhookHeaders: nullString(r.WebhookHeaders), WebhookTemplate: nullString(r.WebhookTemplate), NotifyOnResolve: boolToInt64(r.NotifyOnResolve), + EmailRecipients: encodeRecipients(r.EmailRecipients), } } @@ -69,28 +71,33 @@ func ruleToReplaceParams(r Rule) generated.ReplaceRuleParams { WebhookHeaders: nullString(r.WebhookHeaders), WebhookTemplate: nullString(r.WebhookTemplate), NotifyOnResolve: boolToInt64(r.NotifyOnResolve), + EmailRecipients: encodeRecipients(r.EmailRecipients), } } // alertRuleToRule converts a generated.AlertRule to a domain Rule. func alertRuleToRule(ar generated.AlertRule) Rule { return Rule{ - ID: ar.ID, - Name: ar.Name, - Description: ar.Description.String, - Enabled: ar.Enabled != 0, - Service: ar.Service.String, - Namespace: ar.Namespace.String, - Expression: ar.Expression, - ForSeconds: int(ar.ForSeconds), - CooldownS: int(ar.CooldownS), - RepeatIntervalS: int(ar.RepeatIntervalS), - WebhookURL: ar.WebhookURL.String, - WebhookHeaders: ar.WebhookHeaders.String, - WebhookTemplate: ar.WebhookTemplate.String, - NotifyOnResolve: ar.NotifyOnResolve != 0, - CreatedAt: ar.CreatedAt, - UpdatedAt: ar.UpdatedAt, + ID: ar.ID, + Name: ar.Name, + Description: ar.Description.String, + Enabled: ar.Enabled != 0, + Service: ar.Service.String, + Namespace: ar.Namespace.String, + Expression: ar.Expression, + ForSeconds: int(ar.ForSeconds), + CooldownS: int(ar.CooldownS), + RepeatIntervalS: int(ar.RepeatIntervalS), + WebhookURL: ar.WebhookURL.String, + WebhookHeaders: ar.WebhookHeaders.String, + WebhookTemplate: ar.WebhookTemplate.String, + NotifyOnResolve: ar.NotifyOnResolve != 0, + EmailRecipients: decodeRecipients(ar.ID, ar.EmailRecipients), + EmailStatus: ar.EmailStatus, + EmailStatusDetail: ar.EmailStatusDetail, + EmailStatusAt: ar.EmailStatusAt, + CreatedAt: ar.CreatedAt, + UpdatedAt: ar.UpdatedAt, } } @@ -216,13 +223,17 @@ func (s *Store) DeleteRule(id string) error { // UpsertAlert inserts an alert episode, or updates the one with a.ID. Each // episode is its own row; at most one per (rule_id, service) is open. func (s *Store) UpsertAlert(a Alert) (Alert, error) { + return upsertAlert(s.q, a) +} + +func upsertAlert(q *generated.Queries, a Alert) (Alert, error) { // Only the engine starts an episode. An update without a start keeps the // stored one; a new alert must bring its own. if a.CreatedAt == "" { if a.ID == "" { return Alert{}, fmt.Errorf("store: new alert (%q, %q) needs created_at", a.RuleID, a.Service) } - if _, err := s.q.GetAlertByID(context.Background(), a.ID); errors.Is(err, sql.ErrNoRows) { + if _, err := q.GetAlertByID(context.Background(), a.ID); errors.Is(err, sql.ErrNoRows) { return Alert{}, fmt.Errorf("store: new alert (%q, %q) needs created_at", a.RuleID, a.Service) } else if err != nil { return Alert{}, fmt.Errorf("store: get alert: %w", err) @@ -241,7 +252,7 @@ func (s *Store) UpsertAlert(a Alert) (Alert, error) { } a.ID = id } - ga, err := s.q.UpsertAlert(context.Background(), alertToUpsertParams(a)) + ga, err := q.UpsertAlert(context.Background(), alertToUpsertParams(a)) if err != nil { return Alert{}, fmt.Errorf("store: upsert alert: %w", err) } diff --git a/internal/alert/types.go b/internal/alert/types.go index 146f81719..2d7f8fde1 100644 --- a/internal/alert/types.go +++ b/internal/alert/types.go @@ -18,8 +18,16 @@ type Rule struct { WebhookHeaders string `json:"webhook_headers,omitempty"` WebhookTemplate string `json:"webhook_template,omitempty"` NotifyOnResolve bool `json:"notify_on_resolve"` - CreatedAt string `json:"created_at"` - UpdatedAt string `json:"updated_at"` + // EmailRecipients are this rule's own addresses. Empty means none: the + // default destination then applies if the rule has no webhook either. + EmailRecipients []string `json:"email_recipients,omitempty"` + // EmailStatus reports the rule's latest email notification: sent, retrying, + // failed, suppressed, or unavailable. The server owns these three fields. + EmailStatus string `json:"email_status,omitempty"` + EmailStatusDetail string `json:"email_status_detail,omitempty"` + EmailStatusAt string `json:"email_status_at,omitempty"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` } // Alert represents the current state of an alert instance for a (rule, service) pair. diff --git a/internal/api/alerts.go b/internal/api/alerts.go index cd142fd93..8b2523e52 100644 --- a/internal/api/alerts.go +++ b/internal/api/alerts.go @@ -3,11 +3,13 @@ package api import ( "log/slog" "net/http" + "slices" "strings" "github.com/labstack/echo/v5" "github.com/labstack/fanout/internal/alert" + "github.com/labstack/fanout/internal/auth" ) // RegisterAlertRoutes registers alert management REST endpoints. @@ -19,6 +21,7 @@ func RegisterAlertRoutes(e *echo.Echo, store *alert.Store, engine *alert.Engine) e.GET("/api/alerting/alerts", h.ListAlerts, read) e.GET("/api/alerting/summary", h.AlertSummary, read) e.GET("/api/alerting/rules", h.ListRules, read) + e.GET("/api/alerting/email", h.EmailOptions, read) e.POST("/api/alerting/rules", h.CreateRule, manage) e.PUT("/api/alerting/rules/:id", h.ReplaceRule, manage) e.DELETE("/api/alerting/rules/:id", h.DeleteRule, manage) @@ -76,9 +79,17 @@ func (h *alertHandler) ListRules(c *echo.Context) error { if rules == nil { rules = []alert.Rule{} } + for i := range rules { + rules[i] = h.engine.WithEmailState(rules[i]) + } return c.JSON(200, map[string][]alert.Rule{"rules": rules}) } +// EmailOptions reports whether rules can deliver by email on this instance. +func (h *alertHandler) EmailOptions(c *echo.Context) error { + return c.JSON(200, h.engine.EmailOptions()) +} + // CreateRule creates a new alert rule. func (h *alertHandler) CreateRule(c *echo.Context) error { if h.store == nil { @@ -96,6 +107,9 @@ func (h *alertHandler) CreateRule(c *echo.Context) error { if err := checkRule(req, true); err != nil { return err } + if err := h.checkRecipients(&req); err != nil { + return err + } created, err := h.store.CreateRule(req) if err != nil { @@ -110,7 +124,7 @@ func (h *alertHandler) CreateRule(c *echo.Context) error { } } - return c.JSON(201, created) + return c.JSON(201, h.engine.WithEmailState(created)) } // ReplaceRule replaces an existing alert rule. @@ -133,6 +147,12 @@ func (h *alertHandler) ReplaceRule(c *echo.Context) error { if err := checkRule(req, changed); err != nil { return err } + // Likewise for recipients: a saved rule stays editable after SMTP is removed. + if err != nil || !slices.Equal(stored.EmailRecipients, req.EmailRecipients) { + if err := h.checkRecipients(&req); err != nil { + return err + } + } updated, err := h.store.ReplaceRule(req) if err != nil { @@ -153,7 +173,7 @@ func (h *alertHandler) ReplaceRule(c *echo.Context) error { } } - return c.JSON(200, updated) + return c.JSON(200, h.engine.WithEmailState(updated)) } // DeleteRule deletes an alert rule and its associated alerts. @@ -207,6 +227,23 @@ func (h *alertHandler) TestRule(c *echo.Context) error { }) } +// checkRecipients normalizes a rule's email recipients in place, rejecting +// invalid addresses, too many of them, or any when email is unavailable. +func (h *alertHandler) checkRecipients(rule *alert.Rule) error { + if len(rule.EmailRecipients) == 0 { + return nil + } + if options := h.engine.EmailOptions(); !options.Available { + return newAPIError(http.StatusBadRequest, "email_unavailable", options.Message) + } + recipients, err := auth.NormalizeRecipients(rule.EmailRecipients) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, err.Error()) + } + rule.EmailRecipients = recipients + return nil +} + // checkRule rejects an expression that does not compile and, when headers is // set, webhook headers that are not a JSON object of valid HTTP headers. A // replace may omit the expression. diff --git a/internal/api/alerts_email_test.go b/internal/api/alerts_email_test.go new file mode 100644 index 000000000..58173cf4d --- /dev/null +++ b/internal/api/alerts_email_test.go @@ -0,0 +1,157 @@ +package api + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/labstack/fanout/internal/alert" + "github.com/labstack/fanout/internal/auth" +) + +type alertEmailServer struct { + *testAuthServer + store *alert.Store + cookie *http.Cookie +} + +func newAlertEmailServer(t *testing.T, smtp bool) *alertEmailServer { + t.Helper() + s := newTestAuthServer(t) + store := alert.NewStore(s.db.DB) + engine := alert.NewEngine(store, nil, nil, 30*time.Second, 7) + if smtp { + engine.EnableEmail(alert.EmailConfig{Send: func(context.Context, auth.AlertMail) error { return nil }, Defaults: []string{"oncall@example.com"}, HourlyLimit: 30}) + } + RegisterAlertRoutes(s.e, store, engine) + admin, err := s.users.CreateWithAudit("alert-admin@example.com", "", "admin", auth.AuditEvent{EventType: "user.created", Outcome: "success"}) + if err != nil { + t.Fatal(err) + } + return &alertEmailServer{testAuthServer: s, store: store, cookie: s.login(t, admin)} +} + +func (s *alertEmailServer) do(method, path, body string) *httptest.ResponseRecorder { + req := sessionRequest(method, path, strings.NewReader(body), s.cookie) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + s.e.ServeHTTP(rec, req) + return rec +} + +func ruleBody(recipients string) string { + return `{"name":"Errors","enabled":true,"expression":"error_rate > 0.05","email_recipients":` + recipients + `}` +} + +func TestAlertRuleEmailRecipientsAreValidatedAndNormalized(t *testing.T) { + s := newAlertEmailServer(t, true) + rec := s.do(http.MethodPost, "/api/alerting/rules", ruleBody(`[" A@Example.com ","a@example.com","b@example.com"]`)) + if rec.Code != http.StatusCreated { + t.Fatalf("create = %d %s", rec.Code, rec.Body) + } + var created alert.Rule + if err := json.Unmarshal(rec.Body.Bytes(), &created); err != nil { + t.Fatal(err) + } + if len(created.EmailRecipients) != 2 || created.EmailRecipients[0] != "a@example.com" { + t.Fatalf("recipients = %v", created.EmailRecipients) + } + + for name, recipients := range map[string]string{ + "not an address": `["nope"]`, + "display name": `["Eve "]`, + "line break": `["a@example.com\r\nBcc: x@evil.test"]`, + "over the cap": `["a0@e.co","a1@e.co","a2@e.co","a3@e.co","a4@e.co","a5@e.co","a6@e.co","a7@e.co","a8@e.co","a9@e.co","a10@e.co"]`, + } { + t.Run(name, func(t *testing.T) { + if rec := s.do(http.MethodPost, "/api/alerting/rules", ruleBody(recipients)); rec.Code != http.StatusBadRequest { + t.Fatalf("create = %d %s, want 400", rec.Code, rec.Body) + } + if rec := s.do(http.MethodPut, "/api/alerting/rules/"+created.ID, ruleBody(recipients)); rec.Code != http.StatusBadRequest { + t.Fatalf("replace = %d %s, want 400", rec.Code, rec.Body) + } + }) + } +} + +func TestAlertRuleClientCannotWriteEmailStatus(t *testing.T) { + s := newAlertEmailServer(t, true) + rec := s.do(http.MethodPost, "/api/alerting/rules", `{"name":"Errors","enabled":true,"expression":"true","email_recipients":["a@example.com"],"email_status":"sent","email_status_detail":"forged"}`) + if rec.Code != http.StatusCreated || strings.Contains(rec.Body.String(), "forged") { + t.Fatalf("create = %d %s", rec.Code, rec.Body) + } +} + +func TestEmailOptionIsUnavailableWithoutSMTP(t *testing.T) { + s := newAlertEmailServer(t, false) + rec := s.do(http.MethodPost, "/api/alerting/rules", ruleBody(`["a@example.com"]`)) + if rec.Code != http.StatusBadRequest { + t.Fatalf("create = %d %s, want 400", rec.Code, rec.Body) + } + var body struct{ Code, Message string } + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if body.Code != "email_unavailable" || !strings.Contains(body.Message, "SMTP is not configured") { + t.Fatalf("error = %+v", body) + } + + options := s.do(http.MethodGet, "/api/alerting/email", "") + var got alert.EmailOptions + if err := json.Unmarshal(options.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + if options.Code != http.StatusOK || got.Available || !strings.Contains(got.Message, "SMTP") || got.DefaultDestination { + t.Fatalf("options = %d %s", options.Code, options.Body) + } + if rec := s.do(http.MethodPost, "/api/alerting/rules", `{"name":"Hook","enabled":true,"expression":"true","webhook_url":"https://example.com/hook"}`); rec.Code != http.StatusCreated { + t.Fatalf("webhook rule = %d %s", rec.Code, rec.Body) + } +} + +func TestEmailOptionsReportTheDefaultDestinationWithoutItsAddress(t *testing.T) { + s := newAlertEmailServer(t, true) + rec := s.do(http.MethodGet, "/api/alerting/email", "") + if rec.Code != http.StatusOK || strings.Contains(rec.Body.String(), "oncall@") { + t.Fatalf("options = %d %s", rec.Code, rec.Body) + } + var got alert.EmailOptions + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + if !got.Available || !got.DefaultDestination || got.MaxRecipients != auth.MaxAlertEmailRecipients || got.HourlyLimit != 30 { + t.Fatalf("options = %+v", got) + } +} + +// A rule saved while SMTP worked stays editable and reports the failure. +func TestSavedEmailRuleWithoutSMTPShowsFailureAndStaysEditable(t *testing.T) { + s := newAlertEmailServer(t, false) + stored, err := s.store.CreateRule(alert.Rule{Name: "Errors", Enabled: true, Expression: "true", EmailRecipients: []string{"a@example.com"}}) + if err != nil { + t.Fatal(err) + } + rec := s.do(http.MethodGet, "/api/alerting/rules", "") + var listed struct{ Rules []alert.Rule } + if err := json.Unmarshal(rec.Body.Bytes(), &listed); err != nil { + t.Fatal(err) + } + if len(listed.Rules) != 1 || listed.Rules[0].EmailStatus != "unavailable" || !strings.Contains(listed.Rules[0].EmailStatusDetail, "SMTP") { + t.Fatalf("rules = %s", rec.Body) + } + + toggle := `{"name":"Errors","enabled":false,"expression":"true","email_recipients":["a@example.com"]}` + if rec := s.do(http.MethodPut, "/api/alerting/rules/"+stored.ID, toggle); rec.Code != http.StatusOK { + t.Fatalf("toggle = %d %s", rec.Code, rec.Body) + } + if rec := s.do(http.MethodPut, "/api/alerting/rules/"+stored.ID, ruleBody(`["new@example.com"]`)); rec.Code != http.StatusBadRequest { + t.Fatalf("changed recipients = %d %s, want 400", rec.Code, rec.Body) + } + if rec := s.do(http.MethodPut, "/api/alerting/rules/"+stored.ID, ruleBody(`[]`)); rec.Code != http.StatusOK { + t.Fatalf("clear recipients = %d %s", rec.Code, rec.Body) + } +} diff --git a/internal/api/auth_middleware.go b/internal/api/auth_middleware.go index ed1ac6c8e..da2ab747c 100644 --- a/internal/api/auth_middleware.go +++ b/internal/api/auth_middleware.go @@ -299,7 +299,7 @@ func classifyRoute(method, path string) (routePolicy, bool) { return routePolicy{kind: routePolicyCapability, capability: ReadTelemetry}, read case path == "/api/insights": return routePolicy{kind: routePolicyCapability, capability: ReadTelemetry}, read - case path == "/api/alerting/alerts" || path == "/api/alerting/summary" || path == "/api/alerting/rules": + case path == "/api/alerting/alerts" || path == "/api/alerting/summary" || path == "/api/alerting/rules" || path == "/api/alerting/email": if read { return routePolicy{kind: routePolicyCapability, capability: ReadTelemetry}, true } diff --git a/internal/auth/alert_mail.go b/internal/auth/alert_mail.go new file mode 100644 index 000000000..e0e4010d8 --- /dev/null +++ b/internal/auth/alert_mail.go @@ -0,0 +1,145 @@ +package auth + +import ( + "bytes" + "context" + "errors" + "fmt" + "html/template" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + mail "github.com/wneessen/go-mail" +) + +// AlertMail is one alert notification for one recipient. Every text field may +// hold telemetry-derived or operator-written values; SendAlert neutralizes them. +type AlertMail struct { + // ID identifies the delivery. It becomes the Message-ID, so a resend after + // an interrupted attempt carries the same one. + ID string + To string + Rule string + Description string + Service string + Event string + Value float64 + Time time.Time + // LimitNotice, when set, warns that later notifications will be suppressed. + LimitNotice string +} + +var errInvalidMailAddress = errors.New("invalid mail address") + +// IsPermanentMailError reports whether retrying err cannot succeed: an address +// that does not parse, or a 5xx SMTP rejection. Connection failures and 4xx +// replies are transient. +func IsPermanentMailError(err error) bool { + if errors.Is(err, errInvalidMailAddress) { + return true + } + var sendErr *mail.SendError + return errors.As(err, &sendErr) && !sendErr.IsTemp() +} + +// SendAlert renders and sends one alert notification. +func SendAlert(ctx context.Context, cfg SMTPConfig, alert AlertMail) error { + subject, text, html, err := renderAlertMail(alert) + if err != nil { + return err + } + message, err := newMailMessage(cfg.From, alert.To, subject, text, html) + if err != nil { + return fmt.Errorf("%w: %w", errInvalidMailAddress, err) + } + message.SetMessageIDWithValue(alert.ID + "@alerts.fanout") + return deliver(ctx, cfg, message) +} + +// Lengths, in runes, past which a value is cut. They keep a hostile value from +// filling a subject line or a message. +const ( + maxAlertNameLength = 120 + maxAlertDescriptionLength = 1000 +) + +// singleLine makes a value safe for a header or one line of text: control, +// format and line-separator characters become spaces, so CR/LF cannot start a +// new header or forge a line, and whitespace runs collapse. +func singleLine(value string, limit int) string { + cleaned := strings.Map(func(r rune) rune { + if unicode.IsControl(r) || unicode.Is(unicode.Cf, r) || unicode.Is(unicode.Zl, r) || unicode.Is(unicode.Zp, r) { + return ' ' + } + return r + }, strings.ToValidUTF8(value, " ")) + cleaned = strings.Join(strings.Fields(cleaned), " ") + if utf8.RuneCountInString(cleaned) > limit { + cleaned = strings.TrimSpace(string([]rune(cleaned)[:limit])) + "…" + } + return cleaned +} + +type alertMailData struct { + Preheader string + Label string + Rule string + Description string + Service string + Value string + Time string + LimitNotice string +} + +func alertEventLabel(event string) string { + switch event { + case "firing": + return "Firing" + case "reminder": + return "Still firing" + case "resolved": + return "Resolved" + default: + return "Alert" + } +} + +func renderAlertMail(alert AlertMail) (subject, text, html string, err error) { + data := alertMailData{ + Label: alertEventLabel(alert.Event), + Rule: singleLine(alert.Rule, maxAlertNameLength), + Description: singleLine(alert.Description, maxAlertDescriptionLength), + Service: singleLine(alert.Service, maxAlertNameLength), + Value: strconv.FormatFloat(alert.Value, 'g', 6, 64), + Time: alert.Time.UTC().Format(time.RFC1123), + LimitNotice: singleLine(alert.LimitNotice, maxAlertDescriptionLength), + } + subject = fmt.Sprintf("[Fanout] %s: %s on %s", data.Label, data.Rule, data.Service) + data.Preheader = subject + + var body strings.Builder + fmt.Fprintf(&body, "%s\n\nRule: %s\nService: %s\nValue: %s\nTime: %s\n", data.Label, data.Rule, data.Service, data.Value, data.Time) + if data.Description != "" { + fmt.Fprintf(&body, "Description: %s\n", data.Description) + } + if data.LimitNotice != "" { + fmt.Fprintf(&body, "\n%s\n", data.LimitNotice) + } + + var rendered bytes.Buffer + if err := alertMailTemplate.Execute(&rendered, data); err != nil { + return "", "", "", fmt.Errorf("auth: render alert email: %w", err) + } + return subject, body.String(), rendered.String(), nil +} + +var alertMailTemplate = template.Must(template.New("alert-email").Parse(mailShellStart + ` +

{{.Label}}

+

{{.Rule}}

+

Service: {{.Service}}
Value: {{.Value}}
Time: {{.Time}}

+{{if .Description}}

{{.Description}}

{{end}} +{{if .LimitNotice}}

{{.LimitNotice}}

{{end}} +` + mailShellEnd)) diff --git a/internal/auth/alert_mail_test.go b/internal/auth/alert_mail_test.go new file mode 100644 index 000000000..e6c48130a --- /dev/null +++ b/internal/auth/alert_mail_test.go @@ -0,0 +1,298 @@ +package auth + +import ( + "bufio" + "context" + "encoding/base64" + "fmt" + "io" + "mime" + "mime/multipart" + "mime/quotedprintable" + "net" + "net/mail" + "net/textproto" + "strings" + "sync" + "testing" + "time" +) + +// fakeSMTP accepts one conversation at a time and records message data. A +// nonempty rcptReply replaces the normal reply to RCPT. +type fakeSMTP struct { + listener net.Listener + rcptReply string + mu sync.Mutex + data []string +} + +func startFakeSMTP(t *testing.T, rcptReply string) (*fakeSMTP, SMTPConfig) { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + server := &fakeSMTP{listener: listener, rcptReply: rcptReply} + t.Cleanup(func() { listener.Close() }) + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + go server.serve(conn) + } + }() + return server, SMTPConfig{Host: "127.0.0.1", Port: listener.Addr().(*net.TCPAddr).Port, User: "user", Pass: "pass", From: "Fanout "} +} + +func (s *fakeSMTP) serve(conn net.Conn) { + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) + reader := bufio.NewReader(conn) + reply := func(line string) { fmt.Fprintf(conn, "%s\r\n", line) } + reply("220 fake ESMTP") + for { + line, err := reader.ReadString('\n') + if err != nil { + return + } + verb := strings.ToUpper(strings.Fields(line + " x")[0]) + switch verb { + case "EHLO", "HELO": + reply("250-fake") + reply("250 AUTH CRAM-MD5") + case "AUTH": + reply("334 PDEyMzQ1QGZha2U+") + if _, err := reader.ReadString('\n'); err != nil { + return + } + reply("235 ok") + case "RCPT": + if s.rcptReply != "" { + reply(s.rcptReply) + continue + } + reply("250 ok") + case "DATA": + reply("354 go") + var body strings.Builder + for { + part, err := reader.ReadString('\n') + if err != nil { + return + } + if part == ".\r\n" { + break + } + body.WriteString(strings.TrimPrefix(part, ".")) + } + s.mu.Lock() + s.data = append(s.data, body.String()) + s.mu.Unlock() + reply("250 queued") + case "QUIT": + reply("221 bye") + return + default: + reply("250 ok") + } + } +} + +func (s *fakeSMTP) messages() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.data...) +} + +const hostileService = "api\r\nBcc: victim@evil.test\r\n\r\n\u2028Subject: forged\u202e" + +func hostileAlert() AlertMail { + return AlertMail{ + ID: "0190-delivery", To: "oncall@example.com", Rule: "High errors\r\nX-Injected: yes", Description: "\nline two", + Service: hostileService, Event: "firing", Value: 0.5, Time: time.Date(2026, 11, 8, 12, 0, 0, 0, time.UTC), + } +} + +func decodeParts(t *testing.T, message *mail.Message) (text, html string) { + t.Helper() + _, params, err := mime.ParseMediaType(message.Header.Get("Content-Type")) + if err != nil { + t.Fatal(err) + } + reader := multipart.NewReader(message.Body, params["boundary"]) + for { + part, err := reader.NextRawPart() + if err == io.EOF { + return text, html + } + if err != nil { + t.Fatal(err) + } + var source io.Reader = part + switch strings.ToLower(part.Header.Get("Content-Transfer-Encoding")) { + case "quoted-printable": + source = quotedprintable.NewReader(part) + case "base64": + source = base64.NewDecoder(base64.StdEncoding, stripNewlines{part}) + } + content, err := io.ReadAll(source) + if err != nil { + t.Fatal(err) + } + if strings.HasPrefix(part.Header.Get("Content-Type"), "text/html") { + html = string(content) + } else { + text = string(content) + } + } +} + +type stripNewlines struct{ reader io.Reader } + +func (s stripNewlines) Read(p []byte) (int, error) { + n, err := s.reader.Read(p) + kept := 0 + for _, b := range p[:n] { + if b != '\r' && b != '\n' { + p[kept] = b + kept++ + } + } + return kept, err +} + +func TestSendAlertKeepsHostileValuesOutOfHeadersAndMarkup(t *testing.T) { + server, cfg := startFakeSMTP(t, "") + if err := SendAlert(context.Background(), cfg, hostileAlert()); err != nil { + t.Fatal(err) + } + messages := server.messages() + if len(messages) != 1 { + t.Fatalf("messages = %d", len(messages)) + } + message, err := mail.ReadMessage(strings.NewReader(messages[0])) + if err != nil { + t.Fatal(err) + } + for _, header := range []string{"Bcc", "X-Injected"} { + if value := message.Header.Get(header); value != "" { + t.Fatalf("injected header %s: %q", header, value) + } + } + if got := message.Header[textproto.CanonicalMIMEHeaderKey("Subject")]; len(got) != 1 { + t.Fatalf("subject headers = %q", got) + } + subject, err := new(mime.WordDecoder).DecodeHeader(message.Header.Get("Subject")) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(subject, "[Fanout] Firing: High errors X-Injected: yes on api Bcc:") { + t.Fatalf("subject = %q", subject) + } + if strings.ContainsAny(subject, "\r\n\u2028\u202e") { + t.Fatalf("subject holds control characters: %q", subject) + } + if got := message.Header.Get("Message-Id"); got != "<0190-delivery@alerts.fanout>" { + t.Fatalf("message id = %q", got) + } + + text, html := decodeParts(t, message) + if !strings.Contains(text, "Service: api Bcc: victim@evil.test - + diff --git a/site/src/content/docs/docs/guides/alerts.mdx b/site/src/content/docs/docs/guides/alerts.mdx index d12b8828f..44810753a 100644 --- a/site/src/content/docs/docs/guides/alerts.mdx +++ b/site/src/content/docs/docs/guides/alerts.mdx @@ -1,7 +1,7 @@ --- title: Set up alerts -description: Write alert rules as expressions over service metrics, and deliver them by webhook. -summary: The rule model, the variables available to an expression, the pending/firing/resolved lifecycle, and webhook delivery. +description: Write alert rules as expressions over service metrics, and deliver them by webhook or email. +summary: The rule model, the variables available to an expression, the pending/firing/resolved lifecycle, and webhook and email delivery. read_when: - You want to be told when a service degrades. - A rule is not firing when you expect it to. @@ -10,23 +10,26 @@ status: preview The alert engine runs inside the binary and is on by default. It evaluates persisted rules every `FANOUT_ALERTS_EVALUATION_INTERVAL` (30s) against service -rollups, and delivers by webhook. +rollups, and delivers by webhook, by email, or both. ## Create and manage a rule Open **Alerts** in the sidebar, then choose **New rule**. Enter a name and -expression, plus an optional description and webhook URL. Without a webhook -URL, the rule records alerts but delivers nothing. Leave **Service** empty to +expression, plus an optional description, email recipients and webhook URL. A +rule with neither email recipients nor a webhook URL goes to the +[default email destination](#email-delivery) when one is configured. Without +one, the rule records alerts but delivers nothing. Leave **Service** empty to evaluate every service, or select one service. Choose **Start from a template** for an example expression, then set the timers and save. The new-rule form starts enabled, with **Hold for (s)** 60, **Cooldown (s)** 300 and **Repeat every (s)** 3600. **Notify when resolved** starts off. The name, -description, service, expression and webhook fields start empty. +description, service, expression, email and webhook fields start empty. Use the **Enabled** toggle to pause or resume a rule. The rule's actions menu has **Edit**, **Test against current data** and **Delete**. Testing evaluates -the saved expression without changing alert state or sending a webhook. +the saved expression without changing alert state, sending a webhook or +sending email. Deleting removes the rule and its alert history. Choose **Open in Explore** to chart a watched measure from raw telemetry. @@ -54,7 +57,8 @@ An expression, an optional service to scope it to, timing, and delivery: | `for_seconds` | How long the condition must hold before firing | | `cooldown_s` | Minimum quiet period after resolving before firing again | | `repeat_interval_s` | How often to re-deliver while still firing | -| `webhook_url` | Where to deliver | +| `email_recipients` | Array of up to 10 addresses to email; omit for none | +| `webhook_url` | Where to deliver by webhook | | `webhook_headers` | JSON-encoded object of extra headers, supplied as a string | | `webhook_template` | Go text/template payload body; omit for the default | | `notify_on_resolve` | Also deliver when the alert clears | @@ -85,8 +89,8 @@ An alert moves through three states: - **pending**: the condition is true but has not held for `for_seconds` yet. With `for_seconds: 0` a rule skips this and fires immediately. -- **firing**: it has held. The webhook is delivered, and re-delivered every - `repeat_interval_s` while it stays true. +- **firing**: it has held. The webhook and email are delivered, and re-delivered + every `repeat_interval_s` while it stays true. - **resolved**: the expression went false. If `notify_on_resolve` is set, that is delivered too. @@ -95,8 +99,8 @@ what stops a metric hovering at the threshold from delivering repeatedly. Each alert records its last delivery status and timestamp in `GET /api/alerting/alerts`. Setting `repeat_interval_s` to zero disables repeat delivery. Disabling a rule -removes pending alerts and resolves firing alerts, sending a resolve webhook -when `notify_on_resolve` is set. +removes pending alerts and resolves firing alerts, sending a resolve +notification when `notify_on_resolve` is set. :::caution[The namespace field on a rule is not applied] A rule stores `namespace`, but evaluation combines service rollups across @@ -104,6 +108,65 @@ namespaces. Scope with `service`; services with the same name in different namespaces are evaluated together. ::: +## Email delivery + +Email uses the [SMTP settings](/docs/reference/configuration/smtp) Fanout already +uses for sign-in codes. Without SMTP the option is disabled in the rule form, and +the API rejects `email_recipients` with the code `email_unavailable`. +`GET /api/alerting/email` reports whether email is available. + +Set `FANOUT_ALERTS_EMAIL_DEFAULT_TO` to a comma-separated list of addresses to +give every rule a destination. It applies only to a rule with no webhook and no +recipients of its own, including rules created before email existed. A rule with +a webhook and no recipients stays webhook-only. + +Email follows the rule's existing timing: `for_seconds`, `cooldown_s`, +`repeat_interval_s` and `notify_on_resolve` apply as they do to webhooks. + +Addresses are checked when saved. Each must be a bare address, without a display +name. They are stored in lowercase and a rule can have up to 10. + +### Retries and restarts + +A notification is queued in the control database in the same transaction as the +alert's state change, with one row per recipient. Fanout makes up to five +attempts, waiting 1, 5, 15, then 30 minutes between them. Connection failures and +4xx SMTP replies are retried. A 5xx reply or an unusable address fails at once. + +Delivery is at least once. A restart resumes queued mail. If Fanout stops while a +message is being handed to the mail server, it cannot tell whether the server +took it, so it sends the message again. The repeat has the same `Message-ID`, and +only that one interrupted message per recipient can repeat. + +### The hourly limit + +`FANOUT_ALERTS_EMAIL_HOURLY_LIMIT` (default 30) caps the alert emails sent to one +address in any rolling hour, across all rules. The count is stored, so a restart +does not reset it. The last email within the limit says that the limit has been +reached. Notifications over the limit are suppressed: they are dropped, never +sent later, and logged as a warning. + +### Seeing what happened + +A rule whose latest email did not go through shows it in the **Rules** table, and +in `email_status` and `email_status_detail` on `GET /api/alerting/rules`: + +| `email_status` | Means | +|---|---| +| `sent` | Every recipient accepted the latest notification | +| `retrying` | An attempt failed and another is scheduled | +| `failed` | At least one recipient could not be reached | +| `suppressed` | At least one recipient was over the hourly limit | +| `unavailable` | The rule has recipients but SMTP is not configured | + +For a rule with no webhook, the alert's delivery status under **History** carries +`success`, `failed` or `suppressed` for the email. When a rule has a webhook, that +column stays the webhook's result. + +Rule names, descriptions and service names are reduced to a single line of +printable text in the subject and body, and escaped in the HTML part, so a +hostile service name cannot add mail headers or markup. + ## Why alerts lag the raw data Evaluation reads analytical rollups. The rollup pass follows durable telemetry diff --git a/site/src/content/docs/docs/reference/api.mdx b/site/src/content/docs/docs/reference/api.mdx index 2101702e0..8e298836a 100644 --- a/site/src/content/docs/docs/reference/api.mdx +++ b/site/src/content/docs/docs/reference/api.mdx @@ -100,6 +100,7 @@ Operational responses at `/healthz`, `/readyz` and `/metrics` keep their bodies. | `GET` | `/api/agent/threads/:id` | `agent:run` | | `PATCH` | `/api/agent/threads/:id` | `agent:run` | | `GET` | `/api/alerting/alerts` | `telemetry:read` | +| `GET` | `/api/alerting/email` | `telemetry:read` | | `GET` | `/api/alerting/rules` | `telemetry:read` | | `POST` | `/api/alerting/rules` | `alerts:manage` | | `DELETE` | `/api/alerting/rules/:id` | `alerts:manage` | diff --git a/site/src/content/docs/docs/reference/configuration/alerts.mdx b/site/src/content/docs/docs/reference/configuration/alerts.mdx index 8e68a3544..ea00dacf4 100644 --- a/site/src/content/docs/docs/reference/configuration/alerts.mdx +++ b/site/src/content/docs/docs/reference/configuration/alerts.mdx @@ -1,9 +1,10 @@ --- title: "Alert settings" -description: "Whether the alert engine runs, how often it evaluates, and how long history is kept." -summary: "Settings for the alert evaluation loop." +description: "Whether the alert engine runs, how often it evaluates, how long history is kept, and how alert email is addressed and limited." +summary: "Settings for the alert evaluation loop and alert email." read_when: - "Alerts are evaluating too slowly, or not at all." + - "You are setting a default alert email destination or changing the per-address limit." status: preview generated: true --- @@ -17,12 +18,22 @@ Vendor credentials and endpoints keep the vendor's documented names. | Environment variable | Type | Default | |---|---|---| +| `FANOUT_ALERTS_EMAIL_DEFAULT_TO` | string | — | +| `FANOUT_ALERTS_EMAIL_HOURLY_LIMIT` | integer | `30` | | `FANOUT_ALERTS_ENABLED` | boolean | `true` | | `FANOUT_ALERTS_EVALUATION_INTERVAL` | duration | `30s` | | `FANOUT_ALERTS_HISTORY_DAYS` | integer | `7` | ## Notes +### `FANOUT_ALERTS_EMAIL_DEFAULT_TO` + +A comma-separated list of addresses that receive alerts from rules with no webhook and no email recipients of their own; requires SMTP. + +### `FANOUT_ALERTS_EMAIL_HOURLY_LIMIT` + +Caps the alert emails sent to one address in any rolling hour, across all rules. Notifications over the limit are suppressed, not delayed, and shown on the rule; 0 suppresses all alert email. + ### `FANOUT_ALERTS_ENABLED` Runs the alert evaluation engine. diff --git a/ui/host/src/alerts/api.ts b/ui/host/src/alerts/api.ts index b042b36a5..18ed6d4c2 100644 --- a/ui/host/src/alerts/api.ts +++ b/ui/host/src/alerts/api.ts @@ -5,9 +5,14 @@ export type Rule = { id: string; name: string; description?: string; enabled: boolean; service?: string; namespace?: string; expression: string; for_seconds: number; cooldown_s: number; repeat_interval_s: number; webhook_url?: string; webhook_headers?: string; webhook_template?: string; notify_on_resolve: boolean; + email_recipients?: string[]; + /** Server-owned result of the rule's latest email notification. */ + email_status?: "sent" | "retrying" | "failed" | "suppressed" | "unavailable" | string; email_status_detail?: string; email_status_at?: string; created_at: string; updated_at: string; }; -export type RuleDraft = Omit; +export type RuleDraft = Omit; +/** internal/alert/email.go EmailOptions. */ +export type EmailOptions = { available: boolean; message?: string; default_destination: boolean; max_recipients: number; hourly_limit?: number }; export type Alert = { id: string; rule_id: string; service: string; state: "pending" | "firing" | "resolved" | string; value?: number; fired_at?: string; resolved_at?: string; last_evaluated_at?: string; last_delivery_status?: string; last_delivery_at?: string; created_at: string; @@ -17,10 +22,12 @@ export type RuleTest = { results: { service: string; triggered: boolean; error?: export const rulesKey = ["alerting", "rules"] as const; export const alertsKey = ["alerting", "alerts"] as const; +export const emailOptionsKey = ["alerting", "email"] as const; const rulePath = (id: string) => `/api/alerting/rules/${encodeURIComponent(id)}`; export const listRules = (signal?: AbortSignal) => request<{ rules: Rule[] }>("/api/alerting/rules", { signal }).then(r => r.rules); export const listAlerts = (signal?: AbortSignal) => request<{ alerts: Alert[] }>("/api/alerting/alerts", { signal }).then(r => r.alerts); +export const getEmailOptions = (signal?: AbortSignal) => request("/api/alerting/email", { signal }); export const createRule = (rule: RuleDraft) => request("/api/alerting/rules", { method: "POST", json: rule }); export const replaceRule = (id: string, rule: RuleDraft) => request(rulePath(id), { method: "PUT", json: rule }); export const deleteRule = (id: string) => request(rulePath(id), { method: "DELETE" }); diff --git a/ui/host/src/alerts/email.test.tsx b/ui/host/src/alerts/email.test.tsx new file mode 100644 index 000000000..6e5f6f8ef --- /dev/null +++ b/ui/host/src/alerts/email.test.tsx @@ -0,0 +1,136 @@ +import { MantineProvider } from "@mantine/core"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { fireEvent, within } from "@testing-library/dom"; +import { act } from "react"; +import { createRoot } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { EmailOptions, Rule } from "./api"; + +vi.mock("../auth", async original => ({ ...(await original()), useViewer: () => ({ id: "u", email: "u@example.test", display_name: "U", status: "active", role: "operator" }) })); +import { AlertsPage } from "./page"; + +const rule: Rule = { id: "r1", name: "Checkout errors", enabled: true, service: "checkout", expression: "error_rate > 0.05", for_seconds: 60, cooldown_s: 300, repeat_interval_s: 3600, notify_on_resolve: false, created_at: "", updated_at: "" }; +const available: EmailOptions = { available: true, default_destination: true, max_recipients: 3, hourly_limit: 30 }; +const smtpMessage = "Email delivery is unavailable because SMTP is not configured."; +const unavailable: EmailOptions = { available: false, message: smtpMessage, default_destination: false, max_recipients: 3 }; +const fetchMock = vi.fn(); +const cleanups: (() => void)[] = []; +let rules: Rule[]; +let options: EmailOptions; +const posted = () => fetchMock.mock.calls.filter(([url, init]) => String(url) === "/api/alerting/rules" && init?.method === "POST").map(([, init]) => JSON.parse(String(init?.body))); + +beforeEach(() => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + vi.stubGlobal("fetch", fetchMock); + vi.stubGlobal("ResizeObserver", class { observe() {} unobserve() {} disconnect() {} }); + fetchMock.mockReset(); + rules = [rule]; + options = available; + fetchMock.mockImplementation(async (input, init) => { + const url = String(input), method = init?.method ?? "GET"; + if (url === "/api/alerting/rules" && method === "GET") return Response.json({ rules }); + if (url === "/api/alerting/alerts") return Response.json({ alerts: [] }); + if (url === "/api/alerting/email") return Response.json(options); + if (url === "/api/alerting/rules" && method === "POST") return Response.json({ ...rule, id: "r2" }, { status: 201 }); + if (url === "/api/alerting/rules/r1" && method === "PUT") return Response.json(rule); + if (url === "/api/telemetry/schema?window=1h") return Response.json({ services: [] }); + throw new Error(`Unexpected request: ${method} ${url}`); + }); +}); +afterEach(async () => { await act(async () => cleanups.splice(0).forEach(cleanup => cleanup())); vi.unstubAllGlobals(); document.body.innerHTML = ""; }); + +async function render() { + const host = document.createElement("div"); document.body.append(host); + const root = createRoot(host), client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + await act(async () => root.render()); + cleanups.push(() => { root.unmount(); client.clear(); }); + const ui = within(document.body); + const wait = async (assertion: () => void) => vi.waitFor(async () => { await act(async () => {}); assertion(); }, { timeout: 3000, interval: 10 }); + const input = async (label: RegExp, value: string) => act(async () => { fireEvent.change(ui.getByLabelText(label, { selector: "input, textarea" }), { target: { value } }); }); + const openNew = async () => { await wait(() => ui.getByRole("button", { name: "New rule" })); await act(async () => { fireEvent.click(ui.getAllByRole("button", { name: "New rule" })[0]); }); await wait(() => ui.getByLabelText(/^Email recipients/, { selector: "input" })); }; + return { ui, wait, input, openNew }; +} + +const field = (ui: ReturnType) => ui.getByLabelText(/^Email recipients/, { selector: "input" }) as HTMLInputElement; + +describe("alert rule email", () => { + it("sends the addresses entered on a new rule", async () => { + const { ui, wait, input, openNew } = await render(); + await openNew(); + await wait(() => expect(field(ui).disabled).toBe(false)); + await input(/^Name/, "Errors"); + await input(/^Expression/, "error_rate > 0.05"); + await input(/^Email recipients/, "a@example.com, b@example.com"); + await act(async () => { fireEvent.click(ui.getByRole("button", { name: "Create rule" })); }); + await wait(() => expect(posted()[0]?.email_recipients).toEqual(["a@example.com", "b@example.com"])); + }); + + it("explains the default destination when one is configured", async () => { + const { ui, wait, openNew } = await render(); + await openNew(); + await wait(() => ui.getByText(/use the default destination/)); + }); + + it.each([["not-an-address", /is not an email address/], ["a@e.co b@e.co c@e.co d@e.co", /at most 3/]])("blocks saving %s", async (value, message) => { + const { ui, wait, input, openNew } = await render(); + await openNew(); + await wait(() => expect(field(ui).disabled).toBe(false)); + await input(/^Name/, "Errors"); + await input(/^Expression/, "true"); + await input(/^Email recipients/, value); + await wait(() => ui.getByText(message)); + expect((ui.getByRole("button", { name: "Create rule" }) as HTMLButtonElement).disabled).toBe(true); + }); + + it("disables the option and says why without SMTP", async () => { + options = unavailable; + const { ui, wait, openNew } = await render(); + await openNew(); + await wait(() => ui.getByText(smtpMessage)); + expect(field(ui).disabled).toBe(true); + }); + + it("keeps a saved rule editable without SMTP so its recipients can be removed", async () => { + options = unavailable; + rules = [{ ...rule, email_recipients: ["a@example.com"], email_status: "unavailable", email_status_detail: smtpMessage }]; + const { ui, wait } = await render(); + await wait(() => ui.getByRole("button", { name: "Checkout errors actions" })); + await act(async () => { fireEvent.click(ui.getByRole("button", { name: "Checkout errors actions" })); }); + await wait(() => ui.getByRole("menuitem", { name: "Edit" })); + await act(async () => { fireEvent.click(ui.getByRole("menuitem", { name: "Edit" })); }); + await wait(() => expect(field(ui).value).toBe("a@example.com")); + expect(field(ui).disabled).toBe(false); + }); + + it.each([ + ["failed", "Email failed", "1 of 1 recipients failed: 550 no such user"], + ["unavailable", "Email unavailable", smtpMessage], + ["suppressed", "Email suppressed", "1 of 1 recipients suppressed: Over the limit of 30 emails per hour to this address"], + ["retrying", "Email retrying", "Attempt 1 of 5 failed: 451"], + ])("shows a %s email on the rule", async (status, label, detail) => { + rules = [{ ...rule, email_recipients: ["a@example.com"], email_status: status, email_status_detail: detail }]; + const { ui, wait } = await render(); + await wait(() => ui.getByText(label)); + ui.getByText(detail); + }); + + it("shows nothing on the rule when email was sent", async () => { + rules = [{ ...rule, email_recipients: ["a@example.com"], email_status: "sent" }]; + const { ui, wait } = await render(); + await wait(() => ui.getByText("Checkout errors")); + expect(ui.queryByText(/^Email /)).toBeNull(); + }); + + it("does not send server-owned status back on save", async () => { + rules = [{ ...rule, email_recipients: ["a@example.com"], email_status: "failed", email_status_detail: "x", email_status_at: "t" }]; + const { ui, wait } = await render(); + await wait(() => ui.getByRole("switch", { name: "Enable Checkout errors" })); + await act(async () => { fireEvent.click(ui.getByRole("switch", { name: "Enable Checkout errors" })); }); + await wait(() => expect(fetchMock.mock.calls.some(([, init]) => init?.method === "PUT")).toBe(true)); + const body = JSON.parse(String(fetchMock.mock.calls.find(([, init]) => init?.method === "PUT")?.[1]?.body)); + expect(body.email_recipients).toEqual(["a@example.com"]); + expect(body).not.toHaveProperty("email_status"); + expect(body).not.toHaveProperty("email_status_detail"); + expect(body).not.toHaveProperty("email_status_at"); + }); +}); diff --git a/ui/host/src/alerts/page.tsx b/ui/host/src/alerts/page.tsx index 84f372bd7..4fd563483 100644 --- a/ui/host/src/alerts/page.tsx +++ b/ui/host/src/alerts/page.tsx @@ -28,6 +28,14 @@ function ruleQuery(rule: Pick): string { /** Rules read per-service rollups with error_rate from 0 to 1; Explore reads raw telemetry and shows percent. */ const exploreNote = "Open in Explore charts raw telemetry, so values can differ slightly from a rule's rollups. Explore shows error rate as a percent: a rule's 0.05 is 5 there."; +/** Email outcomes that need attention on a rule; a sent email does not. */ +const emailIssues: Record = { + failed: { tone: "danger", label: "Email failed" }, + unavailable: { tone: "danger", label: "Email unavailable" }, + suppressed: { tone: "warning", label: "Email suppressed" }, + retrying: { tone: "warning", label: "Email retrying" }, +}; + /** Alert rules and their history (proposal 0004, section 4); works without chat. */ export function AlertsPage({ onExplore }: { onExplore(query: string): void }) { const client = useQueryClient(); @@ -80,7 +88,8 @@ export function AlertsPage({ onExplore }: { onExplore(query: string): void }) { {rules.data?.map(rule => { const now = active(rule); return - {rule.name}{rule.description && {rule.description}} + {rule.name}{rule.description && {rule.description}} + {rule.email_status && emailIssues[rule.email_status] && {emailIssues[rule.email_status].label}{rule.email_status_detail}} {rule.service || "All services"} {rule.expression} @@ -141,6 +150,6 @@ export function AlertsPage({ onExplore }: { onExplore(query: string): void }) { } function draftFields(rule: Rule): RuleDraft { - const { id: _id, created_at: _created, updated_at: _updated, ...draft } = rule; + const { id: _id, created_at: _created, updated_at: _updated, email_status: _status, email_status_detail: _detail, email_status_at: _at, ...draft } = rule; return draft; } diff --git a/ui/host/src/alerts/rule-form.tsx b/ui/host/src/alerts/rule-form.tsx index 973c99d97..2a4408ac8 100644 --- a/ui/host/src/alerts/rule-form.tsx +++ b/ui/host/src/alerts/rule-form.tsx @@ -3,7 +3,7 @@ import { Box, Autocomplete, Button, Group, Menu, Modal, NumberInput, Stack, Swi import { useQuery } from "@tanstack/react-query"; import { useState } from "react"; import { useDiscard } from "../dashboards/discard"; -import { listServices, type Rule, type RuleDraft } from "./api"; +import { emailOptionsKey, getEmailOptions, listServices, type Rule, type RuleDraft } from "./api"; import { percentMistake } from "./values"; /** Starters for the documented variables (site docs: reference/alert-expressions). */ @@ -18,7 +18,7 @@ const blank: RuleDraft = { name: "", description: "", enabled: true, service: "" function draftOf(rule?: Rule): RuleDraft { if (!rule) return blank; - const { id: _id, created_at: _created, updated_at: _updated, ...draft } = rule; + const { id: _id, created_at: _created, updated_at: _updated, email_status: _status, email_status_detail: _detail, email_status_at: _at, ...draft } = rule; return { ...blank, ...draft }; } @@ -41,6 +41,16 @@ function headersProblem(headers: string | undefined): string | undefined { return undefined; } +const parseRecipients = (text: string) => text.split(/[,\s]+/).filter(Boolean); +/** A bare address; the server does the full check (internal/auth/email.go NormalizeRecipients). */ +const bareAddress = /^[^\s@,<>]+@[^\s@,<>]+$/; + +function recipientsProblem(recipients: string[], max: number): string | undefined { + const invalid = recipients.find(recipient => !bareAddress.test(recipient)); + if (invalid) return `"${invalid}" is not an email address`; + return new Set(recipients.map(r => r.toLowerCase())).size > max ? `Enter at most ${max} addresses` : undefined; +} + /** Creates or replaces a rule; the server checks the expression and returns why it is invalid. */ export function RuleForm({ rule, onClose, onSave }: { rule?: Rule; onClose(): void; onSave(draft: RuleDraft): Promise }) { const [draft, setDraft] = useState(() => draftOf(rule)); @@ -50,7 +60,13 @@ export function RuleForm({ rule, onClose, onSave }: { rule?: Rule; onClose(): vo const set = (patch: Partial) => { setDraft(current => ({ ...current, ...patch })); setError(undefined); }; // Like the server, check headers only when they change, so other edits still save. const headers = draft.webhook_headers === rule?.webhook_headers ? undefined : headersProblem(draft.webhook_headers); - const ready = draft.name.trim() !== "" && draft.expression.trim() !== "" && !headers; + const email = useQuery({ queryKey: emailOptionsKey, queryFn: ({ signal }) => getEmailOptions(signal), staleTime: 60_000, retry: false }); + const [emailText, setEmailText] = useState(() => (rule?.email_recipients ?? []).join(", ")); + const savedRecipients = (rule?.email_recipients?.length ?? 0) > 0; + const recipientsChanged = JSON.stringify(draft.email_recipients ?? []) !== JSON.stringify(rule?.email_recipients ?? []); + // Like the server, check recipients only when they change, so a rule stays editable after SMTP is removed. + const recipients = recipientsChanged ? recipientsProblem(draft.email_recipients ?? [], email.data?.max_recipients ?? 10) : undefined; + const ready = draft.name.trim() !== "" && draft.expression.trim() !== "" && !headers && !recipients; // A misspelled service saves, but would never fire; say so without blocking. const services = useQuery({ queryKey: ["alerting", "services"], queryFn: ({ signal }) => listServices(signal), staleTime: 60_000, retry: false }); const service = draft.service?.trim() ?? ""; @@ -90,6 +106,13 @@ export function RuleForm({ rule, onClose, onSave }: { rule?: Rule; onClose(): vo {seconds("Cooldown (s)", "cooldown_s", "Quiet period after resolving")} {seconds("Repeat every (s)", "repeat_interval_s", "0 delivers once")} + + { const text = event.currentTarget.value, parsed = parseRecipients(text); setEmailText(text); set({ email_recipients: parsed.length > 0 ? parsed : undefined }); }} /> + {email.data && !email.data.available && {email.data.message}} + {email.isError && Email availability could not be checked.} + set({ webhook_url: event.currentTarget.value })} />