diff --git a/.env.example b/.env.example index 4429cc6a8..4fb911326 100644 --- a/.env.example +++ b/.env.example @@ -34,6 +34,16 @@ # Default: (empty) # FANOUT_AI_PROVIDER= +# FANOUT_ALERTS_EMAIL_DEFAULT_TO; string +# A comma-separated list of addresses that receive alerts from rules with no webhook and no email recipients of their own; requires SMTP. +# Default: (empty) +# FANOUT_ALERTS_EMAIL_DEFAULT_TO= + +# FANOUT_ALERTS_EMAIL_HOURLY_LIMIT; integer +# Caps the alert emails sent to one address in any rolling hour, across all rules. Notifications over the limit are suppressed, not delayed, and shown on the rule; 0 suppresses all alert email. +# Default: 30 +# FANOUT_ALERTS_EMAIL_HOURLY_LIMIT= + # FANOUT_ALERTS_ENABLED; boolean # Runs the alert evaluation engine. # Default: true diff --git a/cmd/fanout-docgen/main.go b/cmd/fanout-docgen/main.go index 8154adc0c..5ca60d46c 100644 --- a/cmd/fanout-docgen/main.go +++ b/cmd/fanout-docgen/main.go @@ -500,9 +500,9 @@ var groups = map[string]groupPage{ }, "alerts": { Title: "Alert settings", - Description: "Whether the alert engine runs, how often it evaluates, and how long history is kept.", - Summary: "Settings for the alert evaluation loop.", - ReadWhen: []string{"Alerts are evaluating too slowly, or not at all."}, + Description: "Whether the alert engine runs, how often it evaluates, how long history is kept, and how alert email is addressed and limited.", + Summary: "Settings for the alert evaluation loop and alert email.", + ReadWhen: []string{"Alerts are evaluating too slowly, or not at all.", "You are setting a default alert email destination or changing the per-address limit."}, }, "metrics": { Title: "Metrics settings", diff --git a/cmd/fanout/main.go b/cmd/fanout/main.go index 3d344cb75..998c72927 100644 --- a/cmd/fanout/main.go +++ b/cmd/fanout/main.go @@ -194,6 +194,14 @@ func main() { os.Exit(1) } + smtpCfg := auth.SMTPConfig{ + Host: cfg.SMTPHost, + Port: cfg.SMTPPort, + User: cfg.SMTPUser, + Pass: cfg.SMTPPass, + From: cfg.SMTPFrom, + } + // Start alert engine var alertStore *alert.Store var alertEngine *alert.Engine @@ -204,6 +212,18 @@ func main() { cfg.AlertEvaluationInterval, cfg.AlertHistoryDays, ) + if cfg.SMTPConfigured() { + defaults, err := cfg.AlertEmailDefaults() + if err != nil { + slog.Error("alert email defaults invalid", "err", err) + os.Exit(1) + } + alertEngine.EnableEmail(alert.EmailConfig{ + Send: func(ctx context.Context, mail auth.AlertMail) error { return auth.SendAlert(ctx, smtpCfg, mail) }, + Defaults: defaults, + HourlyLimit: cfg.AlertEmailHourlyLimit, + }) + } go alertEngine.Run(ctx) slog.Info("alert engine enabled", "interval", cfg.AlertEvaluationInterval) } @@ -299,13 +319,6 @@ func main() { api.RegisterAlertRoutes(e, alertStore, alertEngine) // Auth routes (web-only setup + email-code login) - smtpCfg := auth.SMTPConfig{ - Host: cfg.SMTPHost, - Port: cfg.SMTPPort, - User: cfg.SMTPUser, - Pass: cfg.SMTPPass, - From: cfg.SMTPFrom, - } codeStore := auth.NewCodeStore(sqlite.DB, cfg.AuthCodeSecret) api.RegisterAuthRoutes(e, userStore, codeStore, setup, settingsStore, browserSessions, auditStore, smtpCfg, cfg) oidcDiscovery, err := api.RegisterOIDCRoutes(ctx, e, cfg, userStore, identityStore, browserSessions, auditStore) diff --git a/internal/alert/email.go b/internal/alert/email.go new file mode 100644 index 000000000..54249d066 --- /dev/null +++ b/internal/alert/email.go @@ -0,0 +1,594 @@ +package alert + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "log/slog" + "runtime/debug" + "slices" + "strings" + "time" + + "github.com/labstack/fanout/internal/auth" + "github.com/labstack/fanout/internal/db/generated" +) + +// Rule email statuses, reported on Rule.EmailStatus. Delivery rows use sent, +// failed and suppressed as their terminal states. +const ( + emailSent = "sent" + emailRetrying = "retrying" + emailFailed = "failed" + emailSuppressed = "suppressed" + emailUnavailable = "unavailable" + + emailPending = "pending" + emailSending = "sending" +) + +// EmailUnavailableMessage explains why email cannot be chosen or delivered. +const EmailUnavailableMessage = "Email delivery is unavailable because SMTP is not configured. Set FANOUT_SMTP_HOST, FANOUT_SMTP_USERNAME, FANOUT_SMTP_PASSWORD and FANOUT_SMTP_FROM." + +const ( + emailBatchSize = 50 + emailPollInterval = 2 * time.Second + emailSendTimeout = 30 * time.Second + emailLimitWindow = time.Hour +) + +// emailBackoff holds the wait before each attempt. Mail servers defer delivery +// for minutes at a time (greylisting, brief outages), so it runs longer than +// the webhook's in-memory 2s and 4s retries. +var emailBackoff = []time.Duration{0, time.Minute, 5 * time.Minute, 15 * time.Minute, 30 * time.Minute} + +// EmailConfig enables alert email on an Engine. +type EmailConfig struct { + Send func(context.Context, auth.AlertMail) error + Defaults []string // Addresses for rules with no destination of their own. + HourlyLimit int // Emails per address per rolling hour, across all rules. +} + +// EmailOptions describes alert email on this instance to API clients. +type EmailOptions struct { + Available bool `json:"available"` + Message string `json:"message,omitempty"` + DefaultDestination bool `json:"default_destination"` + MaxRecipients int `json:"max_recipients"` + HourlyLimit int `json:"hourly_limit,omitempty"` +} + +// EnableEmail turns on alert email. Call it before Run. +func (e *Engine) EnableEmail(cfg EmailConfig) { + e.email = &cfg +} + +// EmailOptions reports whether email can be chosen. A nil engine means alerts +// are disabled. +func (e *Engine) EmailOptions() EmailOptions { + options := EmailOptions{MaxRecipients: auth.MaxAlertEmailRecipients} + if e == nil { + options.Message = "Alert delivery is unavailable because alerts are disabled on this instance." + return options + } + if e.email == nil { + options.Message = EmailUnavailableMessage + return options + } + options.Available = true + options.DefaultDestination = len(e.email.Defaults) > 0 + options.HourlyLimit = e.email.HourlyLimit + return options +} + +// WithEmailState adjusts a rule's reported email state for what this instance +// can do now: saved recipients without SMTP are a standing failure, and a rule +// with no email destination has no email state. +func (e *Engine) WithEmailState(rule Rule) Rule { + if e == nil || e.email == nil { + rule.EmailStatus, rule.EmailStatusDetail, rule.EmailStatusAt = "", "", "" + if len(rule.EmailRecipients) > 0 { + rule.EmailStatus = emailUnavailable + rule.EmailStatusDetail = e.EmailOptions().Message + } + return rule + } + if len(e.emailRecipients(rule)) == 0 { + rule.EmailStatus, rule.EmailStatusDetail, rule.EmailStatusAt = "", "", "" + } + return rule +} + +// emailRecipients resolves where a rule's notifications are mailed. The default +// destination applies only to a rule with neither a webhook nor recipients. +func (e *Engine) emailRecipients(rule Rule) []string { + if e.email == nil { + return nil + } + if len(rule.EmailRecipients) > 0 { + return rule.EmailRecipients + } + if strings.TrimSpace(rule.WebhookURL) == "" { + return e.email.Defaults + } + return nil +} + +// saveAlert persists a transition. A non-empty event also queues its email in +// the same transaction, so a crash cannot keep one without the other. +func (e *Engine) saveAlert(rule Rule, a Alert, event string) (Alert, error) { + recipients := e.emailRecipients(rule) + if event == "" || len(recipients) == 0 { + return e.store.UpsertAlert(a) + } + now := e.clock() + return e.store.UpsertAlertWithEmail(a, recipients, emailContent{ + Rule: rule.Name, + Description: rule.Description, + Service: a.Service, + Event: event, + Value: a.Value, + Time: now, + }, now) +} + +func (e *Engine) clock() time.Time { + if e.now != nil { + return e.now() + } + return time.Now() +} + +// emailContent is what a delivery row keeps to render its message. +type emailContent struct { + Rule string `json:"rule"` + Description string `json:"description,omitempty"` + Service string `json:"service"` + Event string `json:"event"` + Value float64 `json:"value"` + Time time.Time `json:"time"` +} + +// UpsertAlertWithEmail saves an alert and queues one delivery per recipient, +// atomically. +func (s *Store) UpsertAlertWithEmail(a Alert, recipients []string, content emailContent, now time.Time) (Alert, error) { + ctx := context.Background() + payload, err := json.Marshal(content) + if err != nil { + return Alert{}, fmt.Errorf("store: encode email: %w", err) + } + notificationID, err := newID() + if err != nil { + return Alert{}, err + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return Alert{}, fmt.Errorf("store: begin: %w", err) + } + defer func() { _ = tx.Rollback() }() + q := s.q.WithTx(tx) + saved, err := upsertAlert(q, a) + if err != nil { + return Alert{}, err + } + at := formatStoredTime(now) + for _, recipient := range recipients { + id, err := newID() + if err != nil { + return Alert{}, err + } + if err := q.InsertAlertEmailDelivery(ctx, generated.InsertAlertEmailDeliveryParams{ + ID: id, + NotificationID: notificationID, + RuleID: saved.RuleID, + AlertID: saved.ID, + Recipient: recipient, + Content: string(payload), + NextAttemptAt: at, + CreatedAt: at, + }); err != nil { + return Alert{}, fmt.Errorf("store: queue email: %w", err) + } + } + if err := tx.Commit(); err != nil { + return Alert{}, fmt.Errorf("store: commit: %w", err) + } + return saved, nil +} + +func encodeRecipients(recipients []string) sql.NullString { + if len(recipients) == 0 { + return sql.NullString{} + } + encoded, err := json.Marshal(recipients) + if err != nil { + return sql.NullString{} + } + return sql.NullString{String: string(encoded), Valid: true} +} + +func decodeRecipients(ruleID string, stored sql.NullString) []string { + if !stored.Valid || stored.String == "" { + return nil + } + var recipients []string + if err := json.Unmarshal([]byte(stored.String), &recipients); err != nil { + slog.Error("alert: invalid stored email recipients", "rule", ruleID, "err", err) + return nil + } + return recipients +} + +// runEmail delivers queued mail until ctx ends. One worker sends sequentially, +// which keeps the per-recipient count exact. +func (e *Engine) runEmail(ctx context.Context) { + ticker := time.NewTicker(emailPollInterval) + defer ticker.Stop() + for { + e.safeDrainEmail(ctx) + select { + case <-ticker.C: + case <-ctx.Done(): + return + } + } +} + +// safeDrainEmail keeps a panic from ending the worker. A row it left marked +// sending is recovered by the next drain. +func (e *Engine) safeDrainEmail(ctx context.Context) { + defer func() { + if r := recover(); r != nil { + slog.Error("alert: email worker panic", "panic", r, "stack", string(debug.Stack())) + } + }() + e.drainEmail(ctx) +} + +// emailOutcome is the bookkeeping for a finished attempt. One that the database +// refuses is held and retried by later drains, so a transient database error +// cannot strand the row or cause mail the server accepted to be sent again. +type emailOutcome struct { + row generated.AlertEmailDelivery + apply func(context.Context) error + after func(context.Context) +} + +// recordOutcome applies an outcome, holding it on failure. It reports success. +func (e *Engine) recordOutcome(ctx context.Context, outcome emailOutcome) bool { + if err := outcome.apply(ctx); err != nil { + slog.Error("alert: record email outcome", "rule", outcome.row.RuleID, "err", err) + if e.emailHeld == nil { + e.emailHeld = make(map[string]emailOutcome) + } + e.emailHeld[outcome.row.ID] = outcome + return false + } + delete(e.emailHeld, outcome.row.ID) + if outcome.after != nil { + outcome.after(ctx) + } + return true +} + +// finishOutcome moves a row to status and then runs after. +func (e *Engine) finishOutcome(row generated.AlertEmailDelivery, status, detail, nextAttemptAt string, after func(context.Context)) emailOutcome { + return emailOutcome{ + row: row, + apply: func(ctx context.Context) error { + return e.finishEmail(ctx, row, status, detail, nextAttemptAt) + }, + after: after, + } +} + +func (e *Engine) settleAfter(row generated.AlertEmailDelivery) func(context.Context) { + return func(ctx context.Context) { e.settleEmail(ctx, row) } +} + +// recoverEmail handles rows left marked sending by a restart or a panic. Such a +// row may or may not have reached the server, so it is sent again: delivery is +// at least once, and the repeated Message-ID lets a mail system collapse the +// duplicate. It runs only between attempts, so no row is in flight, and rows +// with a held outcome are skipped because their result is already known. +func (e *Engine) recoverEmail(ctx context.Context) { + rows, err := e.store.q.ListSendingAlertEmailDeliveries(ctx) + if err != nil { + if ctx.Err() == nil { + slog.Error("alert: list interrupted email", "err", err) + } + return + } + now := formatStoredTime(e.clock()) + for _, row := range rows { + if _, held := e.emailHeld[row.ID]; held { + continue + } + if int(row.Attempts) >= len(emailBackoff) { + e.recordOutcome(ctx, e.finishOutcome(row, emailFailed, "Interrupted during its last attempt", row.NextAttemptAt, e.settleAfter(row))) + continue + } + e.recordOutcome(ctx, e.finishOutcome(row, emailPending, row.LastError, now, nil)) + } +} + +// failUndeliverableEmail settles mail left queued by a run that had SMTP, so it +// shows as failed instead of waiting forever or going out long after the fact. +func (e *Engine) failUndeliverableEmail(ctx context.Context) { + rows, err := e.store.q.ListOpenAlertEmailDeliveries(ctx) + if err != nil { + slog.Error("alert: list open email", "err", err) + return + } + for _, row := range rows { + e.recordOutcome(ctx, e.finishOutcome(row, emailFailed, EmailUnavailableMessage, row.NextAttemptAt, e.settleAfter(row))) + } +} + +// drainEmail processes everything currently due, oldest first. +func (e *Engine) drainEmail(ctx context.Context) { + for _, outcome := range e.emailHeld { + e.recordOutcome(context.WithoutCancel(ctx), outcome) + } + e.recoverEmail(ctx) + for ctx.Err() == nil { + rows, err := e.store.q.ListDueAlertEmailDeliveries(ctx, generated.ListDueAlertEmailDeliveriesParams{ + NextAttemptAt: formatStoredTime(e.clock()), + Limit: emailBatchSize, + }) + if err != nil { + if ctx.Err() == nil { + slog.Error("alert: list due email", "err", err) + } + return + } + if len(rows) == 0 { + return + } + attempted := false + for _, row := range rows { + if _, held := e.emailHeld[row.ID]; held { + continue + } + attempted = true + if ctx.Err() != nil || !e.deliverEmail(ctx, row) { + return + } + } + if !attempted || len(rows) < emailBatchSize { + return + } + } +} + +// deliverEmail makes one attempt, or suppresses the row if its recipient has +// reached the hourly limit. Suppressed mail is dropped rather than delayed, so +// a burst cannot arrive later as a backlog. It returns false when the database +// failed and the row is still due, so the drain stops until the next poll. +func (e *Engine) deliverEmail(ctx context.Context, row generated.AlertEmailDelivery) bool { + if row.Attempts > 0 { + addressed, err := e.stillAddressed(row) + if err != nil { + slog.Error("alert: read rule for email retry", "rule", row.RuleID, "err", err) + return false + } + if !addressed { + if err := e.store.q.DeleteAlertEmailDelivery(ctx, row.ID); err != nil { + slog.Error("alert: drop email delivery", "rule", row.RuleID, "err", err) + return false + } + e.settleEmail(ctx, row) + return true + } + } + + now := e.clock() + sent, err := e.store.q.CountAlertEmailSendsSince(ctx, generated.CountAlertEmailSendsSinceParams{ + Recipient: row.Recipient, + SentAt: formatStoredTime(now.Add(-emailLimitWindow)), + }) + if err != nil { + slog.Error("alert: count sent email", "err", err) + return false + } + limit := e.email.HourlyLimit + if int(sent) >= limit { + slog.Warn("alert: email suppressed by hourly limit", "rule", row.RuleID, "alert", row.AlertID, "limit", limit) + return e.recordOutcome(ctx, e.finishOutcome(row, emailSuppressed, fmt.Sprintf("Over the limit of %d emails per hour to this address", limit), row.NextAttemptAt, e.settleAfter(row))) + } + + var content emailContent + if err := json.Unmarshal([]byte(row.Content), &content); err != nil { + return e.recordOutcome(ctx, e.finishOutcome(row, emailFailed, "The stored message could not be read", row.NextAttemptAt, e.settleAfter(row))) + } + claimed, err := e.store.q.ClaimAlertEmailDelivery(ctx, row.ID) + if err != nil { + slog.Error("alert: claim email", "err", err) + return false + } + if claimed == 0 { + return true // Deleted with its rule since the batch was read. + } + attempt := int(row.Attempts) + 1 + + mail := auth.AlertMail{ + ID: row.ID, + To: row.Recipient, + Rule: content.Rule, + Description: content.Description, + Service: content.Service, + Event: content.Event, + Value: content.Value, + Time: content.Time, + } + if int(sent) == limit-1 { + mail.LimitNotice = fmt.Sprintf("This address has reached the limit of %d alert emails per hour. Further alerts are recorded on their rules in Fanout and not emailed until the count falls.", limit) + } + sendCtx, cancel := context.WithTimeout(ctx, emailSendTimeout) + err = e.email.Send(sendCtx, mail) + cancel() + + detail := "" + if err != nil { + detail = auth.MailFailureDetail(err) + } + // The outcome is recorded even if shutdown began during the send. + record := context.WithoutCancel(ctx) + switch { + case err == nil: + sentAt := e.clock() + return e.recordOutcome(record, emailOutcome{ + row: row, + apply: func(c context.Context) error { return e.store.recordEmailSent(c, row, sentAt) }, + after: e.settleAfter(row), + }) + case ctx.Err() != nil: + // Shutting down: leave the row marked sending for recoverEmail. + case auth.IsPermanentMailError(err) || attempt >= len(emailBackoff): + slog.Warn("alert: email delivery failed", "rule", row.RuleID, "alert", row.AlertID, "attempt", attempt, "reason", detail) + return e.recordOutcome(record, e.finishOutcome(row, emailFailed, detail, row.NextAttemptAt, e.settleAfter(row))) + default: + next := e.clock().Add(emailBackoff[attempt]) + slog.Warn("alert: email attempt failed, will retry", "rule", row.RuleID, "alert", row.AlertID, "attempt", attempt, "reason", detail) + status := fmt.Sprintf("Attempt %d of %d failed: %s", attempt, len(emailBackoff), detail) + return e.recordOutcome(record, e.finishOutcome(row, emailPending, detail, formatStoredTime(next), func(c context.Context) { + e.setRuleEmailStatus(c, row.RuleID, emailRetrying, status) + })) + } + return true +} + +// stillAddressed reports whether a retry's recipient is still a destination of +// its rule. +func (e *Engine) stillAddressed(row generated.AlertEmailDelivery) (bool, error) { + rule, err := e.store.GetRule(row.RuleID) + if err != nil { + if strings.Contains(err.Error(), "not found") { + return false, nil + } + return false, err + } + return slices.Contains(e.emailRecipients(rule), row.Recipient), nil +} + +// recordEmailSent marks a row sent and counts it toward the hourly limit, +// atomically. +func (s *Store) recordEmailSent(ctx context.Context, row generated.AlertEmailDelivery, now time.Time) error { + id, err := newID() + if err != nil { + return err + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + q := s.q.WithTx(tx) + if err := q.FinishAlertEmailDelivery(ctx, generated.FinishAlertEmailDeliveryParams{ + Status: emailSent, + NextAttemptAt: row.NextAttemptAt, + ID: row.ID, + }); err != nil { + return err + } + if err := q.RecordAlertEmailSend(ctx, generated.RecordAlertEmailSendParams{ + ID: id, + Recipient: row.Recipient, + SentAt: formatStoredTime(now), + }); err != nil { + return err + } + return tx.Commit() +} + +func (e *Engine) finishEmail(ctx context.Context, row generated.AlertEmailDelivery, status, detail, nextAttemptAt string) error { + return e.store.q.FinishAlertEmailDelivery(ctx, generated.FinishAlertEmailDeliveryParams{ + Status: status, + LastError: detail, + NextAttemptAt: nextAttemptAt, + ID: row.ID, + }) +} + +// settleEmail publishes a notification's result once every recipient is +// finished: failure outranks suppression, and suppression outranks success. A +// rule without a webhook also reports it on the alert, where the webhook +// result would otherwise go. +func (e *Engine) settleEmail(ctx context.Context, row generated.AlertEmailDelivery) { + rows, err := e.store.q.ListAlertEmailDeliveriesByNotification(ctx, row.NotificationID) + if err != nil { + slog.Error("alert: read notification", "rule", row.RuleID, "err", err) + return + } + if len(rows) == 0 { + // Every recipient was dropped, or the rule went with them. + e.setRuleEmailStatus(ctx, row.RuleID, "", "") + return + } + var failed, suppressed int + var cause string + for _, r := range rows { + switch r.Status { + case emailPending, emailSending: + return + case emailFailed: + failed++ + if cause == "" { + cause = r.LastError + } + case emailSuppressed: + suppressed++ + if cause == "" { + cause = r.LastError + } + } + } + + status, alertStatus, detail := emailSent, "success", "" + switch { + case failed > 0: + status, alertStatus = emailFailed, "failed" + detail = fmt.Sprintf("%d of %d recipients failed: %s", failed, len(rows), cause) + case suppressed > 0: + status, alertStatus = emailSuppressed, "suppressed" + detail = fmt.Sprintf("%d of %d recipients suppressed: %s", suppressed, len(rows), cause) + } + e.setRuleEmailStatus(ctx, row.RuleID, status, detail) + + rule, err := e.store.GetRule(row.RuleID) + if err != nil { + slog.Error("alert: read rule for email status", "rule", row.RuleID, "err", err) + return + } + if strings.TrimSpace(rule.WebhookURL) == "" { + if err := e.store.UpdateDeliveryStatus(row.AlertID, alertStatus, e.clock().Format(time.RFC3339)); err != nil { + slog.Error("alert: update delivery status", "rule", row.RuleID, "err", err) + } + } +} + +func (e *Engine) setRuleEmailStatus(ctx context.Context, ruleID, status, detail string) { + at := "" + if status != "" { + at = e.clock().Format(time.RFC3339) + } + if err := e.store.q.UpdateRuleEmailStatus(ctx, generated.UpdateRuleEmailStatusParams{ + EmailStatus: status, + EmailStatusDetail: detail, + EmailStatusAt: at, + ID: ruleID, + }); err != nil { + slog.Error("alert: update rule email status", "rule", ruleID, "err", err) + } +} + +// pruneEmailDeliveries removes finished rows older than the history window, +// and sends that have aged out of the hourly limit. +func (s *Store) pruneEmailDeliveries(days int, now time.Time) error { + ctx := context.Background() + if _, err := s.q.PruneAlertEmailDeliveries(ctx, formatStoredTime(now.AddDate(0, 0, -days))); err != nil { + return err + } + return s.q.PruneAlertEmailSends(ctx, formatStoredTime(now.Add(-emailLimitWindow))) +} diff --git a/internal/alert/email_test.go b/internal/alert/email_test.go new file mode 100644 index 000000000..ca5a223ff --- /dev/null +++ b/internal/alert/email_test.go @@ -0,0 +1,858 @@ +package alert + +import ( + "bufio" + "context" + "errors" + "fmt" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + mail "github.com/wneessen/go-mail" + + "github.com/labstack/fanout/internal/auth" +) + +// outbox records sends and fails them on demand. +type outbox struct { + mu sync.Mutex + mails []auth.AlertMail + fail func(auth.AlertMail) error +} + +func (o *outbox) send(_ context.Context, m auth.AlertMail) error { + o.mu.Lock() + defer o.mu.Unlock() + if o.fail != nil { + if err := o.fail(m); err != nil { + return err + } + } + o.mails = append(o.mails, m) + return nil +} + +func (o *outbox) to(address string) []auth.AlertMail { + o.mu.Lock() + defer o.mu.Unlock() + var matched []auth.AlertMail + for _, m := range o.mails { + if m.To == address { + matched = append(matched, m) + } + } + return matched +} + +type mailHarness struct { + engine *Engine + store *Store + box *outbox + now time.Time +} + +func newMailHarness(t *testing.T, limit int, defaults ...string) *mailHarness { + t.Helper() + engine, store := newTestEngine(t) + h := &mailHarness{engine: engine, store: store, box: &outbox{}, now: time.Now().UTC()} + engine.now = func() time.Time { return h.now } + engine.EnableEmail(EmailConfig{Send: h.box.send, Defaults: defaults, HourlyLimit: limit}) + return h +} + +func (h *mailHarness) rule(t *testing.T, name string, recipients ...string) Rule { + t.Helper() + rule, err := h.store.CreateRule(Rule{Name: name, Enabled: true, Service: "api", Expression: "error_rate > 0.1", RepeatIntervalS: 3600, EmailRecipients: recipients}) + if err != nil { + t.Fatal(err) + } + return rule +} + +func (h *mailHarness) fire(rule Rule) { + h.engine.transition(rule, "api", true, Env{Service: "api", ErrorRate: 0.5}, h.now) +} + +func (h *mailHarness) drain() { h.engine.drainEmail(context.Background()) } + +func (h *mailHarness) ruleStatus(t *testing.T, id string) Rule { + t.Helper() + rule, err := h.store.GetRule(id) + if err != nil { + t.Fatal(err) + } + return rule +} + +func TestEmailReachesEveryRecipientAndReportsSent(t *testing.T) { + h := newMailHarness(t, 30) + rule := h.rule(t, "errors", "a@example.com", "b@example.com") + h.fire(rule) + h.drain() + if len(h.box.to("a@example.com")) != 1 || len(h.box.to("b@example.com")) != 1 { + t.Fatalf("mails = %+v", h.box.mails) + } + got := h.box.to("a@example.com")[0] + if got.Rule != "errors" || got.Service != "api" || got.Event != "firing" || got.Value != 0.5 { + t.Fatalf("mail = %+v", got) + } + if status := h.ruleStatus(t, rule.ID).EmailStatus; status != "sent" { + t.Fatalf("rule status = %q", status) + } + alert, err := h.store.GetAlert(rule.ID, "api") + if err != nil || alert.LastDeliveryStatus != "success" { + t.Fatalf("alert = %+v, %v", alert, err) + } +} + +func TestDefaultDestinationAppliesOnlyToRulesWithoutOne(t *testing.T) { + h := newMailHarness(t, 30, "oncall@example.com") + bare := h.rule(t, "bare") + own := h.rule(t, "own", "owner@example.com") + webhook, err := h.store.CreateRule(Rule{Name: "hook", Enabled: true, Service: "api", Expression: "true", WebhookURL: "http://127.0.0.1:1/hook"}) + if err != nil { + t.Fatal(err) + } + for _, rule := range []Rule{bare, own, webhook} { + h.fire(rule) + } + h.drain() + if got := h.box.to("oncall@example.com"); len(got) != 1 || got[0].Rule != "bare" { + t.Fatalf("default destination got %+v", got) + } + if got := h.box.to("owner@example.com"); len(got) != 1 { + t.Fatalf("own recipient got %+v", got) + } +} + +// A rule row from before this feature has NULL recipients and must still deliver. +func TestOlderRuleWithoutEmailColumnsUsesDefaultDestination(t *testing.T) { + h := newMailHarness(t, 30, "oncall@example.com") + if _, err := h.store.db.Exec(`INSERT INTO alert_rules (id, name, expression, for_seconds) VALUES ('old', 'old rule', 'true', 0)`); err != nil { + t.Fatal(err) + } + rule := h.ruleStatus(t, "old") + if rule.EmailRecipients != nil || rule.EmailStatus != "" { + t.Fatalf("older rule = %+v", rule) + } + h.engine.transition(rule, "api", true, Env{Service: "api"}, h.now) + h.drain() + if got := h.box.to("oncall@example.com"); len(got) != 1 { + t.Fatalf("older rule mails = %+v", got) + } +} + +func TestEmailRetriesWithBackoffWithoutResendingToDelivered(t *testing.T) { + h := newMailHarness(t, 30) + rule := h.rule(t, "errors", "ok@example.com", "flaky@example.com") + failing := true + h.box.fail = func(m auth.AlertMail) error { + if m.To == "flaky@example.com" && failing { + return errors.New("451 try later") + } + return nil + } + h.fire(rule) + h.drain() + stored := h.ruleStatus(t, rule.ID) + if stored.EmailStatus != "retrying" || !strings.Contains(stored.EmailStatusDetail, "Attempt 1 of 5") || strings.Contains(stored.EmailStatusDetail, "try later") { + t.Fatalf("rule = %+v", stored) + } + + h.now = h.now.Add(30 * time.Second) + h.drain() + if len(h.box.to("flaky@example.com")) != 0 { + t.Fatal("retried before the backoff elapsed") + } + + failing = false + h.now = h.now.Add(31 * time.Second) + h.drain() + if len(h.box.to("flaky@example.com")) != 1 || len(h.box.to("ok@example.com")) != 1 { + t.Fatalf("mails = %+v", h.box.mails) + } + if status := h.ruleStatus(t, rule.ID).EmailStatus; status != "sent" { + t.Fatalf("rule status = %q", status) + } +} + +func TestExhaustedRetriesShowFailureOnRule(t *testing.T) { + h := newMailHarness(t, 30) + rule := h.rule(t, "errors", "a@example.com") + attempts := 0 + h.box.fail = func(auth.AlertMail) error { + attempts++ + return errors.New("connection refused") + } + h.fire(rule) + for range len(emailBackoff) + 2 { + h.drain() + h.now = h.now.Add(time.Hour) + } + if attempts != len(emailBackoff) { + t.Fatalf("attempts = %d, want %d", attempts, len(emailBackoff)) + } + stored := h.ruleStatus(t, rule.ID) + if stored.EmailStatus != "failed" || stored.EmailStatusDetail == "" || strings.Contains(stored.EmailStatusDetail, "connection refused") { + t.Fatalf("rule = %+v", stored) + } + alert, err := h.store.GetAlert(rule.ID, "api") + if err != nil || alert.LastDeliveryStatus != "failed" { + t.Fatalf("alert = %+v, %v", alert, err) + } +} + +func TestPermanentRejectionIsNotRetried(t *testing.T) { + h := newMailHarness(t, 30) + rule := h.rule(t, "errors", "gone@example.com") + attempts := 0 + h.box.fail = func(auth.AlertMail) error { + attempts++ + return fmt.Errorf("%w: rejected", auth.ErrInvalidMailAddress) + } + h.fire(rule) + h.drain() + h.now = h.now.Add(time.Hour) + h.drain() + if attempts != 1 { + t.Fatalf("attempts = %d, want 1", attempts) + } + if status := h.ruleStatus(t, rule.ID).EmailStatus; status != "failed" { + t.Fatalf("rule status = %q", status) + } +} + +func TestBurstAcrossRulesIsBoundedPerRecipientAndSuppressionIsVisible(t *testing.T) { + h := newMailHarness(t, 3) + var rules []Rule + for i := range 20 { + rules = append(rules, h.rule(t, fmt.Sprintf("rule-%d", i), "pager@example.com")) + } + quiet := h.rule(t, "quiet", "other@example.com") + for _, rule := range rules { + h.fire(rule) + h.now = h.now.Add(time.Millisecond) + } + h.fire(quiet) + h.now = h.now.Add(time.Millisecond) + h.drain() + + delivered := h.box.to("pager@example.com") + if len(delivered) != 3 { + t.Fatalf("delivered = %d, want 3", len(delivered)) + } + if delivered[1].LimitNotice != "" || !strings.Contains(delivered[2].LimitNotice, "limit of 3") { + t.Fatalf("limit notices = %q, %q", delivered[1].LimitNotice, delivered[2].LimitNotice) + } + if len(h.box.to("other@example.com")) != 1 { + t.Fatal("another recipient was affected by the limit") + } + suppressed := 0 + for _, rule := range rules { + stored := h.ruleStatus(t, rule.ID) + if stored.EmailStatus == "suppressed" { + suppressed++ + if !strings.Contains(stored.EmailStatusDetail, "limit of 3") { + t.Fatalf("detail = %q", stored.EmailStatusDetail) + } + } + } + if suppressed != 17 { + t.Fatalf("suppressed rules = %d, want 17", suppressed) + } + + // Suppressed mail never arrives later, and the window reopens. + h.now = h.now.Add(2 * time.Hour) + h.drain() + if got := len(h.box.to("pager@example.com")); got != 3 { + t.Fatalf("suppressed mail delivered later: %d", got) + } + late := h.rule(t, "late", "pager@example.com") + h.fire(late) + h.drain() + if got := len(h.box.to("pager@example.com")); got != 4 { + t.Fatalf("after window delivered = %d, want 4", got) + } +} + +func TestLimitSurvivesRestart(t *testing.T) { + h := newMailHarness(t, 1) + h.fire(h.rule(t, "one", "a@example.com")) + h.drain() + + restarted := &Engine{store: h.store, programs: map[string]Program{}, stopped: map[string]bool{}, now: h.engine.now} + box := &outbox{} + restarted.EnableEmail(EmailConfig{Send: box.send, HourlyLimit: 1}) + rule := h.rule(t, "two", "a@example.com") + restarted.transition(rule, "api", true, Env{Service: "api"}, h.now) + restarted.drainEmail(context.Background()) + if len(box.mails) != 0 { + t.Fatalf("limit reset by restart: %+v", box.mails) + } +} + +func TestDeletingARuleDoesNotResetTheHourlyLimit(t *testing.T) { + h := newMailHarness(t, 1) + first := h.rule(t, "first", "a@example.com") + h.fire(first) + h.drain() + if err := h.store.DeleteRule(first.ID); err != nil { + t.Fatal(err) + } + second := h.rule(t, "first", "a@example.com") + h.fire(second) + h.drain() + if got := h.box.to("a@example.com"); len(got) != 1 { + t.Fatalf("limit reset by deleting the rule: %+v", got) + } + if status := h.ruleStatus(t, second.ID).EmailStatus; status != "suppressed" { + t.Fatalf("rule status = %q", status) + } +} + +func TestDeletingAnAlertDoesNotResetTheHourlyLimit(t *testing.T) { + h := newMailHarness(t, 1) + rule := h.rule(t, "errors", "a@example.com") + h.fire(rule) + h.drain() + alert, err := h.store.GetAlert(rule.ID, "api") + if err != nil { + t.Fatal(err) + } + if err := h.store.DeleteAlert(alert.ID); err != nil { + t.Fatal(err) + } + h.fire(rule) + h.drain() + if got := h.box.to("a@example.com"); len(got) != 1 { + t.Fatalf("limit reset by deleting the alert: %+v", got) + } +} + +func TestDeletingARuleCancelsItsQueuedMail(t *testing.T) { + h := newMailHarness(t, 30) + rule := h.rule(t, "errors", "a@example.com") + h.fire(rule) + if err := h.store.DeleteRule(rule.ID); err != nil { + t.Fatal(err) + } + h.drain() + if len(h.box.mails) != 0 { + t.Fatalf("mails = %+v", h.box.mails) + } +} + +func TestPruningKeepsSendsInsideTheLimitWindow(t *testing.T) { + h := newMailHarness(t, 2) + h.fire(h.rule(t, "old", "a@example.com")) + h.drain() + h.now = h.now.Add(2 * time.Hour) + h.fire(h.rule(t, "recent", "a@example.com")) + h.drain() + if err := h.store.pruneEmailDeliveries(7, h.now); err != nil { + t.Fatal(err) + } + var kept int + if err := h.store.db.QueryRow(`SELECT COUNT(*) FROM alert_email_sends`).Scan(&kept); err != nil { + t.Fatal(err) + } + if kept != 1 { + t.Fatalf("sends kept = %d, want 1", kept) + } +} + +func TestStatusDetailLeavesOutServerTextAndAddresses(t *testing.T) { + h := newMailHarness(t, 30) + rule := h.rule(t, "errors", "a@example.com") + h.box.fail = func(auth.AlertMail) error { + return fmt.Errorf("auth: send email: %w: 550 private note for a@example.com", &mail.SendError{Reason: mail.ErrSMTPRcptTo}) + } + h.fire(rule) + h.drain() + stored := h.ruleStatus(t, rule.ID) + if stored.EmailStatus != "retrying" || !strings.Contains(stored.EmailStatusDetail, "RCPT TO") { + t.Fatalf("rule = %+v", stored) + } + for _, leaked := range []string{"private note", "a@example.com"} { + if strings.Contains(stored.EmailStatusDetail, leaked) { + t.Fatalf("detail leaks %q: %q", leaked, stored.EmailStatusDetail) + } + } +} + +func (h *mailHarness) sendsRecorded(t *testing.T) int { + t.Helper() + var n int + if err := h.store.db.QueryRow(`SELECT COUNT(*) FROM alert_email_sends`).Scan(&n); err != nil { + t.Fatal(err) + } + return n +} + +func (h *mailHarness) exec(t *testing.T, statement string) { + t.Helper() + if _, err := h.store.db.Exec(statement); err != nil { + t.Fatal(err) + } +} + +func TestAcceptedMailIsRecordedAfterADatabaseFailureWithoutResending(t *testing.T) { + h := newMailHarness(t, 30) + rule := h.rule(t, "errors", "a@example.com") + h.exec(t, `CREATE TRIGGER block_send BEFORE INSERT ON alert_email_sends BEGIN SELECT RAISE(ABORT, 'blocked'); END`) + h.fire(rule) + h.drain() + h.exec(t, `DROP TRIGGER block_send`) + h.now = h.now.Add(time.Minute) + h.drain() + h.drain() + if got := h.box.to("a@example.com"); len(got) != 1 { + t.Fatalf("sends = %d, want 1", len(got)) + } + if got := h.sendsRecorded(t); got != 1 { + t.Fatalf("recorded sends = %d, want 1", got) + } + if status := h.ruleStatus(t, rule.ID).EmailStatus; status != "sent" { + t.Fatalf("rule status = %q", status) + } +} + +func TestRetryIsScheduledAfterADatabaseFailureRecordingIt(t *testing.T) { + h := newMailHarness(t, 30) + rule := h.rule(t, "errors", "a@example.com") + attempts := 0 + h.box.fail = func(auth.AlertMail) error { attempts++; return errors.New("451 later") } + h.exec(t, `CREATE TRIGGER block_retry BEFORE UPDATE ON alert_email_deliveries WHEN NEW.status = 'pending' BEGIN SELECT RAISE(ABORT, 'blocked'); END`) + h.fire(rule) + h.drain() + h.exec(t, `DROP TRIGGER block_retry`) + h.drain() + if attempts != 1 { + t.Fatalf("attempts = %d, want 1 before the backoff elapses", attempts) + } + if status := h.ruleStatus(t, rule.ID).EmailStatus; status != "retrying" { + t.Fatalf("rule status = %q", status) + } + h.box.fail = nil + h.now = h.now.Add(2 * time.Minute) + h.drain() + if got := h.box.to("a@example.com"); len(got) != 1 { + t.Fatalf("mails = %+v", got) + } +} + +func TestShutdownJustAfterAnAcceptedSendKeepsTheRecord(t *testing.T) { + h := newMailHarness(t, 30) + rule := h.rule(t, "errors", "a@example.com") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + h.box.fail = func(auth.AlertMail) error { cancel(); return nil } + h.fire(rule) + h.engine.drainEmail(ctx) + if got := h.sendsRecorded(t); got != 1 { + t.Fatalf("recorded sends = %d, want 1", got) + } + if status := h.ruleStatus(t, rule.ID).EmailStatus; status != "sent" { + t.Fatalf("rule status = %q", status) + } +} + +func TestRowStrandedInSendingIsRecoveredWhileRunning(t *testing.T) { + h := newMailHarness(t, 30) + rule := h.rule(t, "errors", "a@example.com") + h.fire(rule) + h.exec(t, `UPDATE alert_email_deliveries SET status='sending', attempts=1`) + h.drain() + if got := h.box.to("a@example.com"); len(got) != 1 { + t.Fatalf("mails = %+v", got) + } + if status := h.ruleStatus(t, rule.ID).EmailStatus; status != "sent" { + t.Fatalf("rule status = %q", status) + } +} + +func TestRuleDeletedAfterTheBatchReadSendsNothing(t *testing.T) { + h := newMailHarness(t, 30) + rule := h.rule(t, "errors", "a@example.com") + h.fire(rule) + rows, err := h.store.q.ListOpenAlertEmailDeliveries(context.Background()) + if err != nil || len(rows) != 1 { + t.Fatalf("rows = %v, %v", rows, err) + } + if err := h.store.DeleteRule(rule.ID); err != nil { + t.Fatal(err) + } + h.engine.deliverEmail(context.Background(), rows[0]) + if len(h.box.mails) != 0 { + t.Fatalf("mailed after the rule was deleted: %+v", h.box.mails) + } +} + +func TestRetryDropsARecipientNoLongerOnTheRule(t *testing.T) { + h := newMailHarness(t, 30) + rule := h.rule(t, "errors", "stay@example.com", "gone@example.com") + h.box.fail = func(m auth.AlertMail) error { + if m.To == "gone@example.com" { + return errors.New("451 later") + } + return nil + } + h.fire(rule) + h.drain() + h.box.fail = nil + rule.EmailRecipients = []string{"stay@example.com"} + if _, err := h.store.ReplaceRule(rule); err != nil { + t.Fatal(err) + } + h.now = h.now.Add(2 * time.Minute) + h.drain() + if got := h.box.to("gone@example.com"); len(got) != 0 { + t.Fatalf("mailed a removed recipient: %+v", got) + } + if status := h.ruleStatus(t, rule.ID).EmailStatus; status != "sent" { + t.Fatalf("rule status = %q", status) + } +} + +func TestEmailWorkerKeepsRunningAfterAPanic(t *testing.T) { + h := newMailHarness(t, 30) + h.engine.now = nil + rule := h.rule(t, "errors", "a@example.com") + panicked := false + h.box.fail = func(auth.AlertMail) error { + if !panicked { + panicked = true + panic("boom") + } + return nil + } + h.engine.transition(rule, "api", true, Env{Service: "api"}, time.Now()) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan struct{}) + go func() { defer close(done); h.engine.runEmail(ctx) }() + deadline := time.After(10 * time.Second) + for len(h.box.to("a@example.com")) != 1 { + select { + case <-done: + t.Fatal("worker exited after a panic") + case <-deadline: + t.Fatal("mail was not delivered after the panic") + case <-time.After(10 * time.Millisecond): + } + } + cancel() + <-done +} + +// startDroppingSMTP accepts mail and drops the connection when dropAt arrives, +// returning the settings and a count of accepted messages. +func startDroppingSMTP(t *testing.T, dropAt string) (auth.SMTPConfig, *atomic.Int32) { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { listener.Close() }) + accepted := &atomic.Int32{} + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + go func() { + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) + reader := bufio.NewReader(conn) + fmt.Fprint(conn, "220 fake\r\n") + for { + line, err := reader.ReadString('\n') + if err != nil { + return + } + verb := strings.ToUpper(strings.Fields(line + " x")[0]) + if verb == dropAt { + return + } + switch verb { + case "EHLO", "HELO": + fmt.Fprint(conn, "250-fake\r\n250 AUTH CRAM-MD5\r\n") + case "AUTH": + fmt.Fprint(conn, "334 PDEyMzQ1QGZha2U+\r\n") + if _, err := reader.ReadString('\n'); err != nil { + return + } + fmt.Fprint(conn, "235 ok\r\n") + case "DATA": + fmt.Fprint(conn, "354 go\r\n") + for { + part, err := reader.ReadString('\n') + if err != nil { + return + } + if part == ".\r\n" { + break + } + } + accepted.Add(1) + fmt.Fprint(conn, "250 queued\r\n") + case "QUIT": + fmt.Fprint(conn, "221 bye\r\n") + return + default: + fmt.Fprint(conn, "250 ok\r\n") + } + } + }() + } + }() + return auth.SMTPConfig{Host: "127.0.0.1", Port: listener.Addr().(*net.TCPAddr).Port, User: "u", Pass: "p", From: "noreply@example.com"}, accepted +} + +func TestAcceptedMailIsSentAndCountedOnceWhenTheSessionEndsBadly(t *testing.T) { + for _, verb := range []string{"RSET", "QUIT"} { + t.Run(verb, func(t *testing.T) { + h := newMailHarness(t, 30) + cfg, accepted := startDroppingSMTP(t, verb) + h.engine.email.Send = func(ctx context.Context, m auth.AlertMail) error { return auth.SendAlert(ctx, cfg, m) } + rule := h.rule(t, "errors", "a@example.com") + h.fire(rule) + for range 3 { + h.drain() + h.now = h.now.Add(time.Hour) + } + if got := accepted.Load(); got != 1 { + t.Fatalf("accepted = %d, want 1", got) + } + if got := h.sendsRecorded(t); got != 1 { + t.Fatalf("recorded sends = %d, want 1", got) + } + if status := h.ruleStatus(t, rule.ID).EmailStatus; status != "sent" { + t.Fatalf("rule status = %q", status) + } + }) + } +} + +func TestRestartResumesQueuedMailAndDoesNotResendFinished(t *testing.T) { + h := newMailHarness(t, 30) + done := h.rule(t, "done", "done@example.com") + h.fire(done) + h.drain() + + interrupted := h.rule(t, "interrupted", "cut@example.com") + h.fire(interrupted) + if _, err := h.store.db.Exec(`UPDATE alert_email_deliveries SET status='sending', attempts=1 WHERE recipient='cut@example.com'`); err != nil { + t.Fatal(err) + } + queued := h.rule(t, "queued", "wait@example.com") + h.fire(queued) + + restarted := &Engine{store: h.store, programs: map[string]Program{}, stopped: map[string]bool{}, now: h.engine.now} + box := &outbox{} + restarted.EnableEmail(EmailConfig{Send: box.send, HourlyLimit: 30}) + restarted.recoverEmail(context.Background()) + restarted.drainEmail(context.Background()) + + if len(box.to("done@example.com")) != 0 { + t.Fatal("finished mail was resent") + } + if len(box.to("cut@example.com")) != 1 || len(box.to("wait@example.com")) != 1 { + t.Fatalf("mails = %+v", box.mails) + } + restarted.drainEmail(context.Background()) + if len(box.mails) != 2 { + t.Fatalf("resent on a later drain: %+v", box.mails) + } +} + +func TestResentMailKeepsItsMessageIdentity(t *testing.T) { + h := newMailHarness(t, 30) + rule := h.rule(t, "errors", "a@example.com") + var ids []string + h.box.fail = func(m auth.AlertMail) error { + ids = append(ids, m.ID) + return errors.New("timeout after DATA") + } + h.fire(rule) + h.drain() + h.now = h.now.Add(2 * time.Minute) + h.drain() + if len(ids) != 2 || ids[0] != ids[1] { + t.Fatalf("ids = %v", ids) + } +} + +func TestInterruptedLastAttemptFailsInsteadOfSendingAgain(t *testing.T) { + h := newMailHarness(t, 30) + rule := h.rule(t, "errors", "a@example.com") + h.fire(rule) + if _, err := h.store.db.Exec(`UPDATE alert_email_deliveries SET status='sending', attempts=?`, len(emailBackoff)); err != nil { + t.Fatal(err) + } + h.engine.recoverEmail(context.Background()) + h.drain() + if len(h.box.mails) != 0 { + t.Fatalf("mails = %+v", h.box.mails) + } + if status := h.ruleStatus(t, rule.ID).EmailStatus; status != "failed" { + t.Fatalf("rule status = %q", status) + } +} + +func TestEmailQueuedAtomicallyWithTransition(t *testing.T) { + h := newMailHarness(t, 30) + rule := h.rule(t, "errors", "a@example.com") + if _, err := h.store.db.Exec(`CREATE TRIGGER block_mail BEFORE INSERT ON alert_email_deliveries BEGIN SELECT RAISE(ABORT, 'blocked'); END`); err != nil { + t.Fatal(err) + } + h.fire(rule) + if _, err := h.store.GetAlert(rule.ID, "api"); !errors.Is(err, ErrNotFound) { + t.Fatalf("alert saved without its email: %v", err) + } +} + +func TestRepeatsFollowTheRulesNotificationBehaviour(t *testing.T) { + h := newMailHarness(t, 30) + rule, err := h.store.CreateRule(Rule{Name: "errors", Enabled: true, Service: "api", Expression: "true", RepeatIntervalS: 600, EmailRecipients: []string{"a@example.com"}}) + if err != nil { + t.Fatal(err) + } + h.fire(rule) + h.drain() + h.now = h.now.Add(5 * time.Minute) + h.fire(rule) + h.drain() + if got := h.box.to("a@example.com"); len(got) != 1 { + t.Fatalf("repeated inside the interval: %+v", got) + } + h.now = h.now.Add(6 * time.Minute) + h.fire(rule) + h.drain() + got := h.box.to("a@example.com") + if len(got) != 2 || got[1].Event != "reminder" { + t.Fatalf("reminder mails = %+v", got) + } + + // notify_on_resolve is off, so the resolve is silent. + h.now = h.now.Add(time.Minute) + h.engine.transition(rule, "api", false, Env{Service: "api"}, h.now) + h.drain() + if got := h.box.to("a@example.com"); len(got) != 2 { + t.Fatalf("resolve mailed with notify_on_resolve off: %+v", got) + } + + rule.NotifyOnResolve = true + h.now = h.now.Add(time.Minute) + h.fire(rule) + h.now = h.now.Add(time.Minute) + h.engine.transition(rule, "api", false, Env{Service: "api"}, h.now) + h.drain() + got = h.box.to("a@example.com") + if len(got) != 4 || got[3].Event != "resolved" { + t.Fatalf("resolve mails = %+v", got) + } +} + +func TestDisablingAFiringRuleMailsItsResolve(t *testing.T) { + h := newMailHarness(t, 30) + rule, err := h.store.CreateRule(Rule{Name: "errors", Enabled: true, Service: "api", Expression: "true", NotifyOnResolve: true, EmailRecipients: []string{"a@example.com"}}) + if err != nil { + t.Fatal(err) + } + h.fire(rule) + h.engine.DisableRule(rule) + h.drain() + got := h.box.to("a@example.com") + if len(got) != 2 || got[1].Event != "resolved" { + t.Fatalf("mails = %+v", got) + } +} + +func TestWebhookResultStaysOnAlertWhenRuleAlsoEmails(t *testing.T) { + var calls sync.WaitGroup + calls.Add(1) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + defer calls.Done() + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + h := newMailHarness(t, 30) + rule, err := h.store.CreateRule(Rule{Name: "both", Enabled: true, Service: "api", Expression: "true", WebhookURL: srv.URL, EmailRecipients: []string{"a@example.com"}}) + if err != nil { + t.Fatal(err) + } + h.box.fail = func(auth.AlertMail) error { return fmt.Errorf("%w: rejected", auth.ErrInvalidMailAddress) } + h.fire(rule) + calls.Wait() + h.drain() + + deadline := time.Now().Add(5 * time.Second) + for { + alert, err := h.store.GetAlert(rule.ID, "api") + if err != nil { + t.Fatal(err) + } + if alert.LastDeliveryStatus == "success" { + break + } + if time.Now().After(deadline) { + t.Fatalf("alert status = %q, want the webhook's success", alert.LastDeliveryStatus) + } + time.Sleep(10 * time.Millisecond) + } + if status := h.ruleStatus(t, rule.ID).EmailStatus; status != "failed" { + t.Fatalf("rule email status = %q", status) + } +} + +func TestWithoutSMTPSavedRecipientsReportUnavailable(t *testing.T) { + engine, store := newTestEngine(t) + rule, err := store.CreateRule(Rule{Name: "errors", Enabled: true, Expression: "true", EmailRecipients: []string{"a@example.com"}}) + if err != nil { + t.Fatal(err) + } + got := engine.WithEmailState(rule) + if got.EmailStatus != "unavailable" || !strings.Contains(got.EmailStatusDetail, "SMTP") { + t.Fatalf("rule = %+v", got) + } + if engine.EmailOptions().Available { + t.Fatal("email reported available without SMTP") + } + plain, err := store.CreateRule(Rule{Name: "plain", Enabled: true, Expression: "true"}) + if err != nil { + t.Fatal(err) + } + if got := engine.WithEmailState(plain); got.EmailStatus != "" { + t.Fatalf("rule without email = %+v", got) + } +} + +func TestMailQueuedBeforeSMTPWasRemovedIsMarkedFailed(t *testing.T) { + h := newMailHarness(t, 30) + rule := h.rule(t, "errors", "a@example.com") + h.fire(rule) + + without := &Engine{store: h.store, programs: map[string]Program{}, stopped: map[string]bool{}} + without.failUndeliverableEmail(context.Background()) + stored := h.ruleStatus(t, rule.ID) + if stored.EmailStatus != "failed" || !strings.Contains(stored.EmailStatusDetail, "SMTP") { + t.Fatalf("rule = %+v", stored) + } +} + +func TestRuleWithoutEmailDestinationHasNoEmailState(t *testing.T) { + h := newMailHarness(t, 30) + rule := h.rule(t, "errors") + rule.EmailStatus = "failed" + if got := h.engine.WithEmailState(rule); got.EmailStatus != "" { + t.Fatalf("rule = %+v", got) + } +} diff --git a/internal/alert/engine.go b/internal/alert/engine.go index 01dce18ed..ccf283134 100644 --- a/internal/alert/engine.go +++ b/internal/alert/engine.go @@ -7,6 +7,7 @@ import ( "regexp" "runtime/debug" "sort" + "strings" "sync" "time" @@ -29,8 +30,11 @@ type Engine struct { interval time.Duration histDays int lastPrune time.Time - envOverride map[string]Env // for testing — bypasses DuckDB query - afterList func() // for testing — runs once a tick has listed its rules + envOverride map[string]Env // for testing — bypasses DuckDB query + afterList func() // for testing — runs once a tick has listed its rules + emailHeld map[string]emailOutcome // outcomes the database refused; worker-owned + email *EmailConfig // nil without SMTP + now func() time.Time // for testing — the email clock } // NewEngine creates an Engine ready to Run. @@ -76,7 +80,7 @@ func (e *Engine) DisableRule(rule Rule) { a.State = "resolved" a.ResolvedAt = now a.LastEvaluatedAt = now - saved, err := e.store.UpsertAlert(a) + saved, err := e.saveAlert(rule, a, resolveEvent(rule)) if err != nil { slog.Error("alert: resolve alert of disabled rule", "rule", rule.ID, "service", a.Service, "err", err) continue @@ -108,6 +112,12 @@ func (e *Engine) Run(ctx context.Context) { slog.Error("alert: evaluation interval must be positive", "interval", e.interval) return } + if e.email != nil { + go e.runEmail(ctx) + } else { + e.failUndeliverableEmail(ctx) + } + ticker := time.NewTicker(e.interval) defer ticker.Stop() @@ -262,7 +272,7 @@ func (e *Engine) transition(rule Rule, svc string, triggered bool, env Env, now LastEvaluatedAt: nowStr, CreatedAt: began, } - saved, upsertErr := e.store.UpsertAlert(a) + saved, upsertErr := e.saveAlert(rule, a, "firing") if upsertErr != nil { slog.Error("alert: upsert firing", "rule", rule.ID, "service", svc, "err", upsertErr) return @@ -299,7 +309,7 @@ func (e *Engine) transition(rule Rule, svc string, triggered bool, env Env, now existing.State = "firing" existing.FiredAt = nowStr existing.LastEvaluatedAt = nowStr - saved, upsertErr := e.store.UpsertAlert(existing) + saved, upsertErr := e.saveAlert(rule, existing, "firing") if upsertErr != nil { slog.Error("alert: promote to firing", "rule", rule.ID, "service", svc, "err", upsertErr) return @@ -335,7 +345,7 @@ func (e *Engine) transition(rule Rule, svc string, triggered bool, env Env, now repeatInterval := time.Duration(rule.RepeatIntervalS) * time.Second if parseErr == nil && now.Sub(refTime) >= repeatInterval { existing.RepeatedAt = nowStr - saved, upsertErr := e.store.UpsertAlert(existing) + saved, upsertErr := e.saveAlert(rule, existing, "reminder") if upsertErr != nil { slog.Error("alert: update repeated_at", "rule", rule.ID, "service", svc, "err", upsertErr) return @@ -353,7 +363,7 @@ func (e *Engine) transition(rule Rule, svc string, triggered bool, env Env, now existing.State = "resolved" existing.ResolvedAt = nowStr existing.LastEvaluatedAt = nowStr - saved, upsertErr := e.store.UpsertAlert(existing) + saved, upsertErr := e.saveAlert(rule, existing, resolveEvent(rule)) if upsertErr != nil { slog.Error("alert: resolve alert", "rule", rule.ID, "service", svc, "err", upsertErr) return @@ -387,7 +397,7 @@ func (e *Engine) transition(rule Rule, svc string, triggered bool, env Env, now if rule.ForSeconds == 0 { a.State = "firing" a.FiredAt = nowStr - saved, upsertErr := e.store.UpsertAlert(a) + saved, upsertErr := e.saveAlert(rule, a, "firing") if upsertErr != nil { slog.Error("alert: re-fire from resolved", "rule", rule.ID, "service", svc, "err", upsertErr) return @@ -453,8 +463,22 @@ func (e *Engine) Test(ctx context.Context, rule Rule) ([]TestResult, error) { return results, nil } +// resolveEvent is the event a resolve notifies, or empty when the rule does not +// notify on resolve. +func resolveEvent(rule Rule) string { + if rule.NotifyOnResolve { + return "resolved" + } + return "" +} + // fireWebhookAsync fires the webhook for an alert in a goroutine and updates delivery status. +// A rule delivering only by email has no webhook result to record; the email +// worker reports on the alert instead. func (e *Engine) fireWebhookAsync(rule Rule, alert Alert, env Env, event string) { + if strings.TrimSpace(rule.WebhookURL) == "" && len(e.emailRecipients(rule)) > 0 { + return + } go func() { defer func() { if r := recover(); r != nil { @@ -577,6 +601,10 @@ func (e *Engine) pruneOldAlerts() { slog.Warn("alert: prune resolved", "err", err) return } + if err := e.store.pruneEmailDeliveries(days, time.Now()); err != nil { + slog.Warn("alert: prune email deliveries", "err", err) + return + } e.lastPrune = time.Now() if n > 0 { slog.Info("alert: pruned resolved alerts", "count", n) diff --git a/internal/alert/store.go b/internal/alert/store.go index 6567cd9d6..64dfc4afd 100644 --- a/internal/alert/store.go +++ b/internal/alert/store.go @@ -15,12 +15,13 @@ var ErrNotFound = errors.New("not found") // Store provides CRUD operations for alert rules and alert instances. type Store struct { - q *generated.Queries + db *sql.DB + q *generated.Queries } // NewStore creates a Store backed by the given database connection. func NewStore(db *sql.DB) *Store { - return &Store{q: generated.New(db)} + return &Store{db: db, q: generated.New(db)} } // newID returns a new UUIDv7 string. @@ -49,6 +50,7 @@ func ruleToCreateParams(r Rule) generated.CreateRuleParams { WebhookHeaders: nullString(r.WebhookHeaders), WebhookTemplate: nullString(r.WebhookTemplate), NotifyOnResolve: boolToInt64(r.NotifyOnResolve), + EmailRecipients: encodeRecipients(r.EmailRecipients), } } @@ -69,28 +71,33 @@ func ruleToReplaceParams(r Rule) generated.ReplaceRuleParams { WebhookHeaders: nullString(r.WebhookHeaders), WebhookTemplate: nullString(r.WebhookTemplate), NotifyOnResolve: boolToInt64(r.NotifyOnResolve), + EmailRecipients: encodeRecipients(r.EmailRecipients), } } // alertRuleToRule converts a generated.AlertRule to a domain Rule. func alertRuleToRule(ar generated.AlertRule) Rule { return Rule{ - ID: ar.ID, - Name: ar.Name, - Description: ar.Description.String, - Enabled: ar.Enabled != 0, - Service: ar.Service.String, - Namespace: ar.Namespace.String, - Expression: ar.Expression, - ForSeconds: int(ar.ForSeconds), - CooldownS: int(ar.CooldownS), - RepeatIntervalS: int(ar.RepeatIntervalS), - WebhookURL: ar.WebhookURL.String, - WebhookHeaders: ar.WebhookHeaders.String, - WebhookTemplate: ar.WebhookTemplate.String, - NotifyOnResolve: ar.NotifyOnResolve != 0, - CreatedAt: ar.CreatedAt, - UpdatedAt: ar.UpdatedAt, + ID: ar.ID, + Name: ar.Name, + Description: ar.Description.String, + Enabled: ar.Enabled != 0, + Service: ar.Service.String, + Namespace: ar.Namespace.String, + Expression: ar.Expression, + ForSeconds: int(ar.ForSeconds), + CooldownS: int(ar.CooldownS), + RepeatIntervalS: int(ar.RepeatIntervalS), + WebhookURL: ar.WebhookURL.String, + WebhookHeaders: ar.WebhookHeaders.String, + WebhookTemplate: ar.WebhookTemplate.String, + NotifyOnResolve: ar.NotifyOnResolve != 0, + EmailRecipients: decodeRecipients(ar.ID, ar.EmailRecipients), + EmailStatus: ar.EmailStatus, + EmailStatusDetail: ar.EmailStatusDetail, + EmailStatusAt: ar.EmailStatusAt, + CreatedAt: ar.CreatedAt, + UpdatedAt: ar.UpdatedAt, } } @@ -216,13 +223,17 @@ func (s *Store) DeleteRule(id string) error { // UpsertAlert inserts an alert episode, or updates the one with a.ID. Each // episode is its own row; at most one per (rule_id, service) is open. func (s *Store) UpsertAlert(a Alert) (Alert, error) { + return upsertAlert(s.q, a) +} + +func upsertAlert(q *generated.Queries, a Alert) (Alert, error) { // Only the engine starts an episode. An update without a start keeps the // stored one; a new alert must bring its own. if a.CreatedAt == "" { if a.ID == "" { return Alert{}, fmt.Errorf("store: new alert (%q, %q) needs created_at", a.RuleID, a.Service) } - if _, err := s.q.GetAlertByID(context.Background(), a.ID); errors.Is(err, sql.ErrNoRows) { + if _, err := q.GetAlertByID(context.Background(), a.ID); errors.Is(err, sql.ErrNoRows) { return Alert{}, fmt.Errorf("store: new alert (%q, %q) needs created_at", a.RuleID, a.Service) } else if err != nil { return Alert{}, fmt.Errorf("store: get alert: %w", err) @@ -241,7 +252,7 @@ func (s *Store) UpsertAlert(a Alert) (Alert, error) { } a.ID = id } - ga, err := s.q.UpsertAlert(context.Background(), alertToUpsertParams(a)) + ga, err := q.UpsertAlert(context.Background(), alertToUpsertParams(a)) if err != nil { return Alert{}, fmt.Errorf("store: upsert alert: %w", err) } diff --git a/internal/alert/types.go b/internal/alert/types.go index 146f81719..2d7f8fde1 100644 --- a/internal/alert/types.go +++ b/internal/alert/types.go @@ -18,8 +18,16 @@ type Rule struct { WebhookHeaders string `json:"webhook_headers,omitempty"` WebhookTemplate string `json:"webhook_template,omitempty"` NotifyOnResolve bool `json:"notify_on_resolve"` - CreatedAt string `json:"created_at"` - UpdatedAt string `json:"updated_at"` + // EmailRecipients are this rule's own addresses. Empty means none: the + // default destination then applies if the rule has no webhook either. + EmailRecipients []string `json:"email_recipients,omitempty"` + // EmailStatus reports the rule's latest email notification: sent, retrying, + // failed, suppressed, or unavailable. The server owns these three fields. + EmailStatus string `json:"email_status,omitempty"` + EmailStatusDetail string `json:"email_status_detail,omitempty"` + EmailStatusAt string `json:"email_status_at,omitempty"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` } // Alert represents the current state of an alert instance for a (rule, service) pair. diff --git a/internal/api/alerts.go b/internal/api/alerts.go index cd142fd93..5ddac32e2 100644 --- a/internal/api/alerts.go +++ b/internal/api/alerts.go @@ -3,11 +3,13 @@ package api import ( "log/slog" "net/http" + "slices" "strings" "github.com/labstack/echo/v5" "github.com/labstack/fanout/internal/alert" + "github.com/labstack/fanout/internal/auth" ) // RegisterAlertRoutes registers alert management REST endpoints. @@ -19,6 +21,7 @@ func RegisterAlertRoutes(e *echo.Echo, store *alert.Store, engine *alert.Engine) e.GET("/api/alerting/alerts", h.ListAlerts, read) e.GET("/api/alerting/summary", h.AlertSummary, read) e.GET("/api/alerting/rules", h.ListRules, read) + e.GET("/api/alerting/email", h.EmailOptions, read) e.POST("/api/alerting/rules", h.CreateRule, manage) e.PUT("/api/alerting/rules/:id", h.ReplaceRule, manage) e.DELETE("/api/alerting/rules/:id", h.DeleteRule, manage) @@ -76,9 +79,47 @@ func (h *alertHandler) ListRules(c *echo.Context) error { if rules == nil { rules = []alert.Rule{} } + user := GetCurrentUser(c) + manager := user != nil && HasCapability(*user, ManageAlerts) + for i := range rules { + rules[i] = h.engine.WithEmailState(rules[i]) + if !manager { + rules[i] = readerRule(rules[i]) + } + } return c.JSON(200, map[string][]alert.Rule{"rules": rules}) } +// readerRule is the view of a rule for principals who cannot manage alerts. It +// is an allowlist, so a field added to Rule stays hidden until it is listed +// here. Delivery settings are left out: webhook headers can hold credentials, +// and recipients and failure detail identify people and mail servers. Readers +// still see email_status, so a failing rule is visible to them. +func readerRule(rule alert.Rule) alert.Rule { + return alert.Rule{ + ID: rule.ID, + Name: rule.Name, + Description: rule.Description, + Enabled: rule.Enabled, + Service: rule.Service, + Namespace: rule.Namespace, + Expression: rule.Expression, + ForSeconds: rule.ForSeconds, + CooldownS: rule.CooldownS, + RepeatIntervalS: rule.RepeatIntervalS, + NotifyOnResolve: rule.NotifyOnResolve, + EmailStatus: rule.EmailStatus, + EmailStatusAt: rule.EmailStatusAt, + CreatedAt: rule.CreatedAt, + UpdatedAt: rule.UpdatedAt, + } +} + +// EmailOptions reports whether rules can deliver by email on this instance. +func (h *alertHandler) EmailOptions(c *echo.Context) error { + return c.JSON(200, h.engine.EmailOptions()) +} + // CreateRule creates a new alert rule. func (h *alertHandler) CreateRule(c *echo.Context) error { if h.store == nil { @@ -96,6 +137,9 @@ func (h *alertHandler) CreateRule(c *echo.Context) error { if err := checkRule(req, true); err != nil { return err } + if err := h.checkRecipients(&req); err != nil { + return err + } created, err := h.store.CreateRule(req) if err != nil { @@ -110,7 +154,7 @@ func (h *alertHandler) CreateRule(c *echo.Context) error { } } - return c.JSON(201, created) + return c.JSON(201, h.engine.WithEmailState(created)) } // ReplaceRule replaces an existing alert rule. @@ -133,6 +177,12 @@ func (h *alertHandler) ReplaceRule(c *echo.Context) error { if err := checkRule(req, changed); err != nil { return err } + // Likewise for recipients: a saved rule stays editable after SMTP is removed. + if err != nil || !slices.Equal(stored.EmailRecipients, req.EmailRecipients) { + if err := h.checkRecipients(&req); err != nil { + return err + } + } updated, err := h.store.ReplaceRule(req) if err != nil { @@ -153,7 +203,7 @@ func (h *alertHandler) ReplaceRule(c *echo.Context) error { } } - return c.JSON(200, updated) + return c.JSON(200, h.engine.WithEmailState(updated)) } // DeleteRule deletes an alert rule and its associated alerts. @@ -207,6 +257,23 @@ func (h *alertHandler) TestRule(c *echo.Context) error { }) } +// checkRecipients normalizes a rule's email recipients in place, rejecting +// invalid addresses, too many of them, or any when email is unavailable. +func (h *alertHandler) checkRecipients(rule *alert.Rule) error { + if len(rule.EmailRecipients) == 0 { + return nil + } + if options := h.engine.EmailOptions(); !options.Available { + return newAPIError(http.StatusBadRequest, "email_unavailable", options.Message) + } + recipients, err := auth.NormalizeRecipients(rule.EmailRecipients) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, err.Error()) + } + rule.EmailRecipients = recipients + return nil +} + // checkRule rejects an expression that does not compile and, when headers is // set, webhook headers that are not a JSON object of valid HTTP headers. A // replace may omit the expression. diff --git a/internal/api/alerts_email_test.go b/internal/api/alerts_email_test.go new file mode 100644 index 000000000..8077f8ce0 --- /dev/null +++ b/internal/api/alerts_email_test.go @@ -0,0 +1,210 @@ +package api + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "slices" + "strings" + "testing" + "time" + + "github.com/labstack/fanout/internal/alert" + "github.com/labstack/fanout/internal/auth" +) + +type alertEmailServer struct { + *testAuthServer + store *alert.Store + cookie *http.Cookie +} + +func newAlertEmailServer(t *testing.T, smtp bool) *alertEmailServer { + t.Helper() + s := newTestAuthServer(t) + store := alert.NewStore(s.db.DB) + engine := alert.NewEngine(store, nil, nil, 30*time.Second, 7) + if smtp { + engine.EnableEmail(alert.EmailConfig{Send: func(context.Context, auth.AlertMail) error { return nil }, Defaults: []string{"oncall@example.com"}, HourlyLimit: 30}) + } + RegisterAlertRoutes(s.e, store, engine) + admin, err := s.users.CreateWithAudit("alert-admin@example.com", "", "admin", auth.AuditEvent{EventType: "user.created", Outcome: "success"}) + if err != nil { + t.Fatal(err) + } + return &alertEmailServer{testAuthServer: s, store: store, cookie: s.login(t, admin)} +} + +func (s *alertEmailServer) do(method, path, body string) *httptest.ResponseRecorder { + req := sessionRequest(method, path, strings.NewReader(body), s.cookie) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + s.e.ServeHTTP(rec, req) + return rec +} + +func ruleBody(recipients string) string { + return `{"name":"Errors","enabled":true,"expression":"error_rate > 0.05","email_recipients":` + recipients + `}` +} + +func TestAlertRuleEmailRecipientsAreValidatedAndNormalized(t *testing.T) { + s := newAlertEmailServer(t, true) + rec := s.do(http.MethodPost, "/api/alerting/rules", ruleBody(`[" A@Example.com ","a@example.com","b@example.com"]`)) + if rec.Code != http.StatusCreated { + t.Fatalf("create = %d %s", rec.Code, rec.Body) + } + var created alert.Rule + if err := json.Unmarshal(rec.Body.Bytes(), &created); err != nil { + t.Fatal(err) + } + if len(created.EmailRecipients) != 2 || created.EmailRecipients[0] != "a@example.com" { + t.Fatalf("recipients = %v", created.EmailRecipients) + } + + for name, recipients := range map[string]string{ + "not an address": `["nope"]`, + "display name": `["Eve "]`, + "line break": `["a@example.com\r\nBcc: x@evil.test"]`, + "over the cap": `["a0@e.co","a1@e.co","a2@e.co","a3@e.co","a4@e.co","a5@e.co","a6@e.co","a7@e.co","a8@e.co","a9@e.co","a10@e.co"]`, + } { + t.Run(name, func(t *testing.T) { + if rec := s.do(http.MethodPost, "/api/alerting/rules", ruleBody(recipients)); rec.Code != http.StatusBadRequest { + t.Fatalf("create = %d %s, want 400", rec.Code, rec.Body) + } + if rec := s.do(http.MethodPut, "/api/alerting/rules/"+created.ID, ruleBody(recipients)); rec.Code != http.StatusBadRequest { + t.Fatalf("replace = %d %s, want 400", rec.Code, rec.Body) + } + }) + } +} + +func TestAlertRuleClientCannotWriteEmailStatus(t *testing.T) { + s := newAlertEmailServer(t, true) + rec := s.do(http.MethodPost, "/api/alerting/rules", `{"name":"Errors","enabled":true,"expression":"true","email_recipients":["a@example.com"],"email_status":"sent","email_status_detail":"forged"}`) + if rec.Code != http.StatusCreated || strings.Contains(rec.Body.String(), "forged") { + t.Fatalf("create = %d %s", rec.Code, rec.Body) + } +} + +func TestEmailOptionIsUnavailableWithoutSMTP(t *testing.T) { + s := newAlertEmailServer(t, false) + rec := s.do(http.MethodPost, "/api/alerting/rules", ruleBody(`["a@example.com"]`)) + if rec.Code != http.StatusBadRequest { + t.Fatalf("create = %d %s, want 400", rec.Code, rec.Body) + } + var body struct{ Code, Message string } + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if body.Code != "email_unavailable" || !strings.Contains(body.Message, "SMTP is not configured") { + t.Fatalf("error = %+v", body) + } + + options := s.do(http.MethodGet, "/api/alerting/email", "") + var got alert.EmailOptions + if err := json.Unmarshal(options.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + if options.Code != http.StatusOK || got.Available || !strings.Contains(got.Message, "SMTP") || got.DefaultDestination { + t.Fatalf("options = %d %s", options.Code, options.Body) + } + if rec := s.do(http.MethodPost, "/api/alerting/rules", `{"name":"Hook","enabled":true,"expression":"true","webhook_url":"https://example.com/hook"}`); rec.Code != http.StatusCreated { + t.Fatalf("webhook rule = %d %s", rec.Code, rec.Body) + } +} + +func TestEmailOptionsReportTheDefaultDestinationWithoutItsAddress(t *testing.T) { + s := newAlertEmailServer(t, true) + rec := s.do(http.MethodGet, "/api/alerting/email", "") + if rec.Code != http.StatusOK || strings.Contains(rec.Body.String(), "oncall@") { + t.Fatalf("options = %d %s", rec.Code, rec.Body) + } + var got alert.EmailOptions + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + if !got.Available || !got.DefaultDestination || got.MaxRecipients != auth.MaxAlertEmailRecipients || got.HourlyLimit != 30 { + t.Fatalf("options = %+v", got) + } +} + +func TestRuleDeliverySettingsAreHiddenFromViewers(t *testing.T) { + s := newAlertEmailServer(t, false) + if _, err := s.store.CreateRule(alert.Rule{ + Name: "Errors", Description: "Checkout errors", Enabled: true, Service: "checkout", Namespace: "prod", Expression: "true", NotifyOnResolve: true, + WebhookURL: "https://hooks.example.com/private", WebhookHeaders: `{"Authorization":"Bearer private-token"}`, WebhookTemplate: `{"private":"template"}`, + EmailRecipients: []string{"private@example.com"}, + }); err != nil { + t.Fatal(err) + } + viewer, err := s.users.CreateWithAudit("alert-viewer@example.com", "", "viewer", auth.AuditEvent{EventType: "user.created", Outcome: "success"}) + if err != nil { + t.Fatal(err) + } + rec := httptest.NewRecorder() + s.e.ServeHTTP(rec, sessionRequest(http.MethodGet, "/api/alerting/rules", nil, s.login(t, viewer))) + if rec.Code != http.StatusOK { + t.Fatalf("viewer list = %d %s", rec.Code, rec.Body) + } + for _, hidden := range []string{"private", "webhook_url", "webhook_headers", "webhook_template", "email_recipients", "email_status_detail"} { + if strings.Contains(rec.Body.String(), hidden) { + t.Fatalf("viewer response holds %q: %s", hidden, rec.Body) + } + } + + // An exact key set, so a field added to Rule cannot reach viewers unnoticed. + var listed struct { + Rules []map[string]any `json:"rules"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &listed); err != nil || len(listed.Rules) != 1 { + t.Fatalf("viewer list = %s, %v", rec.Body, err) + } + keys := make([]string, 0, len(listed.Rules[0])) + for key := range listed.Rules[0] { + keys = append(keys, key) + } + slices.Sort(keys) + want := []string{"cooldown_s", "created_at", "description", "email_status", "enabled", "expression", "for_seconds", "id", "name", "namespace", "notify_on_resolve", "repeat_interval_s", "service", "updated_at"} + if !slices.Equal(keys, want) { + t.Fatalf("viewer keys = %v, want %v", keys, want) + } + if listed.Rules[0]["email_status"] != "unavailable" { + t.Fatalf("viewer lost the failure signal: %s", rec.Body) + } + + manager := s.do(http.MethodGet, "/api/alerting/rules", "") + for _, shown := range []string{"webhook_url", "webhook_headers", "webhook_template", "email_recipients", "email_status_detail", "private-token"} { + if !strings.Contains(manager.Body.String(), shown) { + t.Fatalf("manager response lacks %q: %s", shown, manager.Body) + } + } +} + +// A rule saved while SMTP worked stays editable and reports the failure. +func TestSavedEmailRuleWithoutSMTPShowsFailureAndStaysEditable(t *testing.T) { + s := newAlertEmailServer(t, false) + stored, err := s.store.CreateRule(alert.Rule{Name: "Errors", Enabled: true, Expression: "true", EmailRecipients: []string{"a@example.com"}}) + if err != nil { + t.Fatal(err) + } + rec := s.do(http.MethodGet, "/api/alerting/rules", "") + var listed struct{ Rules []alert.Rule } + if err := json.Unmarshal(rec.Body.Bytes(), &listed); err != nil { + t.Fatal(err) + } + if len(listed.Rules) != 1 || listed.Rules[0].EmailStatus != "unavailable" || !strings.Contains(listed.Rules[0].EmailStatusDetail, "SMTP") { + t.Fatalf("rules = %s", rec.Body) + } + + toggle := `{"name":"Errors","enabled":false,"expression":"true","email_recipients":["a@example.com"]}` + if rec := s.do(http.MethodPut, "/api/alerting/rules/"+stored.ID, toggle); rec.Code != http.StatusOK { + t.Fatalf("toggle = %d %s", rec.Code, rec.Body) + } + if rec := s.do(http.MethodPut, "/api/alerting/rules/"+stored.ID, ruleBody(`["new@example.com"]`)); rec.Code != http.StatusBadRequest { + t.Fatalf("changed recipients = %d %s, want 400", rec.Code, rec.Body) + } + if rec := s.do(http.MethodPut, "/api/alerting/rules/"+stored.ID, ruleBody(`[]`)); rec.Code != http.StatusOK { + t.Fatalf("clear recipients = %d %s", rec.Code, rec.Body) + } +} diff --git a/internal/api/auth_middleware.go b/internal/api/auth_middleware.go index 68b408b3c..4a8702530 100644 --- a/internal/api/auth_middleware.go +++ b/internal/api/auth_middleware.go @@ -332,7 +332,7 @@ func classifyNonSCIMRoute(method, path string) (routePolicy, bool) { return routePolicy{kind: routePolicyCapability, capability: ReadTelemetry}, read case path == "/api/insights": return routePolicy{kind: routePolicyCapability, capability: ReadTelemetry}, read - case path == "/api/alerting/alerts" || path == "/api/alerting/summary" || path == "/api/alerting/rules": + case path == "/api/alerting/alerts" || path == "/api/alerting/summary" || path == "/api/alerting/rules" || path == "/api/alerting/email": if read { return routePolicy{kind: routePolicyCapability, capability: ReadTelemetry}, true } diff --git a/internal/auth/alert_mail.go b/internal/auth/alert_mail.go new file mode 100644 index 000000000..d37e540e6 --- /dev/null +++ b/internal/auth/alert_mail.go @@ -0,0 +1,175 @@ +package auth + +import ( + "bytes" + "context" + "errors" + "fmt" + "html/template" + "net" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + mail "github.com/wneessen/go-mail" +) + +// AlertMail is one alert notification for one recipient. Every text field may +// hold telemetry-derived or operator-written values; SendAlert neutralizes them. +type AlertMail struct { + // ID identifies the delivery. It becomes the Message-ID, so a resend after + // an interrupted attempt carries the same one. + ID string + To string + Rule string + Description string + Service string + Event string + Value float64 + Time time.Time + // LimitNotice, when set, warns that later notifications will be suppressed. + LimitNotice string +} + +// ErrInvalidMailAddress marks a sender or recipient that does not parse. +var ErrInvalidMailAddress = errors.New("invalid mail address") + +// IsPermanentMailError reports whether retrying err cannot succeed: an address +// that does not parse, or a 5xx SMTP reply. Everything else is transient, +// including a connection lost mid-session, which go-mail reports as a SendError +// with no server code and does not count as temporary. +func IsPermanentMailError(err error) bool { + if errors.Is(err, ErrInvalidMailAddress) { + return true + } + var sendErr *mail.SendError + if !errors.As(err, &sendErr) { + return false + } + code := sendErr.ErrorCode() + return code >= 500 && code < 600 +} + +// MailFailureDetail describes a send failure by its class and SMTP code, for +// logs and status shown to users. It never includes err's text, which can carry +// server responses and recipient addresses. +func MailFailureDetail(err error) string { + var sendErr *mail.SendError + var netErr net.Error + switch { + case errors.Is(err, ErrInvalidMailAddress): + return "Invalid email address" + case errors.As(err, &sendErr): + if code := sendErr.ErrorCode(); code != 0 { + return fmt.Sprintf("Mail server returned %d while %s", code, sendErr.Reason) + } + return fmt.Sprintf("Delivery failed while %s", sendErr.Reason) + case errors.Is(err, context.DeadlineExceeded): + return "Timed out talking to the mail server" + case errors.As(err, &netErr): + return "Could not reach the mail server" + default: + return "The exchange with the mail server failed" + } +} + +// SendAlert renders and sends one alert notification. +func SendAlert(ctx context.Context, cfg SMTPConfig, alert AlertMail) error { + subject, text, html, err := renderAlertMail(alert) + if err != nil { + return err + } + message, err := newMailMessage(cfg.From, alert.To, subject, text, html) + if err != nil { + return fmt.Errorf("%w: %w", ErrInvalidMailAddress, err) + } + message.SetMessageIDWithValue(alert.ID + "@alerts.fanout") + return deliver(ctx, cfg, message) +} + +// Lengths, in runes, past which a value is cut. They keep a hostile value from +// filling a subject line or a message. +const ( + maxAlertNameLength = 120 + maxAlertDescriptionLength = 1000 +) + +// singleLine makes a value safe for a header or one line of text: control, +// format and line-separator characters become spaces, so CR/LF cannot start a +// new header or forge a line, and whitespace runs collapse. +func singleLine(value string, limit int) string { + cleaned := strings.Map(func(r rune) rune { + if unicode.IsControl(r) || unicode.Is(unicode.Cf, r) || unicode.Is(unicode.Zl, r) || unicode.Is(unicode.Zp, r) { + return ' ' + } + return r + }, strings.ToValidUTF8(value, " ")) + cleaned = strings.Join(strings.Fields(cleaned), " ") + if utf8.RuneCountInString(cleaned) > limit { + cleaned = strings.TrimSpace(string([]rune(cleaned)[:limit])) + "…" + } + return cleaned +} + +type alertMailData struct { + Preheader string + Label string + Rule string + Description string + Service string + Value string + Time string + LimitNotice string +} + +func alertEventLabel(event string) string { + switch event { + case "firing": + return "Firing" + case "reminder": + return "Still firing" + case "resolved": + return "Resolved" + default: + return "Alert" + } +} + +func renderAlertMail(alert AlertMail) (subject, text, html string, err error) { + data := alertMailData{ + Label: alertEventLabel(alert.Event), + Rule: singleLine(alert.Rule, maxAlertNameLength), + Description: singleLine(alert.Description, maxAlertDescriptionLength), + Service: singleLine(alert.Service, maxAlertNameLength), + Value: strconv.FormatFloat(alert.Value, 'g', 6, 64), + Time: alert.Time.UTC().Format(time.RFC1123), + LimitNotice: singleLine(alert.LimitNotice, maxAlertDescriptionLength), + } + subject = fmt.Sprintf("[Fanout] %s: %s on %s", data.Label, data.Rule, data.Service) + data.Preheader = subject + + var body strings.Builder + fmt.Fprintf(&body, "%s\n\nRule: %s\nService: %s\nValue: %s\nTime: %s\n", data.Label, data.Rule, data.Service, data.Value, data.Time) + if data.Description != "" { + fmt.Fprintf(&body, "Description: %s\n", data.Description) + } + if data.LimitNotice != "" { + fmt.Fprintf(&body, "\n%s\n", data.LimitNotice) + } + + var rendered bytes.Buffer + if err := alertMailTemplate.Execute(&rendered, data); err != nil { + return "", "", "", fmt.Errorf("auth: render alert email: %w", err) + } + return subject, body.String(), rendered.String(), nil +} + +var alertMailTemplate = template.Must(template.New("alert-email").Parse(mailShellStart + ` +

{{.Label}}

+

{{.Rule}}

+

Service: {{.Service}}
Value: {{.Value}}
Time: {{.Time}}

+{{if .Description}}

{{.Description}}

{{end}} +{{if .LimitNotice}}

{{.LimitNotice}}

{{end}} +` + mailShellEnd)) diff --git a/internal/auth/alert_mail_test.go b/internal/auth/alert_mail_test.go new file mode 100644 index 000000000..e5ea06940 --- /dev/null +++ b/internal/auth/alert_mail_test.go @@ -0,0 +1,353 @@ +package auth + +import ( + "bufio" + "context" + "encoding/base64" + "fmt" + "io" + "mime" + "mime/multipart" + "mime/quotedprintable" + "net" + "net/mail" + "net/textproto" + "strings" + "sync" + "testing" + "time" +) + +// fakeSMTP accepts one conversation at a time and records message data. A +// nonempty rcptReply replaces the normal reply to RCPT. +type fakeSMTP struct { + listener net.Listener + rcptReply string + // dropAt closes the connection without a reply when this verb arrives; + // BODY drops after the message data, before the server's reply. + dropAt string + mu sync.Mutex + data []string +} + +func startFakeSMTP(t *testing.T, rcptReply string) (*fakeSMTP, SMTPConfig) { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + server := &fakeSMTP{listener: listener, rcptReply: rcptReply} + t.Cleanup(func() { listener.Close() }) + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + go server.serve(conn) + } + }() + return server, SMTPConfig{Host: "127.0.0.1", Port: listener.Addr().(*net.TCPAddr).Port, User: "user", Pass: "pass", From: "Fanout "} +} + +func (s *fakeSMTP) serve(conn net.Conn) { + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) + reader := bufio.NewReader(conn) + reply := func(line string) { fmt.Fprintf(conn, "%s\r\n", line) } + reply("220 fake ESMTP") + for { + line, err := reader.ReadString('\n') + if err != nil { + return + } + verb := strings.ToUpper(strings.Fields(line + " x")[0]) + if verb == s.dropAt { + return + } + switch verb { + case "EHLO", "HELO": + reply("250-fake") + reply("250 AUTH CRAM-MD5") + case "AUTH": + reply("334 PDEyMzQ1QGZha2U+") + if _, err := reader.ReadString('\n'); err != nil { + return + } + reply("235 ok") + case "RCPT": + if s.rcptReply != "" { + reply(s.rcptReply) + continue + } + reply("250 ok") + case "DATA": + reply("354 go") + var body strings.Builder + for { + part, err := reader.ReadString('\n') + if err != nil { + return + } + if part == ".\r\n" { + break + } + body.WriteString(strings.TrimPrefix(part, ".")) + } + s.mu.Lock() + s.data = append(s.data, body.String()) + s.mu.Unlock() + if s.dropAt == "BODY" { + return + } + reply("250 queued") + case "QUIT": + reply("221 bye") + return + default: + reply("250 ok") + } + } +} + +func (s *fakeSMTP) messages() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.data...) +} + +const hostileService = "api\r\nBcc: victim@evil.test\r\n\r\n\u2028Subject: forged\u202e" + +func hostileAlert() AlertMail { + return AlertMail{ + ID: "0190-delivery", To: "oncall@example.com", Rule: "High errors\r\nX-Injected: yes", Description: "\nline two", + Service: hostileService, Event: "firing", Value: 0.5, Time: time.Date(2026, 11, 8, 12, 0, 0, 0, time.UTC), + } +} + +func decodeParts(t *testing.T, message *mail.Message) (text, html string) { + t.Helper() + _, params, err := mime.ParseMediaType(message.Header.Get("Content-Type")) + if err != nil { + t.Fatal(err) + } + reader := multipart.NewReader(message.Body, params["boundary"]) + for { + part, err := reader.NextRawPart() + if err == io.EOF { + return text, html + } + if err != nil { + t.Fatal(err) + } + var source io.Reader = part + switch strings.ToLower(part.Header.Get("Content-Transfer-Encoding")) { + case "quoted-printable": + source = quotedprintable.NewReader(part) + case "base64": + source = base64.NewDecoder(base64.StdEncoding, stripNewlines{part}) + } + content, err := io.ReadAll(source) + if err != nil { + t.Fatal(err) + } + if strings.HasPrefix(part.Header.Get("Content-Type"), "text/html") { + html = string(content) + } else { + text = string(content) + } + } +} + +type stripNewlines struct{ reader io.Reader } + +func (s stripNewlines) Read(p []byte) (int, error) { + n, err := s.reader.Read(p) + kept := 0 + for _, b := range p[:n] { + if b != '\r' && b != '\n' { + p[kept] = b + kept++ + } + } + return kept, err +} + +func TestSendAlertKeepsHostileValuesOutOfHeadersAndMarkup(t *testing.T) { + server, cfg := startFakeSMTP(t, "") + if err := SendAlert(context.Background(), cfg, hostileAlert()); err != nil { + t.Fatal(err) + } + messages := server.messages() + if len(messages) != 1 { + t.Fatalf("messages = %d", len(messages)) + } + message, err := mail.ReadMessage(strings.NewReader(messages[0])) + if err != nil { + t.Fatal(err) + } + for _, header := range []string{"Bcc", "X-Injected"} { + if value := message.Header.Get(header); value != "" { + t.Fatalf("injected header %s: %q", header, value) + } + } + if got := message.Header[textproto.CanonicalMIMEHeaderKey("Subject")]; len(got) != 1 { + t.Fatalf("subject headers = %q", got) + } + subject, err := new(mime.WordDecoder).DecodeHeader(message.Header.Get("Subject")) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(subject, "[Fanout] Firing: High errors X-Injected: yes on api Bcc:") { + t.Fatalf("subject = %q", subject) + } + if strings.ContainsAny(subject, "\r\n\u2028\u202e") { + t.Fatalf("subject holds control characters: %q", subject) + } + if got := message.Header.Get("Message-Id"); got != "<0190-delivery@alerts.fanout>" { + t.Fatalf("message id = %q", got) + } + + text, html := decodeParts(t, message) + if !strings.Contains(text, "Service: api Bcc: victim@evil.test - + diff --git a/site/src/content/docs/docs/guides/alerts.mdx b/site/src/content/docs/docs/guides/alerts.mdx index d12b8828f..160c9cda0 100644 --- a/site/src/content/docs/docs/guides/alerts.mdx +++ b/site/src/content/docs/docs/guides/alerts.mdx @@ -1,7 +1,7 @@ --- title: Set up alerts -description: Write alert rules as expressions over service metrics, and deliver them by webhook. -summary: The rule model, the variables available to an expression, the pending/firing/resolved lifecycle, and webhook delivery. +description: Write alert rules as expressions over service metrics, and deliver them by webhook or email. +summary: The rule model, the variables available to an expression, the pending/firing/resolved lifecycle, and webhook and email delivery. read_when: - You want to be told when a service degrades. - A rule is not firing when you expect it to. @@ -10,23 +10,26 @@ status: preview The alert engine runs inside the binary and is on by default. It evaluates persisted rules every `FANOUT_ALERTS_EVALUATION_INTERVAL` (30s) against service -rollups, and delivers by webhook. +rollups, and delivers by webhook, by email, or both. ## Create and manage a rule Open **Alerts** in the sidebar, then choose **New rule**. Enter a name and -expression, plus an optional description and webhook URL. Without a webhook -URL, the rule records alerts but delivers nothing. Leave **Service** empty to +expression, plus an optional description, email recipients and webhook URL. A +rule with neither email recipients nor a webhook URL goes to the +[default email destination](#email-delivery) when one is configured. Without +one, the rule records alerts but delivers nothing. Leave **Service** empty to evaluate every service, or select one service. Choose **Start from a template** for an example expression, then set the timers and save. The new-rule form starts enabled, with **Hold for (s)** 60, **Cooldown (s)** 300 and **Repeat every (s)** 3600. **Notify when resolved** starts off. The name, -description, service, expression and webhook fields start empty. +description, service, expression, email and webhook fields start empty. Use the **Enabled** toggle to pause or resume a rule. The rule's actions menu has **Edit**, **Test against current data** and **Delete**. Testing evaluates -the saved expression without changing alert state or sending a webhook. +the saved expression without changing alert state, sending a webhook or +sending email. Deleting removes the rule and its alert history. Choose **Open in Explore** to chart a watched measure from raw telemetry. @@ -36,7 +39,9 @@ percent: `0.05` in a rule is `5` there. Read current states and delivery results under **History**. `FANOUT_ALERTS_HISTORY_DAYS` sets resolved-history retention in days (default 7). -Viewers can read rules and history. Creating, editing, toggling, testing and +Viewers can read rules and history, without a rule's delivery settings: webhook +URL, headers and template, email recipients, and email failure detail are returned +only to operators and administrators. Creating, editing, toggling, testing and deleting require an operator or administrator session; see [roles](/docs/reference/roles). @@ -54,7 +59,8 @@ An expression, an optional service to scope it to, timing, and delivery: | `for_seconds` | How long the condition must hold before firing | | `cooldown_s` | Minimum quiet period after resolving before firing again | | `repeat_interval_s` | How often to re-deliver while still firing | -| `webhook_url` | Where to deliver | +| `email_recipients` | Array of up to 10 addresses to email; omit for none | +| `webhook_url` | Where to deliver by webhook | | `webhook_headers` | JSON-encoded object of extra headers, supplied as a string | | `webhook_template` | Go text/template payload body; omit for the default | | `notify_on_resolve` | Also deliver when the alert clears | @@ -85,8 +91,8 @@ An alert moves through three states: - **pending**: the condition is true but has not held for `for_seconds` yet. With `for_seconds: 0` a rule skips this and fires immediately. -- **firing**: it has held. The webhook is delivered, and re-delivered every - `repeat_interval_s` while it stays true. +- **firing**: it has held. The webhook and email are delivered, and re-delivered + every `repeat_interval_s` while it stays true. - **resolved**: the expression went false. If `notify_on_resolve` is set, that is delivered too. @@ -95,8 +101,8 @@ what stops a metric hovering at the threshold from delivering repeatedly. Each alert records its last delivery status and timestamp in `GET /api/alerting/alerts`. Setting `repeat_interval_s` to zero disables repeat delivery. Disabling a rule -removes pending alerts and resolves firing alerts, sending a resolve webhook -when `notify_on_resolve` is set. +removes pending alerts and resolves firing alerts, sending a resolve +notification when `notify_on_resolve` is set. :::caution[The namespace field on a rule is not applied] A rule stores `namespace`, but evaluation combines service rollups across @@ -104,6 +110,74 @@ namespaces. Scope with `service`; services with the same name in different namespaces are evaluated together. ::: +## Email delivery + +Email uses the [SMTP settings](/docs/reference/configuration/smtp) Fanout already +uses for sign-in codes. Without SMTP the option is disabled in the rule form, and +the API rejects `email_recipients` with the code `email_unavailable`. +`GET /api/alerting/email` reports whether email is available. + +Set `FANOUT_ALERTS_EMAIL_DEFAULT_TO` to a comma-separated list of addresses to +give every rule a destination. It applies only to a rule with no webhook and no +recipients of its own, including rules created before email existed. A rule with +a webhook and no recipients stays webhook-only. + +Email follows the rule's existing timing: `for_seconds`, `cooldown_s`, +`repeat_interval_s` and `notify_on_resolve` apply as they do to webhooks. + +Addresses are checked when saved. Each must be a bare address, without a display +name. They are stored in lowercase and a rule can have up to 10. + +### Retries and restarts + +A notification is queued in the control database in the same transaction as the +alert's state change, with one row per recipient. Fanout makes up to five +attempts, waiting 1, 5, 15, then 30 minutes between them. Connection failures, +including a connection lost partway through a session, and 4xx SMTP replies are +retried. A 5xx reply or an unusable address fails at once. Once the server has +accepted a message, a failure while closing the session does not count against +it. Before a retry, Fanout drops a recipient no longer on the rule, and deleting +a rule cancels its queued mail. + +Delivery is at least once. A restart resumes queued mail. If Fanout stops or the +worker panics while a message is being handed to the mail server, or a connection +drops before the server confirms it, Fanout cannot tell whether the server took +it, so it sends the message again. The repeat has the same `Message-ID`. The same +happens after a restart if the database could not record an accepted message +before Fanout stopped. + +### The hourly limit + +`FANOUT_ALERTS_EMAIL_HOURLY_LIMIT` (default 30) caps the alert emails sent to one +address in any rolling hour, across all rules. The count is stored apart from +rules, so neither a restart nor deleting a rule resets it. The last email within the limit says that the limit has been +reached. Notifications over the limit are suppressed: they are dropped, never +sent later, and logged as a warning. + +### Seeing what happened + +A rule whose latest email did not go through shows it in the **Rules** table, and +in `email_status` and `email_status_detail` on `GET /api/alerting/rules`: + +| `email_status` | Means | +|---|---| +| `sent` | Every recipient accepted the latest notification | +| `retrying` | An attempt failed and another is scheduled | +| `failed` | At least one recipient could not be reached | +| `suppressed` | At least one recipient was over the hourly limit | +| `unavailable` | The rule has recipients but SMTP is not configured | + +Viewers see `email_status` only. `email_status_detail` gives the failure class and +SMTP code, not the mail server's response text. + +For a rule with no webhook, the alert's delivery status under **History** carries +`success`, `failed` or `suppressed` for the email. When a rule has a webhook, that +column stays the webhook's result. + +Rule names, descriptions and service names are reduced to a single line of +printable text in the subject and body, and escaped in the HTML part, so a +hostile service name cannot add mail headers or markup. + ## Why alerts lag the raw data Evaluation reads analytical rollups. The rollup pass follows durable telemetry diff --git a/site/src/content/docs/docs/reference/api.mdx b/site/src/content/docs/docs/reference/api.mdx index 2589edf0c..ca70114ff 100644 --- a/site/src/content/docs/docs/reference/api.mdx +++ b/site/src/content/docs/docs/reference/api.mdx @@ -100,6 +100,7 @@ Operational responses at `/healthz`, `/readyz` and `/metrics` keep their bodies. | `GET` | `/api/agent/threads/:id` | `agent:run` | | `PATCH` | `/api/agent/threads/:id` | `agent:run` | | `GET` | `/api/alerting/alerts` | `telemetry:read` | +| `GET` | `/api/alerting/email` | `telemetry:read` | | `GET` | `/api/alerting/rules` | `telemetry:read` | | `POST` | `/api/alerting/rules` | `alerts:manage` | | `DELETE` | `/api/alerting/rules/:id` | `alerts:manage` | diff --git a/site/src/content/docs/docs/reference/configuration/alerts.mdx b/site/src/content/docs/docs/reference/configuration/alerts.mdx index 8e68a3544..ea00dacf4 100644 --- a/site/src/content/docs/docs/reference/configuration/alerts.mdx +++ b/site/src/content/docs/docs/reference/configuration/alerts.mdx @@ -1,9 +1,10 @@ --- title: "Alert settings" -description: "Whether the alert engine runs, how often it evaluates, and how long history is kept." -summary: "Settings for the alert evaluation loop." +description: "Whether the alert engine runs, how often it evaluates, how long history is kept, and how alert email is addressed and limited." +summary: "Settings for the alert evaluation loop and alert email." read_when: - "Alerts are evaluating too slowly, or not at all." + - "You are setting a default alert email destination or changing the per-address limit." status: preview generated: true --- @@ -17,12 +18,22 @@ Vendor credentials and endpoints keep the vendor's documented names. | Environment variable | Type | Default | |---|---|---| +| `FANOUT_ALERTS_EMAIL_DEFAULT_TO` | string | — | +| `FANOUT_ALERTS_EMAIL_HOURLY_LIMIT` | integer | `30` | | `FANOUT_ALERTS_ENABLED` | boolean | `true` | | `FANOUT_ALERTS_EVALUATION_INTERVAL` | duration | `30s` | | `FANOUT_ALERTS_HISTORY_DAYS` | integer | `7` | ## Notes +### `FANOUT_ALERTS_EMAIL_DEFAULT_TO` + +A comma-separated list of addresses that receive alerts from rules with no webhook and no email recipients of their own; requires SMTP. + +### `FANOUT_ALERTS_EMAIL_HOURLY_LIMIT` + +Caps the alert emails sent to one address in any rolling hour, across all rules. Notifications over the limit are suppressed, not delayed, and shown on the rule; 0 suppresses all alert email. + ### `FANOUT_ALERTS_ENABLED` Runs the alert evaluation engine. diff --git a/ui/host/src/alerts/api.ts b/ui/host/src/alerts/api.ts index b042b36a5..18ed6d4c2 100644 --- a/ui/host/src/alerts/api.ts +++ b/ui/host/src/alerts/api.ts @@ -5,9 +5,14 @@ export type Rule = { id: string; name: string; description?: string; enabled: boolean; service?: string; namespace?: string; expression: string; for_seconds: number; cooldown_s: number; repeat_interval_s: number; webhook_url?: string; webhook_headers?: string; webhook_template?: string; notify_on_resolve: boolean; + email_recipients?: string[]; + /** Server-owned result of the rule's latest email notification. */ + email_status?: "sent" | "retrying" | "failed" | "suppressed" | "unavailable" | string; email_status_detail?: string; email_status_at?: string; created_at: string; updated_at: string; }; -export type RuleDraft = Omit; +export type RuleDraft = Omit; +/** internal/alert/email.go EmailOptions. */ +export type EmailOptions = { available: boolean; message?: string; default_destination: boolean; max_recipients: number; hourly_limit?: number }; export type Alert = { id: string; rule_id: string; service: string; state: "pending" | "firing" | "resolved" | string; value?: number; fired_at?: string; resolved_at?: string; last_evaluated_at?: string; last_delivery_status?: string; last_delivery_at?: string; created_at: string; @@ -17,10 +22,12 @@ export type RuleTest = { results: { service: string; triggered: boolean; error?: export const rulesKey = ["alerting", "rules"] as const; export const alertsKey = ["alerting", "alerts"] as const; +export const emailOptionsKey = ["alerting", "email"] as const; const rulePath = (id: string) => `/api/alerting/rules/${encodeURIComponent(id)}`; export const listRules = (signal?: AbortSignal) => request<{ rules: Rule[] }>("/api/alerting/rules", { signal }).then(r => r.rules); export const listAlerts = (signal?: AbortSignal) => request<{ alerts: Alert[] }>("/api/alerting/alerts", { signal }).then(r => r.alerts); +export const getEmailOptions = (signal?: AbortSignal) => request("/api/alerting/email", { signal }); export const createRule = (rule: RuleDraft) => request("/api/alerting/rules", { method: "POST", json: rule }); export const replaceRule = (id: string, rule: RuleDraft) => request(rulePath(id), { method: "PUT", json: rule }); export const deleteRule = (id: string) => request(rulePath(id), { method: "DELETE" }); diff --git a/ui/host/src/alerts/email.test.tsx b/ui/host/src/alerts/email.test.tsx new file mode 100644 index 000000000..3400f6a9a --- /dev/null +++ b/ui/host/src/alerts/email.test.tsx @@ -0,0 +1,150 @@ +import { MantineProvider } from "@mantine/core"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { fireEvent, within } from "@testing-library/dom"; +import { act } from "react"; +import { createRoot } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { EmailOptions, Rule } from "./api"; + +const viewer = vi.hoisted(() => ({ role: "operator" })); +vi.mock("../auth", async original => ({ ...(await original()), useViewer: () => ({ id: "u", email: "u@example.test", display_name: "U", status: "active", role: viewer.role }) })); +import { AlertsPage } from "./page"; + +const rule: Rule = { id: "r1", name: "Checkout errors", enabled: true, service: "checkout", expression: "error_rate > 0.05", for_seconds: 60, cooldown_s: 300, repeat_interval_s: 3600, notify_on_resolve: false, created_at: "", updated_at: "" }; +const available: EmailOptions = { available: true, default_destination: true, max_recipients: 3, hourly_limit: 30 }; +const smtpMessage = "Email delivery is unavailable because SMTP is not configured."; +const unavailable: EmailOptions = { available: false, message: smtpMessage, default_destination: false, max_recipients: 3 }; +const fetchMock = vi.fn(); +const cleanups: (() => void)[] = []; +let rules: Rule[]; +let options: EmailOptions; +const posted = () => fetchMock.mock.calls.filter(([url, init]) => String(url) === "/api/alerting/rules" && init?.method === "POST").map(([, init]) => JSON.parse(String(init?.body))); + +beforeEach(() => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + vi.stubGlobal("fetch", fetchMock); + vi.stubGlobal("ResizeObserver", class { observe() {} unobserve() {} disconnect() {} }); + fetchMock.mockReset(); + viewer.role = "operator"; + rules = [rule]; + options = available; + fetchMock.mockImplementation(async (input, init) => { + const url = String(input), method = init?.method ?? "GET"; + if (url === "/api/alerting/rules" && method === "GET") return Response.json({ rules }); + if (url === "/api/alerting/alerts") return Response.json({ alerts: [] }); + if (url === "/api/alerting/email") return Response.json(options); + if (url === "/api/alerting/rules" && method === "POST") return Response.json({ ...rule, id: "r2" }, { status: 201 }); + if (url === "/api/alerting/rules/r1" && method === "PUT") return Response.json(rule); + if (url === "/api/telemetry/schema?window=1h") return Response.json({ services: [] }); + throw new Error(`Unexpected request: ${method} ${url}`); + }); +}); +afterEach(async () => { await act(async () => cleanups.splice(0).forEach(cleanup => cleanup())); vi.unstubAllGlobals(); document.body.innerHTML = ""; }); + +async function render() { + const host = document.createElement("div"); document.body.append(host); + const root = createRoot(host), client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + await act(async () => root.render()); + cleanups.push(() => { root.unmount(); client.clear(); }); + const ui = within(document.body); + const wait = async (assertion: () => void) => vi.waitFor(async () => { await act(async () => {}); assertion(); }, { timeout: 3000, interval: 10 }); + const input = async (label: RegExp, value: string) => act(async () => { fireEvent.change(ui.getByLabelText(label, { selector: "input, textarea" }), { target: { value } }); }); + const openNew = async () => { await wait(() => ui.getByRole("button", { name: "New rule" })); await act(async () => { fireEvent.click(ui.getAllByRole("button", { name: "New rule" })[0]); }); await wait(() => ui.getByLabelText(/^Email recipients/, { selector: "input" })); }; + return { ui, wait, input, openNew }; +} + +const field = (ui: ReturnType) => ui.getByLabelText(/^Email recipients/, { selector: "input" }) as HTMLInputElement; + +describe("alert rule email", () => { + it("sends the addresses entered on a new rule", async () => { + const { ui, wait, input, openNew } = await render(); + await openNew(); + await wait(() => expect(field(ui).disabled).toBe(false)); + await input(/^Name/, "Errors"); + await input(/^Expression/, "error_rate > 0.05"); + await input(/^Email recipients/, "a@example.com, b@example.com"); + await act(async () => { fireEvent.click(ui.getByRole("button", { name: "Create rule" })); }); + await wait(() => expect(posted()[0]?.email_recipients).toEqual(["a@example.com", "b@example.com"])); + }); + + it("explains the default destination when one is configured", async () => { + const { ui, wait, openNew } = await render(); + await openNew(); + await wait(() => ui.getByText(/use the default destination/)); + }); + + it.each([["not-an-address", /is not an email address/], ["a@e.co b@e.co c@e.co d@e.co", /at most 3/]])("blocks saving %s", async (value, message) => { + const { ui, wait, input, openNew } = await render(); + await openNew(); + await wait(() => expect(field(ui).disabled).toBe(false)); + await input(/^Name/, "Errors"); + await input(/^Expression/, "true"); + await input(/^Email recipients/, value); + await wait(() => ui.getByText(message)); + expect((ui.getByRole("button", { name: "Create rule" }) as HTMLButtonElement).disabled).toBe(true); + }); + + it("disables the option and says why without SMTP", async () => { + options = unavailable; + const { ui, wait, openNew } = await render(); + await openNew(); + await wait(() => ui.getByText(smtpMessage)); + expect(field(ui).disabled).toBe(true); + }); + + it("keeps a saved rule editable without SMTP so its recipients can be removed", async () => { + options = unavailable; + rules = [{ ...rule, email_recipients: ["a@example.com"], email_status: "unavailable", email_status_detail: smtpMessage }]; + const { ui, wait } = await render(); + await wait(() => ui.getByRole("button", { name: "Checkout errors actions" })); + await act(async () => ui.getByRole("button", { name: "Checkout errors actions" }).click()); + await act(async () => ui.getByRole("menuitem", { name: "Edit", hidden: true }).click()); + await wait(() => expect(field(ui).value).toBe("a@example.com")); + expect(field(ui).disabled).toBe(false); + }); + + it.each([ + ["failed", "Email failed", "1 of 1 recipients failed: 550 no such user"], + ["unavailable", "Email unavailable", smtpMessage], + ["suppressed", "Email suppressed", "1 of 1 recipients suppressed: Over the limit of 30 emails per hour to this address"], + ["retrying", "Email retrying", "Attempt 1 of 5 failed: 451"], + ])("shows a %s email on the rule", async (status, label, detail) => { + rules = [{ ...rule, email_recipients: ["a@example.com"], email_status: status, email_status_detail: detail }]; + const { ui, wait } = await render(); + await wait(() => ui.getByText(label)); + ui.getByText(detail); + }); + + it("renders the server's reduced rule for a read-only user", async () => { + viewer.role = "viewer"; + rules = [{ ...rule, email_status: "failed" }]; + const { ui, wait } = await render(); + await wait(() => ui.getByText("Email failed")); + ui.getByText("Checkout errors"); + ui.getByText("error_rate > 0.05"); + expect(ui.queryByRole("button", { name: "Checkout errors actions" })).toBeNull(); + expect(ui.queryByRole("button", { name: "New rule" })).toBeNull(); + expect(ui.getByRole("switch", { name: "Enable Checkout errors" }).disabled).toBe(true); + expect(fetchMock.mock.calls.some(([, init]) => (init?.method ?? "GET") !== "GET")).toBe(false); + }); + + it("shows nothing on the rule when email was sent", async () => { + rules = [{ ...rule, email_recipients: ["a@example.com"], email_status: "sent" }]; + const { ui, wait } = await render(); + await wait(() => ui.getByText("Checkout errors")); + expect(ui.queryByText(/^Email /)).toBeNull(); + }); + + it("does not send server-owned status back on save", async () => { + rules = [{ ...rule, email_recipients: ["a@example.com"], email_status: "failed", email_status_detail: "x", email_status_at: "t" }]; + const { ui, wait } = await render(); + await wait(() => ui.getByRole("switch", { name: "Enable Checkout errors" })); + await act(async () => { fireEvent.click(ui.getByRole("switch", { name: "Enable Checkout errors" })); }); + await wait(() => expect(fetchMock.mock.calls.some(([, init]) => init?.method === "PUT")).toBe(true)); + const body = JSON.parse(String(fetchMock.mock.calls.find(([, init]) => init?.method === "PUT")?.[1]?.body)); + expect(body.email_recipients).toEqual(["a@example.com"]); + expect(body).not.toHaveProperty("email_status"); + expect(body).not.toHaveProperty("email_status_detail"); + expect(body).not.toHaveProperty("email_status_at"); + }); +}); diff --git a/ui/host/src/alerts/page.tsx b/ui/host/src/alerts/page.tsx index 84f372bd7..283ce789c 100644 --- a/ui/host/src/alerts/page.tsx +++ b/ui/host/src/alerts/page.tsx @@ -28,6 +28,14 @@ function ruleQuery(rule: Pick): string { /** Rules read per-service rollups with error_rate from 0 to 1; Explore reads raw telemetry and shows percent. */ const exploreNote = "Open in Explore charts raw telemetry, so values can differ slightly from a rule's rollups. Explore shows error rate as a percent: a rule's 0.05 is 5 there."; +/** Email outcomes that need attention on a rule; a sent email does not. */ +const emailIssues: Record = { + failed: { tone: "danger", label: "Email failed" }, + unavailable: { tone: "danger", label: "Email unavailable" }, + suppressed: { tone: "warning", label: "Email suppressed" }, + retrying: { tone: "warning", label: "Email retrying" }, +}; + /** Alert rules and their history (proposal 0004, section 4); works without chat. */ export function AlertsPage({ onExplore }: { onExplore(query: string): void }) { const client = useQueryClient(); @@ -80,7 +88,8 @@ export function AlertsPage({ onExplore }: { onExplore(query: string): void }) { {rules.data?.map(rule => { const now = active(rule); return - {rule.name}{rule.description && {rule.description}} + {rule.name}{rule.description && {rule.description}} + {rule.email_status && emailIssues[rule.email_status] && {emailIssues[rule.email_status].label}{rule.email_status_detail && {rule.email_status_detail}}} {rule.service || "All services"} {rule.expression} @@ -141,6 +150,6 @@ export function AlertsPage({ onExplore }: { onExplore(query: string): void }) { } function draftFields(rule: Rule): RuleDraft { - const { id: _id, created_at: _created, updated_at: _updated, ...draft } = rule; + const { id: _id, created_at: _created, updated_at: _updated, email_status: _status, email_status_detail: _detail, email_status_at: _at, ...draft } = rule; return draft; } diff --git a/ui/host/src/alerts/rule-form.tsx b/ui/host/src/alerts/rule-form.tsx index 973c99d97..2a4408ac8 100644 --- a/ui/host/src/alerts/rule-form.tsx +++ b/ui/host/src/alerts/rule-form.tsx @@ -3,7 +3,7 @@ import { Box, Autocomplete, Button, Group, Menu, Modal, NumberInput, Stack, Swi import { useQuery } from "@tanstack/react-query"; import { useState } from "react"; import { useDiscard } from "../dashboards/discard"; -import { listServices, type Rule, type RuleDraft } from "./api"; +import { emailOptionsKey, getEmailOptions, listServices, type Rule, type RuleDraft } from "./api"; import { percentMistake } from "./values"; /** Starters for the documented variables (site docs: reference/alert-expressions). */ @@ -18,7 +18,7 @@ const blank: RuleDraft = { name: "", description: "", enabled: true, service: "" function draftOf(rule?: Rule): RuleDraft { if (!rule) return blank; - const { id: _id, created_at: _created, updated_at: _updated, ...draft } = rule; + const { id: _id, created_at: _created, updated_at: _updated, email_status: _status, email_status_detail: _detail, email_status_at: _at, ...draft } = rule; return { ...blank, ...draft }; } @@ -41,6 +41,16 @@ function headersProblem(headers: string | undefined): string | undefined { return undefined; } +const parseRecipients = (text: string) => text.split(/[,\s]+/).filter(Boolean); +/** A bare address; the server does the full check (internal/auth/email.go NormalizeRecipients). */ +const bareAddress = /^[^\s@,<>]+@[^\s@,<>]+$/; + +function recipientsProblem(recipients: string[], max: number): string | undefined { + const invalid = recipients.find(recipient => !bareAddress.test(recipient)); + if (invalid) return `"${invalid}" is not an email address`; + return new Set(recipients.map(r => r.toLowerCase())).size > max ? `Enter at most ${max} addresses` : undefined; +} + /** Creates or replaces a rule; the server checks the expression and returns why it is invalid. */ export function RuleForm({ rule, onClose, onSave }: { rule?: Rule; onClose(): void; onSave(draft: RuleDraft): Promise }) { const [draft, setDraft] = useState(() => draftOf(rule)); @@ -50,7 +60,13 @@ export function RuleForm({ rule, onClose, onSave }: { rule?: Rule; onClose(): vo const set = (patch: Partial) => { setDraft(current => ({ ...current, ...patch })); setError(undefined); }; // Like the server, check headers only when they change, so other edits still save. const headers = draft.webhook_headers === rule?.webhook_headers ? undefined : headersProblem(draft.webhook_headers); - const ready = draft.name.trim() !== "" && draft.expression.trim() !== "" && !headers; + const email = useQuery({ queryKey: emailOptionsKey, queryFn: ({ signal }) => getEmailOptions(signal), staleTime: 60_000, retry: false }); + const [emailText, setEmailText] = useState(() => (rule?.email_recipients ?? []).join(", ")); + const savedRecipients = (rule?.email_recipients?.length ?? 0) > 0; + const recipientsChanged = JSON.stringify(draft.email_recipients ?? []) !== JSON.stringify(rule?.email_recipients ?? []); + // Like the server, check recipients only when they change, so a rule stays editable after SMTP is removed. + const recipients = recipientsChanged ? recipientsProblem(draft.email_recipients ?? [], email.data?.max_recipients ?? 10) : undefined; + const ready = draft.name.trim() !== "" && draft.expression.trim() !== "" && !headers && !recipients; // A misspelled service saves, but would never fire; say so without blocking. const services = useQuery({ queryKey: ["alerting", "services"], queryFn: ({ signal }) => listServices(signal), staleTime: 60_000, retry: false }); const service = draft.service?.trim() ?? ""; @@ -90,6 +106,13 @@ export function RuleForm({ rule, onClose, onSave }: { rule?: Rule; onClose(): vo {seconds("Cooldown (s)", "cooldown_s", "Quiet period after resolving")} {seconds("Repeat every (s)", "repeat_interval_s", "0 delivers once")} + + { const text = event.currentTarget.value, parsed = parseRecipients(text); setEmailText(text); set({ email_recipients: parsed.length > 0 ? parsed : undefined }); }} /> + {email.data && !email.data.available && {email.data.message}} + {email.isError && Email availability could not be checked.} + set({ webhook_url: event.currentTarget.value })} />