config: report all receiver validation errors instead of only the first

config/common/notifierconfig.go

@@ -13,6 +13,13 @@
 
 package common
 
+import (
+	"errors"
+	"strings"
+
+	"gopkg.in/yaml.v2"
+)
+
 // NotifierConfig contains base options common across all notifier configurations.
 type NotifierConfig struct {
 	VSendResolved bool `yaml:"send_resolved" json:"send_resolved"`
@@ -28,3 +35,43 @@ func (nc *NotifierConfig) SendResolved() bool {
 type Validator interface {
 	Validate() error
 }
+
+// AsValidationError returns err as a *yaml.TypeError so that the YAML decoder
+// records it and continues decoding instead of aborting at the first invalid
+// notifier config. The decoder collects every such error across the document,
+// which is what allows a configuration to report all of its invalid notifier
+// configs at once. Errors that already are a *yaml.TypeError, including the
+// decoder's own type and unknown field errors, are returned unchanged, and so
+// is nil.
+func AsValidationError(err error) error {
+	if err == nil {
+		return nil
+	}
+	if te, ok := errors.AsType[*yaml.TypeError](err); ok {
+		return te
+	}
+	return &yaml.TypeError{Errors: []string{err.Error()}}
+}
+
+// FlattenValidationErrors turns the errors collected through AsValidationError
+// back into plain errors at the top level of the configuration, so that a single
+// validation error reads exactly as it did before and several read one per
+// line. Errors reported by the decoder itself, which carry a "line N:" prefix,
+// keep their usual *yaml.TypeError form.
+func FlattenValidationErrors(err error) error {
+	te, ok := errors.AsType[*yaml.TypeError](err)
+	if !ok {
+		return err
+	}
+	errs := make([]error, 0, len(te.Errors))
+	for _, msg := range te.Errors {
+		if strings.HasPrefix(msg, "line ") {
+			return te
+		}
+		errs = append(errs, errors.New(msg))
+	}
+	if len(errs) == 1 {
+		return errs[0]
+	}
+	return errors.Join(errs...)
+}

config/config.go

@@ -317,7 +317,7 @@ func (c *Config) UnmarshalYAML(unmarshal func(any) error) error {
 	// again, we have to hide it using a type indirection.
 	type plain Config
 	if err := unmarshal((*plain)(c)); err != nil {
-		return err
+		return amcommoncfg.FlattenValidationErrors(err)
 	}
 
 	// If a global block was open but empty the default global config is overwritten.

config/config_test.go

@@ -2017,3 +2017,103 @@ receivers:
 		t.Errorf("expected local proxy_url %q, got %q", "http://local-proxy.example.com:8080", got)
 	}
 }
+
+func TestReceiverValidationErrorsAccumulate(t *testing.T) {
+	tests := []struct {
+		name     string
+		in       string
+		expected string
+	}{
+		{
+			// https://github.com/prometheus/alertmanager/issues/4990
+			name: "multiple invalid notifier configs in a single receiver",
+			in: `
+route:
+    receiver: team-X
+
+receivers:
+- name: 'team-X'
+  webhook_configs:
+  - send_resolved: true
+  - url: 'http://example.com/'
+    url_file: '/etc/secrets/webhook-url'
+`,
+			expected: "one of url or url_file must be configured\n" +
+				"at most one of url & url_file must be configured",
+		},
+		{
+			// The example from https://github.com/prometheus/alertmanager/issues/4990:
+			// the decoder's own type error and the validation error are reported
+			// together instead of the latter hiding the former.
+			name: "type errors and validation errors are reported together",
+			in: `
+receivers:
+  - name: 'webhook-critical'
+    webhook_configs:
+      - url: 'http://example.com/api'
+        send_resolved: "yes_please"
+      - url: ''
+        http_config:
+          basic_auth:
+            username: 'admin'
+route:
+  group_by: ['alertname']
+  receiver: 'webhook-critical'
+`,
+			expected: "yaml: unmarshal errors:\n" +
+				"  line 6: cannot unmarshal !!str `yes_please` into bool\n" +
+				"  one of url or url_file must be configured",
+		},
+		{
+			name: "invalid notifier configs of different types in a single receiver",
+			in: `
+route:
+    receiver: team-X
+
+receivers:
+- name: 'team-X'
+  webhook_configs:
+  - send_resolved: true
+  pagerduty_configs:
+  - url: 'https://example.com/'
+`,
+			expected: "one of url or url_file must be configured\n" +
+				"missing service or routing key in PagerDuty config",
+		},
+		{
+			// https://github.com/prometheus/alertmanager/issues/4991
+			name: "invalid notifier configs across multiple receivers",
+			in: `
+route:
+    receiver: team-A
+
+receivers:
+- name: 'team-A'
+  webhook_configs:
+  - send_resolved: true
+- name: 'team-B'
+  email_configs:
+  - smarthost: 'smtp.example.com:587'
+    from: 'alertmanager@example.com'
+- name: 'team-C'
+  pagerduty_configs:
+  - url: 'https://example.com/'
+`,
+			expected: "one of url or url_file must be configured\n" +
+				"missing to address in email config\n" +
+				"missing service or routing key in PagerDuty config",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			_, err := Load(tc.in)
+			if err == nil {
+				t.Fatalf("no error returned, expected:\n%v", tc.expected)
+			}
+			if err.Error() != tc.expected {
+				t.Errorf("\nexpected:\n%v\ngot:\n%v", tc.expected, err.Error())
+			}
+		})
+	}
+}

config/notifiers.go

@@ -105,9 +105,9 @@ func (c *WebexConfig) UnmarshalYAML(unmarshal func(any) error) error {
 	*c = DefaultWebexConfig
 	type plain WebexConfig
 	if err := unmarshal((*plain)(c)); err != nil {
-		return err
+		return amcommoncfg.AsValidationError(err)
 	}
-	return c.Validate()
+	return amcommoncfg.AsValidationError(c.Validate())
 }
 
 func (c *WebexConfig) Validate() error {
@@ -161,7 +161,7 @@ func (c *EmailConfig) UnmarshalYAML(unmarshal func(any) error) error {
 	*c = DefaultEmailConfig
 	type plain EmailConfig
 	if err := unmarshal((*plain)(c)); err != nil {
-		return err
+		return amcommoncfg.AsValidationError(err)
 	}
 	// Header names are case insensitive. The normalization loop below
 	// detects duplicates and builds a canonical header map in one pass.
@@ -171,13 +171,13 @@ func (c *EmailConfig) UnmarshalYAML(unmarshal func(any) error) error {
 	for h, v := range c.Headers {
 		normalized := textproto.CanonicalMIMEHeaderKey(h)
 		if _, ok := normalizedHeaders[normalized]; ok {
-			return fmt.Errorf("duplicate header %q in email config", normalized)
+			return amcommoncfg.AsValidationError(fmt.Errorf("duplicate header %q in email config", normalized))
 		}
 		normalizedHeaders[normalized] = v
 	}
 	c.Headers = normalizedHeaders
 
-	return c.Validate()
+	return amcommoncfg.AsValidationError(c.Validate())
 }
 
 func (c *EmailConfig) Validate() error {
@@ -217,7 +217,7 @@ type SlackAction struct {
 func (c *SlackAction) UnmarshalYAML(unmarshal func(any) error) error {
 	type plain SlackAction
 	if err := unmarshal((*plain)(c)); err != nil {
-		return err
+		return amcommoncfg.AsValidationError(err)
 	}
 	if c.URL != "" {
 		// Clear all message action fields.
@@ -227,9 +227,9 @@ func (c *SlackAction) UnmarshalYAML(unmarshal func(any) error) error {
 	} else if c.Name != "" {
 		c.URL = ""
 	} else {
-		return errors.New("missing name or url in Slack action configuration")
+		return amcommoncfg.AsValidationError(errors.New("missing name or url in Slack action configuration"))
 	}
-	return c.Validate()
+	return amcommoncfg.AsValidationError(c.Validate())
 }
 
 func (c *SlackAction) Validate() error {
@@ -256,9 +256,9 @@ type SlackConfirmationField struct {
 func (c *SlackConfirmationField) UnmarshalYAML(unmarshal func(any) error) error {
 	type plain SlackConfirmationField
 	if err := unmarshal((*plain)(c)); err != nil {
-		return err
+		return amcommoncfg.AsValidationError(err)
 	}
-	return c.Validate()
+	return amcommoncfg.AsValidationError(c.Validate())
 }
 
 func (c *SlackConfirmationField) Validate() error {
@@ -282,9 +282,9 @@ type SlackField struct {
 func (c *SlackField) UnmarshalYAML(unmarshal func(any) error) error {
 	type plain SlackField
 	if err := unmarshal((*plain)(c)); err != nil {
-		return err
+		return amcommoncfg.AsValidationError(err)
 	}
-	return c.Validate()
+	return amcommoncfg.AsValidationError(c.Validate())
 }
 
 func (c *SlackField) Validate() error {
@@ -346,9 +346,9 @@ func (c *SlackConfig) UnmarshalYAML(unmarshal func(any) error) error {
 	*c = DefaultSlackConfig
 	type plain SlackConfig
 	if err := unmarshal((*plain)(c)); err != nil {
-		return err
+		return amcommoncfg.AsValidationError(err)
 	}
-	return c.Validate()
+	return amcommoncfg.AsValidationError(c.Validate())
 }
 
 func (c *SlackConfig) Validate() error {
@@ -396,14 +396,14 @@ func (c *WechatConfig) UnmarshalYAML(unmarshal func(any) error) error {
 	*c = DefaultWechatConfig
 	type plain WechatConfig
 	if err := unmarshal((*plain)(c)); err != nil {
-		return err
+		return amcommoncfg.AsValidationError(err)
 	}
 
 	if c.MessageType == "" {
 		c.MessageType = "text"
 	}
 
-	return c.Validate()
+	return amcommoncfg.AsValidationError(c.Validate())
 }
 
 func (c *WechatConfig) Validate() error {
@@ -440,9 +440,9 @@ func (c *VictorOpsConfig) UnmarshalYAML(unmarshal func(any) error) error {
 	*c = DefaultVictorOpsConfig
 	type plain VictorOpsConfig
 	if err := unmarshal((*plain)(c)); err != nil {
-		return err
+		return amcommoncfg.AsValidationError(err)
 	}
-	return c.Validate()
+	return amcommoncfg.AsValidationError(c.Validate())
 }
 
 func (c *VictorOpsConfig) Validate() error {

config/notifiers_test.go

@@ -35,7 +35,7 @@ to: ''
 	if err == nil {
 		t.Fatalf("no error returned, expected:\n%v", expected)
 	}
-	if err.Error() != expected {
+	if err.Error() != "yaml: unmarshal errors:\n  "+expected {
 		t.Errorf("\nexpected:\n%v\ngot:\n%v", expected, err.Error())
 	}
 }
@@ -55,7 +55,7 @@ headers:
 	if err == nil {
 		t.Fatalf("no error returned, expected:\n%v", expected)
 	}
-	if err.Error() != expected {
+	if err.Error() != "yaml: unmarshal errors:\n  "+expected {
 		t.Errorf("\nexpected:\n%v\ngot:\n%v", expected, err.Error())
 	}
 }
@@ -126,7 +126,7 @@ routing_key: ''
 		if err == nil {
 			t.Fatalf("no error returned, expected:\n%v", expected)
 		}
-		if err.Error() != expected {
+		if err.Error() != "yaml: unmarshal errors:\n  "+expected {
 			t.Errorf("\nexpected:\n%v\ngot:\n%v", expected, err.Error())
 		}
 	})
@@ -145,7 +145,7 @@ api_key_file: /global_file
 		if err == nil {
 			t.Fatalf("no error returned, expected:\n%v", expected)
 		}
-		if err.Error() != expected {
+		if err.Error() != "yaml: unmarshal errors:\n  "+expected {
 			t.Errorf("\nexpected:\n%v\ngot:\n%v", expected, err.Error())
 		}
 	})
@@ -165,7 +165,7 @@ custom_fields:
 	if err == nil {
 		t.Fatalf("no error returned, expected:\n%v", expected)
 	}
-	if err.Error() != expected {
+	if err.Error() != "yaml: unmarshal errors:\n  "+expected {
 		t.Errorf("\nexpected:\n%v\ngot:\n%v", expected, err.Error())
 	}
 
@@ -304,7 +304,7 @@ api_url: 'https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXX
 			t.Fatalf("\nno error returned, expected:\n%v", rt.expectedErr)
 		}
 		// Check that the error that occurred was what was expected.
-		if err != nil && err.Error() != rt.expectedErr {
+		if err != nil && err.Error() != "yaml: unmarshal errors:\n  "+rt.expectedErr {
 			t.Errorf("\nexpected:\n%v\ngot:\n%v", rt.expectedErr, err.Error())
 		}
 	}
@@ -361,7 +361,7 @@ fields:
 			t.Fatalf("\nno error returned, expected:\n%v", rt.expected)
 		}
 		// Check that the error that occurred was what was expected.
-		if err != nil && err.Error() != rt.expected {
+		if err != nil && err.Error() != "yaml: unmarshal errors:\n  "+rt.expected {
 			t.Errorf("\nexpected:\n%v\ngot:\n%v", rt.expected, err.Error())
 		}
 	}
@@ -551,7 +551,11 @@ http_config:
 			var cfg WebexConfig
 			err := yaml.UnmarshalStrict([]byte(tt.in), &cfg)
 
-			require.Equal(t, tt.expected, err)
+			if tt.expected != nil {
+				require.EqualError(t, err, "yaml: unmarshal errors:\n  "+tt.expected.Error())
+			} else {
+				require.NoError(t, err)
+			}
 		})
 	}
 }
@@ -609,7 +613,11 @@ headers: {X-Custom-Header: CustomValue, X-CUSTOM-HEADER: AnotherValue}
 			var cfg EmailConfig
 			err := yaml.UnmarshalStrict([]byte(tt.in), &cfg)
 
-			require.Equal(t, tt.expected, err)
+			if tt.expected != nil {
+				require.EqualError(t, err, "yaml: unmarshal errors:\n  "+tt.expected.Error())
+			} else {
+				require.NoError(t, err)
+			}
 		})
 	}
 }

notify/discord/config.go

@@ -50,10 +50,10 @@ func (c *DiscordConfig) UnmarshalYAML(unmarshal func(any) error) error {
 	*c = defaultDiscordConfig
 	type plain DiscordConfig
 	if err := unmarshal((*plain)(c)); err != nil {
-		return err
+		return amcommoncfg.AsValidationError(err)
 	}
 
-	return c.Validate()
+	return amcommoncfg.AsValidationError(c.Validate())
 }
 
 // Validate checks the DiscordConfig for correctness.