[ENGMP-365] Add protection triggered counter metric and log action logging

pkg/queryfrontend/protection_middleware.go

@@ -19,10 +19,11 @@ import (
 )
 
 type protectionMiddleware struct {
-	next         queryrange.Handler
-	engine       *ProtectionEngine
-	logger       log.Logger
-	totalLatency prometheus.Histogram
+	next           queryrange.Handler
+	engine         *ProtectionEngine
+	logger         log.Logger
+	totalLatency   prometheus.Histogram
+	triggeredTotal *prometheus.CounterVec
 }
 
 // NewProtectionMiddleware creates a new middleware that applies protection rules to queries.
@@ -34,13 +35,20 @@ func NewProtectionMiddleware(engine *ProtectionEngine, logger log.Logger, reg pr
 		Help:      "Total duration of the protection middleware, including parsing and evaluation.",
 		Buckets:   []float64{0.0001, 0.00025, 0.0005, 0.001, 0.005, 0.01, 0.1, 1.0, 10.0},
 	})
+	triggeredTotal := promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
+		Namespace: "thanos",
+		Subsystem: "query_frontend",
+		Name:      "protection_triggered_total",
+		Help:      "Total number of queries matched by a protection rule, labeled by action and rule name.",
+	}, []string{"action", "rule"})
 
 	return queryrange.MiddlewareFunc(func(next queryrange.Handler) queryrange.Handler {
 		return &protectionMiddleware{
-			next:         next,
-			engine:       engine,
-			logger:       logger,
-			totalLatency: totalLatency,
+			next:           next,
+			engine:         engine,
+			logger:         logger,
+			totalLatency:   totalLatency,
+			triggeredTotal: triggeredTotal,
 		}
 	})
 }
@@ -80,16 +88,28 @@ func (m *protectionMiddleware) Do(ctx context.Context, r queryrange.Request) (qu
 
 	if protectionResult != nil {
 		ctx = WithProtectionResult(ctx, protectionResult)
-	}
-
-	// Return 400 Bad Request error code if a protection rule is triggered.
-	if protectionResult != nil && protectionResult.Action == RuleActionBlock {
-		return nil, httpgrpc.Errorf(http.StatusBadRequest, "query blocked by protection rule: %s", protectionResult.RuleName)
+		if err := m.applyProtectionResult(protectionResult, query); err != nil {
+			return nil, err
+		}
 	}
 
 	return m.next.Do(ctx, r)
 }
 
+// applyProtectionResult performs the action indicated by the protection result.
+// Returns an error if the query should be blocked, nil otherwise.
+func (m *protectionMiddleware) applyProtectionResult(result *ProtectionResult, query string) error {
+	switch result.Action {
+	case RuleActionBlock:
+		m.triggeredTotal.WithLabelValues("block", result.RuleName).Inc()
+		return httpgrpc.Errorf(http.StatusForbidden, "query blocked by protection rule: %s", result.RuleName)
+	case RuleActionLog:
+		m.triggeredTotal.WithLabelValues("log", result.RuleName).Inc()
+		level.Info(m.logger).Log("msg", "protection rule triggered", "rule", result.RuleName, "action", "log", "query", query)
+	}
+	return nil
+}
+
 // extractActor returns the actor identifier from the request headers.
 // Uses X-Source header as the actor identifier.
 func extractActor(r queryrange.Request) string {

pkg/queryfrontend/protection_middleware_test.go

@@ -87,7 +87,7 @@ func TestProtectionMiddleware_RuleActionLog_PassesThrough(t *testing.T) {
 	require.True(t, called)
 }
 
-func TestProtectionMiddleware_RuleActionBlock_Returns400(t *testing.T) {
+func TestProtectionMiddleware_RuleActionBlock_Returns403(t *testing.T) {
 	called := false
 	next := queryrange.HandlerFunc(func(ctx context.Context, r queryrange.Request) (queryrange.Response, error) {
 		called = true
@@ -104,7 +104,7 @@ func TestProtectionMiddleware_RuleActionBlock_Returns400(t *testing.T) {
 
 	httpErr, ok := httpgrpc.HTTPResponseFromError(err)
 	require.True(t, ok)
-	require.Equal(t, int32(http.StatusBadRequest), httpErr.Code)
+	require.Equal(t, int32(http.StatusForbidden), httpErr.Code)
 }
 
 func TestProtectionMiddleware_ProtectionResultInContext(t *testing.T) {

test/e2e/query_frontend_test.go

@@ -1215,7 +1215,7 @@ func TestQueryFrontendProtection(t *testing.T) {
 	q := e2ethanos.NewQuerierBuilder(e, "1").Init()
 	testutil.Ok(t, e2e.StartAndWaitReady(q))
 
-	t.Run("block action returns 400", func(t *testing.T) {
+	t.Run("block action returns 403", func(t *testing.T) {
 		qfe := e2ethanos.NewQueryFrontend(e, "block", "http://"+q.InternalEndpoint("http"),
 			queryfrontend.Config{},
 			queryfrontend.CacheProviderConfig{Type: queryfrontend.INMEMORY},
@@ -1237,7 +1237,7 @@ rules:
 		require.NoError(t, err)
 		t.Cleanup(func() { require.NoError(t, resp.Body.Close()) })
 
-		require.Equal(t, http.StatusBadRequest, resp.StatusCode)
+		require.Equal(t, http.StatusForbidden, resp.StatusCode)
 	})
 
 	t.Run("log action passes through", func(t *testing.T) {