diff --git a/pkg/queryfrontend/protection_middleware.go b/pkg/queryfrontend/protection_middleware.go index 8f6e9ec3234..ef201116916 100644 --- a/pkg/queryfrontend/protection_middleware.go +++ b/pkg/queryfrontend/protection_middleware.go @@ -19,10 +19,11 @@ import ( ) type protectionMiddleware struct { - next queryrange.Handler - engine *ProtectionEngine - logger log.Logger - totalLatency prometheus.Histogram + next queryrange.Handler + engine *ProtectionEngine + logger log.Logger + totalLatency prometheus.Histogram + triggeredTotal *prometheus.CounterVec } // NewProtectionMiddleware creates a new middleware that applies protection rules to queries. @@ -34,13 +35,20 @@ func NewProtectionMiddleware(engine *ProtectionEngine, logger log.Logger, reg pr Help: "Total duration of the protection middleware, including parsing and evaluation.", Buckets: []float64{0.0001, 0.00025, 0.0005, 0.001, 0.005, 0.01, 0.1, 1.0, 10.0}, }) + triggeredTotal := promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ + Namespace: "thanos", + Subsystem: "query_frontend", + Name: "protection_triggered_total", + Help: "Total number of queries matched by a protection rule, labeled by action and rule name.", + }, []string{"action", "rule"}) return queryrange.MiddlewareFunc(func(next queryrange.Handler) queryrange.Handler { return &protectionMiddleware{ - next: next, - engine: engine, - logger: logger, - totalLatency: totalLatency, + next: next, + engine: engine, + logger: logger, + totalLatency: totalLatency, + triggeredTotal: triggeredTotal, } }) } @@ -80,16 +88,28 @@ func (m *protectionMiddleware) Do(ctx context.Context, r queryrange.Request) (qu if protectionResult != nil { ctx = WithProtectionResult(ctx, protectionResult) - } - - // Return 400 Bad Request error code if a protection rule is triggered. - if protectionResult != nil && protectionResult.Action == RuleActionBlock { - return nil, httpgrpc.Errorf(http.StatusBadRequest, "query blocked by protection rule: %s", protectionResult.RuleName) + if err := m.applyProtectionResult(protectionResult, query); err != nil { + return nil, err + } } return m.next.Do(ctx, r) } +// applyProtectionResult performs the action indicated by the protection result. +// Returns an error if the query should be blocked, nil otherwise. +func (m *protectionMiddleware) applyProtectionResult(result *ProtectionResult, query string) error { + switch result.Action { + case RuleActionBlock: + m.triggeredTotal.WithLabelValues("block", result.RuleName).Inc() + return httpgrpc.Errorf(http.StatusForbidden, "query blocked by protection rule: %s", result.RuleName) + case RuleActionLog: + m.triggeredTotal.WithLabelValues("log", result.RuleName).Inc() + level.Info(m.logger).Log("msg", "protection rule triggered", "rule", result.RuleName, "action", "log", "query", query) + } + return nil +} + // extractActor returns the actor identifier from the request headers. // Uses X-Source header as the actor identifier. func extractActor(r queryrange.Request) string { diff --git a/pkg/queryfrontend/protection_middleware_test.go b/pkg/queryfrontend/protection_middleware_test.go index 817f317218d..004ccd39502 100644 --- a/pkg/queryfrontend/protection_middleware_test.go +++ b/pkg/queryfrontend/protection_middleware_test.go @@ -87,7 +87,7 @@ func TestProtectionMiddleware_RuleActionLog_PassesThrough(t *testing.T) { require.True(t, called) } -func TestProtectionMiddleware_RuleActionBlock_Returns400(t *testing.T) { +func TestProtectionMiddleware_RuleActionBlock_Returns403(t *testing.T) { called := false next := queryrange.HandlerFunc(func(ctx context.Context, r queryrange.Request) (queryrange.Response, error) { called = true @@ -104,7 +104,7 @@ func TestProtectionMiddleware_RuleActionBlock_Returns400(t *testing.T) { httpErr, ok := httpgrpc.HTTPResponseFromError(err) require.True(t, ok) - require.Equal(t, int32(http.StatusBadRequest), httpErr.Code) + require.Equal(t, int32(http.StatusForbidden), httpErr.Code) } func TestProtectionMiddleware_ProtectionResultInContext(t *testing.T) { diff --git a/test/e2e/query_frontend_test.go b/test/e2e/query_frontend_test.go index 55f5ed17b8f..2434db7a0ca 100644 --- a/test/e2e/query_frontend_test.go +++ b/test/e2e/query_frontend_test.go @@ -1215,7 +1215,7 @@ func TestQueryFrontendProtection(t *testing.T) { q := e2ethanos.NewQuerierBuilder(e, "1").Init() testutil.Ok(t, e2e.StartAndWaitReady(q)) - t.Run("block action returns 400", func(t *testing.T) { + t.Run("block action returns 403", func(t *testing.T) { qfe := e2ethanos.NewQueryFrontend(e, "block", "http://"+q.InternalEndpoint("http"), queryfrontend.Config{}, queryfrontend.CacheProviderConfig{Type: queryfrontend.INMEMORY}, @@ -1237,7 +1237,7 @@ rules: require.NoError(t, err) t.Cleanup(func() { require.NoError(t, resp.Body.Close()) }) - require.Equal(t, http.StatusBadRequest, resp.StatusCode) + require.Equal(t, http.StatusForbidden, resp.StatusCode) }) t.Run("log action passes through", func(t *testing.T) {