From 49763983151e071a650ff9895b4cb3fed4537c73 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sat, 30 May 2026 23:30:24 -0500 Subject: [PATCH 01/14] Add OpenSpec proposal for personal rate notifications Create cut-down MVP spec for personal rate-based notifications informed by issue #177. Includes proposal, design, tasks, and spec with Given/When/Then scenarios covering: - Personal per-user rules (project and stack scoped) - Cache-backed 1-minute UTC bucket counters - Async evaluator job with distributed lock - Cooldown and snooze noise controls - Email delivery with full validation - Svelte UI for rule management Intentionally excludes digests, webhooks, Slack, anomaly detection, quiet hours, and other advanced alerting features. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../add-personal-rate-notifications/design.md | 393 ++++++++++++++++++ .../proposal.md | 117 ++++++ .../specs/rate-notifications/spec.md | 333 +++++++++++++++ .../add-personal-rate-notifications/tasks.md | 143 +++++++ 4 files changed, 986 insertions(+) create mode 100644 openspec/changes/add-personal-rate-notifications/design.md create mode 100644 openspec/changes/add-personal-rate-notifications/proposal.md create mode 100644 openspec/changes/add-personal-rate-notifications/specs/rate-notifications/spec.md create mode 100644 openspec/changes/add-personal-rate-notifications/tasks.md diff --git a/openspec/changes/add-personal-rate-notifications/design.md b/openspec/changes/add-personal-rate-notifications/design.md new file mode 100644 index 0000000000..71bd3d7031 --- /dev/null +++ b/openspec/changes/add-personal-rate-notifications/design.md @@ -0,0 +1,393 @@ +# Design: Add Personal Rate Notifications + +## Current Architecture Summary + +- `Project` stores `NotificationSettings` as a `Dictionary` keyed by user ID or integration key. +- Existing `NotificationSettings` are simple booleans: `ReportNewErrors`, `ReportCriticalErrors`, `ReportEventRegressions`, `ReportNewEvents`, `ReportCriticalEvents`, `SendDailySummary`. +- `QueueNotificationAction` (priority 70) enqueues `EventNotification` for existing per-event notifications based on project notification settings. +- `EventNotificationsJob` loads event/project/user and sends Slack/email with existing throttles. +- `Mailer` queues `MailMessage` and `MailMessageJob` sends it. +- `UsageService` already uses cache-backed 5-minute bucket counters for usage tracking via `ICacheClient`. +- Production insulation (`Exceptionless.Insulation`) replaces in-memory cache/queues with Redis/Azure/SQS. + +## Scope + +v1 is personal rate notifications only. No organization-level rules, no webhooks, no digests. + +## Data Model + +### RateNotificationRule + +```csharp +public class RateNotificationRule : IOwnedByOrganizationAndProjectWithIdentity +{ + public string Id { get; set; } + public string OrganizationId { get; set; } + public string ProjectId { get; set; } + public string UserId { get; set; } + public int Version { get; set; } + public string Name { get; set; } + public bool IsEnabled { get; set; } + public RateNotificationSignal Signal { get; set; } + public RateNotificationSubject Subject { get; set; } + public string? StackId { get; set; } + public int Threshold { get; set; } + public TimeSpan Window { get; set; } + public TimeSpan Cooldown { get; set; } + public DateTime? SnoozedUntilUtc { get; set; } + public DateTime CreatedUtc { get; set; } + public DateTime UpdatedUtc { get; set; } + public bool IsDeleted { get; set; } +} +``` + +### Enums + +```csharp +public enum RateNotificationSignal +{ + AllEvents, + Errors, + CriticalErrors, + NewErrors, + Regressions +} + +public enum RateNotificationSubject +{ + Project, + Stack +} +``` + +### Design decisions + +- Rules are stored separately from `Project.NotificationSettings`. +- Existing `NotificationSettings` should not be expanded because rate rules are richer and independently mutable. +- No tag/environment/time-of-day/external-recipient fields in v1. + +## Repository + +### IRateNotificationRuleRepository + +```csharp +public interface IRateNotificationRuleRepository : IRepositoryOwnedByOrganizationAndProject +{ + Task> GetByProjectIdAndUserIdAsync(string projectId, string userId, CommandOptionsDescriptor? options = null); + Task> GetEnabledByProjectIdAsync(string projectId, CommandOptionsDescriptor? options = null); + Task CountByProjectIdAndUserIdAsync(string projectId, string userId); +} +``` + +### RateNotificationRuleRepository + +Elasticsearch-backed repository following existing patterns (e.g., `StackRepository`, `WebHookRepository`). + +## API + +### Routes + +| Method | Route | Description | +|--------|-------|-------------| +| GET | `/api/v2/users/{userId}/projects/{projectId}/rate-notifications` | List user's rules for project | +| POST | `/api/v2/users/{userId}/projects/{projectId}/rate-notifications` | Create rule | +| GET | `/api/v2/users/{userId}/projects/{projectId}/rate-notifications/{ruleId}` | Get rule | +| PUT | `/api/v2/users/{userId}/projects/{projectId}/rate-notifications/{ruleId}` | Update rule | +| DELETE | `/api/v2/users/{userId}/projects/{projectId}/rate-notifications/{ruleId}` | Delete rule | +| POST | `/api/v2/users/{userId}/projects/{projectId}/rate-notifications/{ruleId}/snooze` | Snooze rule | +| POST | `/api/v2/users/{userId}/projects/{projectId}/rate-notifications/{ruleId}/unsnooze` | Unsnooze rule | + +### Authorization + +- Current user can manage their own rules. +- Global admin can manage any user's rules. +- User must have access to the project's organization. +- External recipients are not supported in v1. + +### Validation + +- `threshold > 0` +- `window` must be one of: 1m, 5m, 10m, 15m, 30m, 1h +- `cooldown` must be at least the window duration +- Recommended default cooldown: 30m +- Project subject must not specify `stack_id` +- Stack subject must specify `stack_id` +- `stack_id` must belong to the same project +- User cannot exceed 20 rules per project +- Rule name must be non-empty and ≤ 100 characters + +## Rule Index + +### RateNotificationRuleIndex service + +Purpose: + +- Load enabled rules for a project. +- Cache compiled counter definitions briefly. +- Ensure event pipeline can cheaply determine which counters to increment. +- Invalidate project rule index on create/update/delete/snooze/unsnooze. + +**Important:** The event pipeline must not increment every possible counter. It must increment only counters required by enabled rules for that project. + +Cache key: `rate:v1:rules:project:{projectId}` + +## Counter Architecture + +### RateCounterService + +Uses 1-minute UTC buckets. + +### Cache keys + +``` +rate:v1:count:{epochMinute}:{counterKey} +rate:v1:active:{epochMinute} +rate:v1:cooldown:{ruleId}:{subjectKey} +rate:v1:rules:project:{projectId} +``` + +### Counter key examples + +``` +project:{projectId}:signal:Errors +project:{projectId}:signal:CriticalErrors +project:{projectId}:signal:NewErrors +project:{projectId}:signal:Regressions +project:{projectId}:signal:AllEvents +project:{projectId}:stack:{stackId}:signal:Errors +project:{projectId}:stack:{stackId}:signal:CriticalErrors +project:{projectId}:stack:{stackId}:signal:NewErrors +project:{projectId}:stack:{stackId}:signal:Regressions +project:{projectId}:stack:{stackId}:signal:AllEvents +``` + +### TTL guidance + +- Counter bucket TTL: 3 hours +- Active bucket TTL: 3 hours +- Cooldown TTL: cooldown duration + 10 minutes + +### Signal matching + +| Signal | Matches when | +|--------|-------------| +| AllEvents | Any event | +| Errors | `ev.IsError()` | +| CriticalErrors | `ev.IsError() && ev.IsCritical()` | +| NewErrors | `ctx.IsNew && ev.IsError()` | +| Regressions | `ctx.IsRegression` | + +## Event Pipeline Action + +### UpdateRateCountersAction + +- Runs after stack assignment (priority > 70, e.g., 75) +- Loads `RateNotificationRuleIndex` for project +- Exits fast if no enabled rules +- Matches event against compiled counter definitions +- Increments matching counters via `RateCounterService` +- Adds counter key to active bucket list/set +- Never sends notifications directly +- Never queries Elasticsearch per event + +## Evaluator Job + +### RateNotificationEvaluatorJob + +- Runs periodically (recommended: every 60 seconds) +- Acquires distributed lock so only one evaluator runs per cluster +- Inspects recently active counters from active bucket sets +- Sums buckets for each rule's configured window +- Skips disabled rules +- Skips snoozed rules (where `SnoozedUntilUtc > now`) +- Compares observed count ≥ threshold +- Enforces cooldown per rule + subject +- Enqueues `RateNotification` work item on threshold crossing +- Sets cooldown key when enqueue succeeds +- Logs fired/skipped reasons with structured context + +This v1 does NOT need a full Normal/Pending/Firing/Recovering state machine. Simple threshold + cooldown + snooze is sufficient. + +## Queue Model + +### RateNotification + +```csharp +public class RateNotification +{ + public string RuleId { get; set; } + public int RuleVersion { get; set; } + public string OrganizationId { get; set; } + public string ProjectId { get; set; } + public string UserId { get; set; } + public string SubjectKey { get; set; } + public string? StackId { get; set; } + public DateTime WindowStartUtc { get; set; } + public DateTime WindowEndUtc { get; set; } + public long ObservedCount { get; set; } + public int Threshold { get; set; } +} +``` + +## Delivery Job + +### RateNotificationsJob + +- Loads rule by ID +- Loads project +- Loads user +- Validates: + - Rule still exists + - Rule is enabled + - Rule version matches or is compatible + - User belongs to organization + - User email is verified + - User email notifications are enabled + - Project/org still exists +- Sends email through `IMailer` +- Skips with structured logs when validation fails +- Does not send Slack/webhooks in v1 (marked as future work) + +## Email + +### IMailer.SendRateNotificationAsync + +```csharp +Task SendRateNotificationAsync(User user, Project project, RateNotificationRule rule, long observedCount, DateTime windowStart, DateTime windowEnd, Stack? stack); +``` + +Email includes: + +- Rule name +- Project name +- Observed count +- Threshold +- Window +- Subject type (project or stack) +- Stack title (when subject is stack and available) +- Link to project or stack +- Cooldown explanation +- No "everything is fine" messaging + +**Example subject:** `[ProjectName] Error rate exceeded` + +**Example body:** + +``` +Rule: Production error storm +Observed: 241 errors in 5 minutes +Threshold: 100 errors in 5 minutes +Cooldown: Further notifications for this rule are suppressed for 30 minutes. +``` + +## Frontend + +### Feature module + +`src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/` + +UI supports: + +- List rules for current user/project +- Create rule +- Edit rule +- Delete rule +- Enable/disable rule +- Snooze/unsnooze rule + +### Form fields + +- Name +- Signal (dropdown: All Events, Errors, Critical Errors, New Errors, Regressions) +- Subject (Project or Stack) +- Stack selector (shown when Subject = Stack) +- Threshold (number) +- Window (dropdown: 1m, 5m, 10m, 15m, 30m, 1h) +- Cooldown (duration, minimum = window) +- Enabled (toggle) + +### Not built in v1 + +- Rule history tab +- Delivery history +- Action builder +- Webhook builder +- Slack builder +- Digest UI +- Quiet hours UI +- Organization inheritance UI +- Preview charts + +### Noise warning + +Display when creating/editing: "This rule may be noisy. Use a cooldown to avoid repeated emails." + +## Metrics and Logging + +### Metrics + +- `rate_notification.rules.loaded` +- `rate_notification.counters.incremented` +- `rate_notification.evaluator.runs` +- `rate_notification.evaluator.rules_evaluated` +- `rate_notification.evaluator.notifications_enqueued` +- `rate_notification.delivery.sent` +- `rate_notification.delivery.skipped` + +### Structured log fields + +- `RuleId` +- `ProjectId` +- `UserId` +- `ObservedCount` +- `Threshold` +- `Reason` (skipped/fired) + +## Bootstrap / DI + +- Register `IRateNotificationRuleRepository` / `RateNotificationRuleRepository` +- Register `RateNotificationRuleIndex` +- Register `RateCounterService` +- Register `RateNotificationEvaluatorJob` +- Register `IQueue` notification queue +- Register `RateNotificationsJob` (delivery) +- Update `Exceptionless.Insulation` queue registration for Redis/Azure/SQS providers +- Add queue health check if existing patterns support it + +## Testing Strategy + +### Unit tests + +- Rule validation (threshold, window, cooldown, subject/stack consistency, name) +- Counter key builder +- Counter increments +- Bucket summing +- Signal matching +- Cooldown behavior +- Snooze behavior +- Evaluator threshold crossing logic + +### Integration tests + +- User can CRUD own rules +- User cannot manage another user's rules +- Global admin can manage another user's rules +- Stack rule rejects stack from another project +- Evaluator enqueues notification when threshold crossed +- Evaluator does not enqueue below threshold +- Evaluator respects cooldown +- Evaluator respects snooze +- Delivery skips disabled rule +- Delivery skips unverified email +- Delivery skips user not in org +- Delivery sends email for valid rule + +### Not tested in v1 + +- Action execution +- Webhooks +- Slack +- Digests +- No-data alerts +- Backtesting +- Rule history UI diff --git a/openspec/changes/add-personal-rate-notifications/proposal.md b/openspec/changes/add-personal-rate-notifications/proposal.md new file mode 100644 index 0000000000..583f87408f --- /dev/null +++ b/openspec/changes/add-personal-rate-notifications/proposal.md @@ -0,0 +1,117 @@ +# Proposal: Add Personal Rate Notifications + +## Summary + +Add personal, rate-based project and stack notifications. Users can configure rules like: + +- "Email me when this project has more than 100 errors in 5 minutes." +- "Email me when this stack occurs more than 20 times in 10 minutes." + +The implementation is intentionally small and focused: + +- Cheap cache-backed counters (1-minute UTC buckets) +- Asynchronous evaluator job +- Email delivery +- Cooldown and snooze to reduce noise + +This feature is informed by [issue #177](https://github.com/exceptionless/Exceptionless/issues/177), but intentionally does not implement the full notification wishlist. Issue #177's core goal — notifications should keep users informed without overwhelming them — is addressed through mandatory cooldowns, snooze support, and cache-only hot paths. + +## Classification + +- **Type:** Feature +- **Affected areas:** + - Backend/API + - Event pipeline + - Cache/Redis + - Queue jobs + - Email + - Svelte UI + - Tests +- **OpenSpec justification:** + - New API endpoints + - New persisted rule model + - New event-pipeline counter behavior + - New evaluator/delivery jobs + - Cross-cutting notification behavior + - User-facing notification settings + +## Goals + +- Notify users when project or stack activity crosses a configured threshold. +- Detect high-volume repeated errors that current new/error/regression notifications can miss. +- Reduce notification noise versus per-event email streams. +- Keep event ingestion cheap. +- Support horizontal scaling with distributed cache and queues. +- Validate user/project/org state before sending. +- Support snoozing a noisy rule. +- Provide enough logging, metrics, and tests to trust the system. + +## Non-goals + +- Anomaly detection or machine learning +- Percent-change alerts +- Arbitrary query/filter language +- Tag rules +- Environment rules +- Time-of-day rules +- Quiet hours / quiet days +- Daily/weekly/monthly summary changes +- Digest emails +- No-data alerts +- Recovery/resolved notifications +- Notification grouping/digesting +- Generic webhook actions +- Slack actions (marked as future work) +- PagerDuty/OpsGenie integrations +- External recipients who are not Exceptionless users +- Mutating automated actions +- Action execution engine +- Durable action execution +- Rule history UI +- Delivery history UI +- In-app notification center +- Billing/overage notification changes +- Email sender/from-address overhaul +- Queue/system health notification UI +- Reduce-noise in-app callouts + +## Deliberate Cutbacks + +Issue #177 is broad — it covers configurable rules based on type, tags, time of day, number of exceptions, environment, snoozing, periodic digest emails, reduced-noise behavior, in-app notices, and third-party integrations. + +This change intentionally implements only **personal rate notifications** — the minimum useful product: + +> "When this project or stack exceeds X matching events in Y minutes, email me, but not more than once per cooldown, and let me snooze the rule." + +The following are explicitly deferred: + +- Digests and periodic summaries +- Quiet hours and time-of-day logic +- Advanced conditional rules (tags, environment, arbitrary filters) +- External recipients and non-user notification targets +- Webhooks, Slack, PagerDuty, and other delivery channels +- Mutating or automated actions +- Organization-level rules and inheritance + +The first release should prove the cheap counter architecture and noise-control model before adding advanced alerting features. + +## Compatibility Risks + +| Risk | Mitigation | +|------|-----------| +| Existing project notification settings remain unchanged | Rate rules are stored separately; `Project.NotificationSettings` is not modified | +| Existing `EventNotificationsJob` behavior remains unchanged | Rate counters are a new pipeline action; existing notification queueing is unaffected | +| Existing `DailySummaryJob` behavior remains unchanged | No changes to daily summary logic | +| Existing Slack/webhook integrations are not changed | New delivery path is email-only; existing `WebHookNotification` queue untouched | +| Rate counters depend on distributed cache in production | In-memory cache/queues remain development-only; production requires Redis/Azure providers | +| Notification noise could increase if defaults are bad | Mandatory cooldowns, validation, and max rules per project (20) | +| Rules may become stale if user/project/org state changes | Delivery job re-validates rule, user, project, and org state before sending | + +## Rollback Plan + +1. Disable the rate notification evaluator job. +2. Disable the `UpdateRateCountersAction` in the event pipeline. +3. Existing event notifications and daily summaries continue to operate unchanged. +4. Delete or ignore persisted rate notification rules if needed. +5. Remove new UI route/feature module. +6. Remove new queues and cache keys if rolling back fully. diff --git a/openspec/changes/add-personal-rate-notifications/specs/rate-notifications/spec.md b/openspec/changes/add-personal-rate-notifications/specs/rate-notifications/spec.md new file mode 100644 index 0000000000..09ff924d02 --- /dev/null +++ b/openspec/changes/add-personal-rate-notifications/specs/rate-notifications/spec.md @@ -0,0 +1,333 @@ +# Spec: Rate Notifications + +## ADDED Requirements + +### Requirement: User can list their project rate notification rules + +The API MUST allow authenticated users to retrieve their own rate notification rules for a specific project. + +#### Scenario: Authenticated user lists own rules + +Given authenticated user with access to project +When GET `/api/v2/users/{userId}/projects/{projectId}/rate-notifications` +Then return only that user's rules for that project. + +### Requirement: User can create a project rate notification rule + +The API MUST allow authenticated users to create a rate notification rule scoped to a project with a threshold, window, cooldown, signal, and subject. + +#### Scenario: Valid project rule is persisted + +Given authenticated user with access to project +When POST valid rule with threshold, window, cooldown, signal, subject=Project +Then rule is persisted with organization_id, project_id, user_id, threshold, window, cooldown, signal, subject. + +### Requirement: User can create a stack rate notification rule + +The API MUST allow authenticated users to create a rate notification rule scoped to a specific stack within a project. + +#### Scenario: Valid stack rule is persisted + +Given authenticated user with access to project +And stack belongs to project +When POST subject=Stack and stack_id +Then rule is persisted. + +### Requirement: Invalid stack scope is rejected + +The API MUST reject a rule targeting a stack that does not belong to the specified project. + +#### Scenario: Stack from another project is rejected + +Given stack does not belong to project +When user creates stack rule +Then response is 400 or 404. + +### Requirement: User cannot manage another user's rules + +The API MUST deny non-admin users access to rate notification rules belonging to other users. + +#### Scenario: Non-admin user is denied access to other user's rules + +Given non-admin user A +When they request user B's rate notification rules +Then response is 404 or 403. + +### Requirement: Global admin can manage another user's rules + +The API MUST allow global administrators to create, read, update, and delete rate notification rules on behalf of any user. + +#### Scenario: Admin accesses another user's rules + +Given global admin +When they manage another user's rate notification rules +Then request succeeds. + +### Requirement: Rule validation prevents noisy/invalid rules + +The API MUST enforce validation constraints to prevent misconfigured or excessively noisy rules. + +#### Scenario: Threshold must be positive + +Given threshold <= 0 +When user creates rule +Then response is 400 with validation error. + +#### Scenario: Unsupported window is rejected + +Given window is not one of 1m, 5m, 10m, 15m, 30m, 1h +When user creates rule +Then response is 400 with validation error. + +#### Scenario: Cooldown shorter than window is rejected + +Given cooldown < window +When user creates rule +Then response is 400 with validation error. + +#### Scenario: Project subject with stack_id is rejected + +Given subject = Project and stack_id is set +When user creates rule +Then response is 400 with validation error. + +#### Scenario: Stack subject without stack_id is rejected + +Given subject = Stack and stack_id is null +When user creates rule +Then response is 400 with validation error. + +#### Scenario: Empty name is rejected + +Given name is empty +When user creates rule +Then response is 400 with validation error. + +#### Scenario: Name exceeding 100 characters is rejected + +Given name length > 100 +When user creates rule +Then response is 400 with validation error. + +#### Scenario: Exceeding max rules per project is rejected + +Given user already has 20 rules for project +When user creates another rule +Then response is 400 with validation error. + +### Requirement: Event pipeline increments matching configured counters + +The event pipeline MUST increment the corresponding rate counter in the current UTC minute bucket when an event matches an enabled rule's signal. + +#### Scenario: Matching event increments counter + +Given enabled project rule exists +When matching event is processed +Then matching rate counter for current UTC minute bucket is incremented. + +### Requirement: Event pipeline skips projects without enabled rules + +The event pipeline MUST NOT perform any counter operations for projects with no enabled rate notification rules. + +#### Scenario: No rules means no counter work + +Given no enabled rate notification rules for project +When event is processed +Then no rate counter is incremented. + +### Requirement: Event pipeline increments only required counters + +The pipeline MUST only increment counters that are required by at least one enabled rule, not all possible signal counters. + +#### Scenario: Only configured signal counters are incremented + +Given project has only an Errors rule +When critical error event is processed +Then Errors counter is incremented +And unrelated counters are not incremented unless required by configured rules. + +### Requirement: Stack counters are stack-specific + +Stack-scoped counters MUST only be incremented by events on the specific stack referenced by the rule. + +#### Scenario: Event on different stack does not increment + +Given stack rule exists for stack A +When event occurs on stack B +Then stack A counter is not incremented. + +### Requirement: Evaluator enqueues notification when threshold is crossed + +The evaluator job MUST enqueue a RateNotification work item when the sum of counter buckets for a rule's window meets or exceeds the threshold. + +#### Scenario: Threshold reached triggers enqueue + +Given rule threshold is 100 errors in 5 minutes +And matching counters sum to 100 or more +When evaluator runs +Then RateNotification work item is enqueued. + +### Requirement: Evaluator does not enqueue below threshold + +The evaluator MUST NOT enqueue a notification when the observed event count is below the rule threshold. + +#### Scenario: Below threshold is silent + +Given rule threshold is 100 +And observed count is 99 +When evaluator runs +Then no work item is enqueued. + +### Requirement: Cooldown suppresses repeated sends + +The evaluator MUST NOT enqueue a new notification for a rule until the cooldown period from the previous firing has expired. + +#### Scenario: Active cooldown prevents re-enqueue + +Given rule has fired +And cooldown has not expired +When threshold is crossed again +Then no new work item is enqueued. + +### Requirement: Snooze suppresses sends + +The evaluator MUST NOT fire a snoozed rule until the snooze period expires. + +#### Scenario: Snoozed rule does not fire + +Given rule snoozed_until_utc is in the future +When threshold is crossed +Then no work item is enqueued. + +### Requirement: Disabled rules do not fire + +The evaluator MUST skip disabled rules during evaluation. + +#### Scenario: Disabled rule is skipped + +Given rule is disabled +When threshold is crossed +Then no work item is enqueued. + +### Requirement: No healthy/no-activity emails + +The system MUST NOT send notifications indicating that everything is fine or that no activity occurred. + +#### Scenario: Below threshold produces no notification + +Given threshold is not crossed +When evaluator runs +Then no healthy/no-activity notification is sent. + +### Requirement: Delivery sends email for valid notification + +The delivery job MUST send a rate notification email when the rule, project, user, and email state are all valid. + +#### Scenario: Valid state delivers email + +Given RateNotification work item +And rule/project/user are valid +And user email is verified +And user email notifications are enabled +When delivery job processes item +Then rate notification email is queued. + +### Requirement: Delivery skips invalid user state + +The delivery job MUST skip sending and log structured reasons when any precondition is not met. + +#### Scenario: Unverified email is skipped + +Given user email is unverified +When delivery job processes item +Then notification is skipped with log. + +#### Scenario: Notifications disabled is skipped + +Given user email notifications are disabled +When delivery job processes item +Then notification is skipped with log. + +#### Scenario: User not in organization is skipped + +Given user no longer belongs to organization +When delivery job processes item +Then notification is skipped with log. + +#### Scenario: Deleted project is skipped + +Given project is deleted +When delivery job processes item +Then notification is skipped with log. + +#### Scenario: Deleted rule is skipped + +Given rule is deleted +When delivery job processes item +Then notification is skipped with log. + +#### Scenario: Disabled rule is skipped during delivery + +Given rule is disabled +When delivery job processes item +Then notification is skipped with log. + +#### Scenario: Stale rule version is skipped + +Given rule version does not match work item +When delivery job processes item +Then notification is skipped with log. + +### Requirement: Email includes actionable context + +Rate notification emails MUST include all information the user needs to understand and act on the alert. + +#### Scenario: Email body contains all required fields + +Given notification email is queued +Then email includes rule name, project name, observed count, threshold, window, subject type, stack title (when applicable), link to project or stack, and cooldown explanation. + +### Requirement: User can manage project rate rules in Svelte UI + +The Svelte UI MUST provide a full CRUD interface for rate notification rules within project settings. + +#### Scenario: Full CRUD and controls available + +Given user opens project notification settings +Then they can list, create, edit, delete, enable/disable, snooze, and unsnooze rate rules. + +### Requirement: UI avoids advanced/deferred features + +The v1 UI MUST NOT expose features that are deferred to future iterations. + +#### Scenario: Deferred features are not exposed + +Given user opens rate notification management UI +Then UI does not expose digests, webhooks, automated actions, quiet hours, arbitrary filters, external recipients, or no-data alerts. + +### Requirement: Rate notification hot path is cache-only + +The event pipeline counter path MUST use only cache operations and MUST NOT query Elasticsearch per event. + +#### Scenario: No Elasticsearch per event + +Given event is processed +Then rate notification countering does not query Elasticsearch per event. + +### Requirement: Production requires distributed cache/queue + +Production deployments MUST use provider-backed (Redis/Azure/SQS) cache and queues for rate notification infrastructure. + +#### Scenario: Production uses provider-backed infrastructure + +Given production deployment +Then rate notification counters and queues use provider-backed cache/queues, not in-memory providers. + +### Requirement: Rollback does not affect existing notifications + +Disabling rate notification components MUST NOT impact existing event notification or daily summary behavior. + +#### Scenario: Disabling rate notifications leaves existing behavior intact + +Given rate notification evaluator/action is disabled +Then existing event notifications and daily summaries continue to operate unchanged. diff --git a/openspec/changes/add-personal-rate-notifications/tasks.md b/openspec/changes/add-personal-rate-notifications/tasks.md new file mode 100644 index 0000000000..d4cde871fe --- /dev/null +++ b/openspec/changes/add-personal-rate-notifications/tasks.md @@ -0,0 +1,143 @@ +# Tasks: Add Personal Rate Notifications + +## Backend - Models and Repositories + +- [ ] **Add RateNotificationRule model and enums** + - Create `RateNotificationRule` model with all fields (Id, OrganizationId, ProjectId, UserId, Version, Name, IsEnabled, Signal, Subject, StackId, Threshold, Window, Cooldown, SnoozedUntilUtc, CreatedUtc, UpdatedUtc, IsDeleted) + - Create `RateNotificationSignal` enum (AllEvents, Errors, CriticalErrors, NewErrors, Regressions) + - Create `RateNotificationSubject` enum (Project, Stack) + +- [ ] **Add IRateNotificationRuleRepository and implementation** + - Create interface extending `IRepositoryOwnedByOrganizationAndProject` + - Add methods: `GetByProjectIdAndUserIdAsync`, `GetEnabledByProjectIdAsync`, `CountByProjectIdAndUserIdAsync` + - Create Elasticsearch-backed implementation following existing repository patterns + +- [ ] **Register repository in DI** + - Register `IRateNotificationRuleRepository` / `RateNotificationRuleRepository` in `Bootstrapper.cs` + +## Backend - Countering and Evaluation + +- [ ] **Add RateNotificationRuleIndex service** + - Load enabled rules for a project from repository + - Cache compiled counter definitions with short TTL + - Invalidate on rule create/update/delete/snooze/unsnooze + - Cache key: `rate:v1:rules:project:{projectId}` + +- [ ] **Add RateCounterService** + - Implement 1-minute UTC bucket counters via `ICacheClient` + - Methods: increment counter, sum buckets for window, check/set cooldown + - Counter key format: `rate:v1:count:{epochMinute}:{counterKey}` + - Active bucket tracking: `rate:v1:active:{epochMinute}` + - Cooldown key format: `rate:v1:cooldown:{ruleId}:{subjectKey}` + - TTLs: counter/active = 3h, cooldown = cooldown + 10m + +- [ ] **Add UpdateRateCountersAction (event pipeline)** + - Priority after stack assignment (e.g., 75) + - Load rule index for project; exit fast if no enabled rules + - Match event against compiled counter definitions (signal matching) + - Increment matching counters; add to active bucket set + - Never query Elasticsearch per event; never send notifications directly + +- [ ] **Add RateNotificationEvaluatorJob** + - Periodic job (60s interval) with distributed lock + - Inspect recently active counters + - Sum buckets for each rule's window + - Skip disabled/snoozed rules + - Compare observed ≥ threshold; enforce cooldown per rule+subject + - Enqueue `RateNotification` work item on threshold crossing + - Set cooldown key on successful enqueue + - Structured logging for fired/skipped reasons + +- [ ] **Add RateNotification queue model** + - Fields: RuleId, RuleVersion, OrganizationId, ProjectId, UserId, SubjectKey, StackId, WindowStartUtc, WindowEndUtc, ObservedCount, Threshold + +- [ ] **Register counter/evaluator services in DI** + - Register `RateNotificationRuleIndex`, `RateCounterService`, `RateNotificationEvaluatorJob` + - Register `IQueue` in Core and Insulation bootstrappers + +## Backend - Delivery + +- [ ] **Add RateNotificationsJob (delivery)** + - Load rule, project, user + - Validate: rule exists, enabled, version compatible, user in org, email verified, notifications enabled, project/org exists + - Send email via `IMailer.SendRateNotificationAsync` + - Skip with structured logs on validation failure + +- [ ] **Add IMailer.SendRateNotificationAsync** + - Add method to `IMailer` interface and `Mailer` implementation + - Email includes: rule name, project name, observed count, threshold, window, subject type, stack title (when applicable), link, cooldown explanation + - Subject format: `[ProjectName] Error rate exceeded` + - No "everything is fine" messaging + +- [ ] **Update Insulation queue registration** + - Register `IQueue` for Redis/Azure/SQS providers in `Exceptionless.Insulation/Bootstrapper.cs` + +## Backend - API + +- [ ] **Add RateNotificationRuleController** + - Routes: GET list, POST create, GET by id, PUT update, DELETE, POST snooze, POST unsnooze + - Authorization: current user manages own rules; global admin manages any user's rules; user must access project org + - Validation: threshold > 0, supported windows, cooldown ≥ window, subject/stack consistency, max 20 rules per user per project, name non-empty and ≤ 100 chars + +- [ ] **Add request/response DTOs** + - Create/update request models with validation attributes + - Snooze request model (duration or until timestamp) + +- [ ] **Update tests/http files** + - Add HTTP sample requests for all rate notification endpoints + +## Frontend + +- [ ] **Add rate-notifications feature module** + - Create `src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/` + - Models/types matching API DTOs + - TanStack Query API wrappers (list, create, update, delete, snooze, unsnooze) + +- [ ] **Add rate notification rule list component** + - List rules for current user/project + - Show enable/disable toggle, snooze status + - Delete confirmation + +- [ ] **Add rate notification rule form component** + - Fields: Name, Signal, Subject, Stack selector, Threshold, Window, Cooldown, Enabled + - Validation matching backend constraints + - Noise warning copy: "This rule may be noisy. Use a cooldown to avoid repeated emails." + +- [ ] **Integrate into project settings** + - Add rate notifications section/tab in project notification settings + - Route and navigation entry + +## Tests + +- [ ] **Add unit tests** + - Rule validation (threshold, window, cooldown, subject/stack, name) + - Counter key builder + - Counter increments and bucket summing + - Signal matching logic + - Cooldown behavior + - Snooze behavior + - Evaluator threshold crossing + +- [ ] **Add integration tests** + - User CRUD own rules (list, create, get, update, delete) + - User cannot manage another user's rules + - Global admin can manage another user's rules + - Stack rule rejects stack from another project + - Evaluator enqueues when threshold crossed + - Evaluator does not enqueue below threshold + - Evaluator respects cooldown + - Evaluator respects snooze + - Delivery skips disabled rule + - Delivery skips unverified email + - Delivery skips user not in org + - Delivery sends email for valid rule + +## Validation + +- [ ] **Run OpenSpec validation** + - `openspec validate add-personal-rate-notifications --strict` + +- [ ] **Run relevant builds and tests** + - `dotnet build` + - `dotnet test` + - Frontend build and lint From f821e7f59a453275d5a7d5661e111a3a7bf07208 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sun, 31 May 2026 07:04:58 -0500 Subject: [PATCH 02/14] fix: address five spec review findings in personal rate notifications Bugs identified via staff-engineer review comparing against existing notification pipeline (070_QueueNotificationAction, EventNotificationsJob, Stack.AllowNotifications): 1. Missing premium feature gate: The spec introduced a new free notification channel but the existing system requires HasPremiumFeatures. Added to proposal risks table, design gating section, evaluator, and Svelte UI. 2. Muted-stack / discarded-event suppression: Counters would have counted events on Ignored/Snoozed/Discarded stacks and known-bot requests. Added AllowNotifications and IsBot checks to UpdateRateCountersAction, mirroring existing 070_QueueNotificationAction logic. 3. Snooze back-alert bug (shared-counter race): Rate counters are keyed by project/stack + signal, not per-rule. After snooze expires, activity gathered during the muted window would immediately fire because the counter reflects global traffic. Fix: evaluator uses max(windowStartUtc, rule.SnoozedUntilUtc) as the lower bucket boundary so only post-snooze traffic counts toward the threshold. 4. Orphaned rule lifecycle: No cleanup on membership removal or project/org deletion. Added CleanupRateNotificationRulesAsync pattern mirroring OrganizationService.CleanupProjectNotificationSettingsAsync. 5. Stack-scoped email missing context: Delivery job never loaded the stack, so stack-scoped email copy had no title/link. Added stack lookup requirement to delivery design and spec. Also expanded test matrix to cover all five findings. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../add-personal-rate-notifications/design.md | 38 ++++++++- .../proposal.md | 9 +++ .../specs/rate-notifications/spec.md | 77 +++++++++++++++++++ .../add-personal-rate-notifications/tasks.md | 22 +++++- 4 files changed, 144 insertions(+), 2 deletions(-) diff --git a/openspec/changes/add-personal-rate-notifications/design.md b/openspec/changes/add-personal-rate-notifications/design.md index 71bd3d7031..416337c9dc 100644 --- a/openspec/changes/add-personal-rate-notifications/design.md +++ b/openspec/changes/add-personal-rate-notifications/design.md @@ -12,7 +12,7 @@ ## Scope -v1 is personal rate notifications only. No organization-level rules, no webhooks, no digests. +v1 is personal rate notifications only. No organization-level rules, no webhooks, no digests. It also follows the existing premium-only occurrence-notification model instead of introducing a new free notification channel. ## Data Model @@ -65,6 +65,7 @@ public enum RateNotificationSubject - Rules are stored separately from `Project.NotificationSettings`. - Existing `NotificationSettings` should not be expanded because rate rules are richer and independently mutable. - No tag/environment/time-of-day/external-recipient fields in v1. +- `SnoozedUntilUtc` is also the rule's resume boundary. Manual unsnooze sets it to the current UTC time instead of clearing it so the evaluator can ignore activity gathered during the muted period. ## Repository @@ -104,6 +105,11 @@ Elasticsearch-backed repository following existing patterns (e.g., `StackReposit - User must have access to the project's organization. - External recipients are not supported in v1. +### Premium gating + +- Rate notifications follow the current occurrence-notification premium model. +- Rules may remain persisted across plan downgrade, but countering, evaluation, delivery, and Svelte create/enable controls MUST treat the feature as unavailable until premium is restored. + ### Validation - `threshold > 0` @@ -182,8 +188,12 @@ project:{projectId}:stack:{stackId}:signal:AllEvents ### UpdateRateCountersAction - Runs after stack assignment (priority > 70, e.g., 75) +- Exits fast when the organization does not have premium features - Loads `RateNotificationRuleIndex` for project - Exits fast if no enabled rules +- Skips events on stacks where `!ctx.Stack.AllowNotifications` +- Skips canceled/discarded events that would not produce occurrence notifications +- Skips requests already marked as bots by request-info enrichment - Matches event against compiled counter definitions - Increments matching counters via `RateCounterService` - Adds counter key to active bucket list/set @@ -197,7 +207,9 @@ project:{projectId}:stack:{stackId}:signal:AllEvents - Runs periodically (recommended: every 60 seconds) - Acquires distributed lock so only one evaluator runs per cluster - Inspects recently active counters from active bucket sets +- Skips organizations without premium features - Sums buckets for each rule's configured window +- Uses `max(windowStartUtc, rule.SnoozedUntilUtc)` as the lower bound when a snooze boundary falls inside the evaluation window so a rule resumes from a fresh baseline - Skips disabled rules - Skips snoozed rules (where `SnoozedUntilUtc > now`) - Compares observed count ≥ threshold @@ -208,6 +220,12 @@ project:{projectId}:stack:{stackId}:signal:AllEvents This v1 does NOT need a full Normal/Pending/Firing/Recovering state machine. Simple threshold + cooldown + snooze is sufficient. +### Snooze semantics + +- Snooze suppresses delivery immediately. +- When a snooze expires or a user manually unsnoozes a rule, the rule resumes from a fresh baseline. +- Activity observed entirely during the snooze window MUST NOT trigger an immediate post-snooze alert, even when another enabled rule kept the shared counter hot. + ## Queue Model ### RateNotification @@ -236,6 +254,7 @@ public class RateNotification - Loads rule by ID - Loads project - Loads user +- Loads stack for stack-scoped rules so email copy can include stack title and deep link - Validates: - Rule still exists - Rule is enabled @@ -248,6 +267,13 @@ public class RateNotification - Skips with structured logs when validation fails - Does not send Slack/webhooks in v1 (marked as future work) +## Lifecycle cleanup + +- Remove rate notification rules when a user loses organization access. +- Remove rate notification rules when a project or organization is deleted. +- Invalidate cached rule indexes when cleanup runs so orphaned rules stop consuming evaluator work immediately. +- Reuse the same work-item cleanup pattern already used for `Project.NotificationSettings` updates where practical. + ## Email ### IMailer.SendRateNotificationAsync @@ -294,6 +320,7 @@ UI supports: - Delete rule - Enable/disable rule - Snooze/unsnooze rule +- Disabled/upgrade state when the organization lacks premium features ### Form fields @@ -363,8 +390,11 @@ Display when creating/editing: "This rule may be noisy. Use a cooldown to avoid - Counter increments - Bucket summing - Signal matching +- Premium gating +- `AllowNotifications` / bot suppression checks - Cooldown behavior - Snooze behavior +- Fresh-baseline behavior after snooze expiry or manual unsnooze - Evaluator threshold crossing logic ### Integration tests @@ -373,14 +403,20 @@ Display when creating/editing: "This rule may be noisy. Use a cooldown to avoid - User cannot manage another user's rules - Global admin can manage another user's rules - Stack rule rejects stack from another project +- Non-premium organizations do not counter, evaluate, or deliver rate notifications +- Events on stacks with `AllowNotifications = false` do not increment rate counters +- Bot-marked requests do not increment rate counters - Evaluator enqueues notification when threshold crossed - Evaluator does not enqueue below threshold - Evaluator respects cooldown - Evaluator respects snooze +- Activity gathered during snooze does not fire immediately when the rule resumes - Delivery skips disabled rule - Delivery skips unverified email - Delivery skips user not in org - Delivery sends email for valid rule +- Delivery loads stack context for stack-scoped emails +- Membership/project/org cleanup removes orphaned rules ### Not tested in v1 diff --git a/openspec/changes/add-personal-rate-notifications/proposal.md b/openspec/changes/add-personal-rate-notifications/proposal.md index 583f87408f..6f215db58f 100644 --- a/openspec/changes/add-personal-rate-notifications/proposal.md +++ b/openspec/changes/add-personal-rate-notifications/proposal.md @@ -13,6 +13,8 @@ The implementation is intentionally small and focused: - Asynchronous evaluator job - Email delivery - Cooldown and snooze to reduce noise +- Premium-gated runtime behavior that matches existing occurrence notifications +- Existing notification suppression semantics so muted traffic does not reappear as rate noise This feature is informed by [issue #177](https://github.com/exceptionless/Exceptionless/issues/177), but intentionally does not implement the full notification wishlist. Issue #177's core goal — notifications should keep users informed without overwhelming them — is addressed through mandatory cooldowns, snooze support, and cache-only hot paths. @@ -42,8 +44,11 @@ This feature is informed by [issue #177](https://github.com/exceptionless/Except - Reduce notification noise versus per-event email streams. - Keep event ingestion cheap. - Support horizontal scaling with distributed cache and queues. +- Preserve current premium-only occurrence-notification behavior. +- Honor existing notification suppression so ignored, snoozed, discarded, and fixed stacks — and bot traffic already excluded from occurrence emails — do not generate rate alerts. - Validate user/project/org state before sending. - Support snoozing a noisy rule. +- Resume from a fresh baseline after snooze instead of back-alerting on traffic that happened while the rule was muted. - Provide enough logging, metrics, and tests to trust the system. ## Non-goals @@ -103,9 +108,13 @@ The first release should prove the cheap counter architecture and noise-control | Existing `EventNotificationsJob` behavior remains unchanged | Rate counters are a new pipeline action; existing notification queueing is unaffected | | Existing `DailySummaryJob` behavior remains unchanged | No changes to daily summary logic | | Existing Slack/webhook integrations are not changed | New delivery path is email-only; existing `WebHookNotification` queue untouched | +| Existing premium-only occurrence notification behavior could drift | Countering, evaluation, delivery, and Svelte enablement follow the same premium gating model as existing occurrence notifications | | Rate counters depend on distributed cache in production | In-memory cache/queues remain development-only; production requires Redis/Azure providers | +| Muted stacks or bot traffic could reappear as new rate noise | Countering honors `Stack.AllowNotifications`, canceled/discarded contexts, and request-info bot markers before incrementing counters | +| Snooze could defer a notification instead of suppressing it | Evaluation resumes from a fresh baseline using the snooze boundary so activity gathered during snooze does not fire immediately on resume | | Notification noise could increase if defaults are bad | Mandatory cooldowns, validation, and max rules per project (20) | | Rules may become stale if user/project/org state changes | Delivery job re-validates rule, user, project, and org state before sending | +| Orphaned rules could be indexed and evaluated forever | Add cleanup on user membership changes and project/org deletion, plus cache invalidation for removed rules | ## Rollback Plan diff --git a/openspec/changes/add-personal-rate-notifications/specs/rate-notifications/spec.md b/openspec/changes/add-personal-rate-notifications/specs/rate-notifications/spec.md index 09ff924d02..5c71129d33 100644 --- a/openspec/changes/add-personal-rate-notifications/specs/rate-notifications/spec.md +++ b/openspec/changes/add-personal-rate-notifications/specs/rate-notifications/spec.md @@ -135,6 +135,18 @@ Given no enabled rate notification rules for project When event is processed Then no rate counter is incremented. +### Requirement: Rate notifications honor premium feature gating + +Rate notifications MUST follow the existing premium-only occurrence-notification model and MUST NOT become a free notification channel. + +#### Scenario: Non-premium organizations do not activate rate notifications + +Given project organization does not have premium features +When matching events are processed or the evaluator runs +Then no rate counters are incremented +And no rate notification work item is enqueued +And no rate notification email is sent. + ### Requirement: Event pipeline increments only required counters The pipeline MUST only increment counters that are required by at least one enabled rule, not all possible signal counters. @@ -156,6 +168,22 @@ Given stack rule exists for stack A When event occurs on stack B Then stack A counter is not incremented. +### Requirement: Rate notifications honor existing occurrence-notification suppression + +Rate notifications MUST respect existing occurrence-notification suppression semantics so muted traffic does not reappear as rate noise. + +#### Scenario: Events on muted stacks do not increment counters + +Given the event stack has `AllowNotifications = false` +When the event is processed +Then no rate counter is incremented for that event. + +#### Scenario: Bot-marked requests do not increment counters + +Given request enrichment has marked the event request as a bot +When the event is processed +Then no rate counter is incremented for that event. + ### Requirement: Evaluator enqueues notification when threshold is crossed The evaluator job MUST enqueue a RateNotification work item when the sum of counter buckets for a rule's window meets or exceeds the threshold. @@ -199,6 +227,24 @@ Given rule snoozed_until_utc is in the future When threshold is crossed Then no work item is enqueued. +### Requirement: Snooze resumes from a fresh baseline + +When a snoozed rule resumes, the evaluator MUST ignore activity observed entirely during the snooze window so the rule does not back-alert immediately after unsnooze or natural expiry. + +#### Scenario: Unsnoozing does not immediately fire on snoozed activity + +Given a rule was snoozed while matching events continued +And the shared subject counter remained active for another enabled rule +When the user unsnoozes the rule +Then the evaluator does not enqueue a rate notification until new post-unsnooze activity crosses the threshold. + +#### Scenario: Snooze expiry does not immediately fire on snoozed activity + +Given a rule remained snoozed until its snooze window expired +And matching activity during the snooze window crossed the threshold +When the evaluator next runs after the snooze expires +Then the evaluator does not enqueue a rate notification until new post-expiry activity crosses the threshold. + ### Requirement: Disabled rules do not fire The evaluator MUST skip disabled rules during evaluation. @@ -287,6 +333,12 @@ Rate notification emails MUST include all information the user needs to understa Given notification email is queued Then email includes rule name, project name, observed count, threshold, window, subject type, stack title (when applicable), link to project or stack, and cooldown explanation. +#### Scenario: Stack-scoped email includes stack context + +Given a stack-scoped rate notification email is queued +When the delivery job loads the stack context +Then the email includes the stack title and a deep link to the stack. + ### Requirement: User can manage project rate rules in Svelte UI The Svelte UI MUST provide a full CRUD interface for rate notification rules within project settings. @@ -296,6 +348,13 @@ The Svelte UI MUST provide a full CRUD interface for rate notification rules wit Given user opens project notification settings Then they can list, create, edit, delete, enable/disable, snooze, and unsnooze rate rules. +#### Scenario: Non-premium organizations show rate notifications as unavailable + +Given the organization does not have premium features +When the user opens project notification settings +Then the UI shows the feature as unavailable +And the user cannot create or enable active rate notification rules. + ### Requirement: UI avoids advanced/deferred features The v1 UI MUST NOT expose features that are deferred to future iterations. @@ -331,3 +390,21 @@ Disabling rate notification components MUST NOT impact existing event notificati Given rate notification evaluator/action is disabled Then existing event notifications and daily summaries continue to operate unchanged. + +### Requirement: Rule lifecycle cleanup removes orphaned rules + +The system MUST remove or invalidate orphaned rate notification rules when the owning membership, project, or organization is deleted. + +#### Scenario: Membership removal cleans up user rules + +Given a user is removed from the organization +When cleanup runs +Then that user's rate notification rules for the organization are removed or invalidated +And cached rule indexes are invalidated. + +#### Scenario: Project deletion cleans up project rules + +Given a project is deleted +When cleanup runs +Then rate notification rules for that project are removed or invalidated +And cached rule indexes are invalidated. diff --git a/openspec/changes/add-personal-rate-notifications/tasks.md b/openspec/changes/add-personal-rate-notifications/tasks.md index d4cde871fe..2744258d0a 100644 --- a/openspec/changes/add-personal-rate-notifications/tasks.md +++ b/openspec/changes/add-personal-rate-notifications/tasks.md @@ -33,15 +33,19 @@ - [ ] **Add UpdateRateCountersAction (event pipeline)** - Priority after stack assignment (e.g., 75) + - Exit fast if organization lacks premium features - Load rule index for project; exit fast if no enabled rules + - Skip events on stacks where `AllowNotifications` is false + - Skip canceled/discarded events and requests already marked as bots - Match event against compiled counter definitions (signal matching) - Increment matching counters; add to active bucket set - Never query Elasticsearch per event; never send notifications directly - [ ] **Add RateNotificationEvaluatorJob** - Periodic job (60s interval) with distributed lock + - Skip organizations without premium features - Inspect recently active counters - - Sum buckets for each rule's window + - Sum buckets for each rule's window using a fresh baseline after snooze/unsnooze - Skip disabled/snoozed rules - Compare observed ≥ threshold; enforce cooldown per rule+subject - Enqueue `RateNotification` work item on threshold crossing @@ -59,6 +63,7 @@ - [ ] **Add RateNotificationsJob (delivery)** - Load rule, project, user + - Load stack for stack-scoped rules - Validate: rule exists, enabled, version compatible, user in org, email verified, notifications enabled, project/org exists - Send email via `IMailer.SendRateNotificationAsync` - Skip with structured logs on validation failure @@ -78,6 +83,7 @@ - Routes: GET list, POST create, GET by id, PUT update, DELETE, POST snooze, POST unsnooze - Authorization: current user manages own rules; global admin manages any user's rules; user must access project org - Validation: threshold > 0, supported windows, cooldown ≥ window, subject/stack consistency, max 20 rules per user per project, name non-empty and ≤ 100 chars + - Preserve persisted rules across plan downgrade, but do not allow the runtime or Svelte UI to treat non-premium orgs as active rate-notification senders - [ ] **Add request/response DTOs** - Create/update request models with validation attributes @@ -96,6 +102,7 @@ - [ ] **Add rate notification rule list component** - List rules for current user/project - Show enable/disable toggle, snooze status + - Show premium-gated disabled state / upgrade messaging when org lacks premium features - Delete confirmation - [ ] **Add rate notification rule form component** @@ -114,8 +121,11 @@ - Counter key builder - Counter increments and bucket summing - Signal matching logic + - Premium gating + - `AllowNotifications` / bot suppression - Cooldown behavior - Snooze behavior + - Fresh-baseline behavior after snooze/unsnooze - Evaluator threshold crossing - [ ] **Add integration tests** @@ -123,14 +133,24 @@ - User cannot manage another user's rules - Global admin can manage another user's rules - Stack rule rejects stack from another project + - Non-premium organizations do not counter, evaluate, or deliver rate notifications + - Events on stacks with `AllowNotifications = false` do not increment rate counters + - Bot-marked requests do not increment rate counters - Evaluator enqueues when threshold crossed - Evaluator does not enqueue below threshold - Evaluator respects cooldown - Evaluator respects snooze + - Activity gathered during snooze does not fire immediately when the rule resumes - Delivery skips disabled rule - Delivery skips unverified email - Delivery skips user not in org - Delivery sends email for valid rule + - Delivery loads stack context for stack-scoped emails + - Membership/project/org cleanup removes orphaned rules + +- [ ] **Add lifecycle cleanup** + - Remove/invalidate rules when user membership changes + - Remove/invalidate rules when project or organization is deleted ## Validation From c8773f8c872da474b6bdc0ee2588efaba61dcd17 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sun, 31 May 2026 07:41:52 -0500 Subject: [PATCH 03/14] feat: add RateNotificationRule model, repository, and ES index Introduces the core domain model for personal rate notification rules. Each rule belongs to an org/project/user and configures a threshold- based alert for a specific signal (errors, critical errors, etc.) with snooze/cooldown semantics to prevent alert fatigue. --- .../Models/Enums/RateNotificationSignal.cs | 10 +++ .../Models/Enums/RateNotificationSubject.cs | 7 +++ .../Models/Queues/RateNotification.cs | 16 +++++ .../Models/RateNotificationRule.cs | 55 ++++++++++++++++ .../ExceptionlessElasticConfiguration.cs | 2 + .../Indexes/RateNotificationRuleIndex.cs | 43 +++++++++++++ .../IRateNotificationRuleRepository.cs | 13 ++++ .../RateNotificationRuleRepository.cs | 63 +++++++++++++++++++ 8 files changed, 209 insertions(+) create mode 100644 src/Exceptionless.Core/Models/Enums/RateNotificationSignal.cs create mode 100644 src/Exceptionless.Core/Models/Enums/RateNotificationSubject.cs create mode 100644 src/Exceptionless.Core/Models/Queues/RateNotification.cs create mode 100644 src/Exceptionless.Core/Models/RateNotificationRule.cs create mode 100644 src/Exceptionless.Core/Repositories/Configuration/Indexes/RateNotificationRuleIndex.cs create mode 100644 src/Exceptionless.Core/Repositories/Interfaces/IRateNotificationRuleRepository.cs create mode 100644 src/Exceptionless.Core/Repositories/RateNotificationRuleRepository.cs diff --git a/src/Exceptionless.Core/Models/Enums/RateNotificationSignal.cs b/src/Exceptionless.Core/Models/Enums/RateNotificationSignal.cs new file mode 100644 index 0000000000..e45a6f276f --- /dev/null +++ b/src/Exceptionless.Core/Models/Enums/RateNotificationSignal.cs @@ -0,0 +1,10 @@ +namespace Exceptionless.Core.Models; + +public enum RateNotificationSignal +{ + AllEvents = 0, + Errors = 1, + CriticalErrors = 2, + NewErrors = 3, + Regressions = 4 +} diff --git a/src/Exceptionless.Core/Models/Enums/RateNotificationSubject.cs b/src/Exceptionless.Core/Models/Enums/RateNotificationSubject.cs new file mode 100644 index 0000000000..44bca330a3 --- /dev/null +++ b/src/Exceptionless.Core/Models/Enums/RateNotificationSubject.cs @@ -0,0 +1,7 @@ +namespace Exceptionless.Core.Models; + +public enum RateNotificationSubject +{ + Project = 0, + Stack = 1 +} diff --git a/src/Exceptionless.Core/Models/Queues/RateNotification.cs b/src/Exceptionless.Core/Models/Queues/RateNotification.cs new file mode 100644 index 0000000000..55e81254ef --- /dev/null +++ b/src/Exceptionless.Core/Models/Queues/RateNotification.cs @@ -0,0 +1,16 @@ +namespace Exceptionless.Core.Queues.Models; + +public class RateNotification +{ + public required string RuleId { get; set; } + public required int RuleVersion { get; set; } + public required string OrganizationId { get; set; } + public required string ProjectId { get; set; } + public required string UserId { get; set; } + public required string SubjectKey { get; set; } + public string? StackId { get; set; } + public required DateTime WindowStartUtc { get; set; } + public required DateTime WindowEndUtc { get; set; } + public required long ObservedCount { get; set; } + public required int Threshold { get; set; } +} diff --git a/src/Exceptionless.Core/Models/RateNotificationRule.cs b/src/Exceptionless.Core/Models/RateNotificationRule.cs new file mode 100644 index 0000000000..0ffa6ad49a --- /dev/null +++ b/src/Exceptionless.Core/Models/RateNotificationRule.cs @@ -0,0 +1,55 @@ +using System.ComponentModel.DataAnnotations; +using Exceptionless.Core.Attributes; +using Foundatio.Repositories.Models; + +namespace Exceptionless.Core.Models; + +public class RateNotificationRule : IOwnedByOrganizationAndProjectWithIdentity, IHaveDates +{ + [ObjectId] + public string Id { get; set; } = null!; + + [Required] + [ObjectId] + public string OrganizationId { get; set; } = null!; + + [Required] + [ObjectId] + public string ProjectId { get; set; } = null!; + + [Required] + [ObjectId] + public string UserId { get; set; } = null!; + + public int Version { get; set; } = 1; + + [Required] + [MaxLength(100)] + public string Name { get; set; } = null!; + + public bool IsEnabled { get; set; } = true; + + public RateNotificationSignal Signal { get; set; } + + public RateNotificationSubject Subject { get; set; } + + [ObjectId] + public string? StackId { get; set; } + + [Range(1, int.MaxValue)] + public int Threshold { get; set; } = 10; + + public TimeSpan Window { get; set; } = TimeSpan.FromHours(1); + + public TimeSpan Cooldown { get; set; } = TimeSpan.FromHours(1); + + public DateTime? SnoozedUntilUtc { get; set; } + + public DateTime? LastFiredUtc { get; set; } + + public bool IsDeleted { get; set; } + + public DateTime CreatedUtc { get; set; } + + public DateTime UpdatedUtc { get; set; } +} diff --git a/src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs b/src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs index 2158adeade..127008a6dc 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs @@ -44,6 +44,7 @@ ILoggerFactory loggerFactory AddIndex(Migrations = new MigrationIndex(this, _appOptions.ElasticsearchOptions.ScopePrefix + "migrations", appOptions.ElasticsearchOptions.NumberOfReplicas)); AddIndex(Organizations = new OrganizationIndex(this)); AddIndex(Projects = new ProjectIndex(this)); + AddIndex(RateNotificationRules = new RateNotificationRuleIndex(this)); AddIndex(SavedViews = new SavedViewIndex(this)); AddIndex(Tokens = new TokenIndex(this)); AddIndex(Users = new UserIndex(this)); @@ -72,6 +73,7 @@ public override void ConfigureGlobalQueryBuilders(ElasticQueryBuilder builder) public MigrationIndex Migrations { get; } public OrganizationIndex Organizations { get; } public ProjectIndex Projects { get; } + public RateNotificationRuleIndex RateNotificationRules { get; } public SavedViewIndex SavedViews { get; } public TokenIndex Tokens { get; } public UserIndex Users { get; } diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/RateNotificationRuleIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/RateNotificationRuleIndex.cs new file mode 100644 index 0000000000..aab51d9184 --- /dev/null +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/RateNotificationRuleIndex.cs @@ -0,0 +1,43 @@ +using Exceptionless.Core.Models; +using Foundatio.Repositories.Elasticsearch.Configuration; +using Foundatio.Repositories.Elasticsearch.Extensions; +using Nest; + +namespace Exceptionless.Core.Repositories.Configuration; + +public sealed class RateNotificationRuleIndex : VersionedIndex +{ + private readonly ExceptionlessElasticConfiguration _configuration; + + public RateNotificationRuleIndex(ExceptionlessElasticConfiguration configuration) + : base(configuration, configuration.Options.ScopePrefix + "rate-notification-rules", 1) + { + _configuration = configuration; + } + + public override TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + { + return map + .Dynamic(false) + .Properties(p => p + .SetupDefaults() + .Keyword(f => f.Name(e => e.OrganizationId)) + .Keyword(f => f.Name(e => e.ProjectId)) + .Keyword(f => f.Name(e => e.UserId)) + .Keyword(f => f.Name(e => e.StackId)) + .Keyword(f => f.Name(e => e.Signal)) + .Keyword(f => f.Name(e => e.Subject)) + .Boolean(f => f.Name(e => e.IsEnabled)) + .Boolean(f => f.Name(e => e.IsDeleted)) + .Text(f => f.Name(e => e.Name).AddKeywordField()) + ); + } + + public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + { + return base.ConfigureIndex(idx.Settings(s => s + .NumberOfShards(_configuration.Options.NumberOfShards) + .NumberOfReplicas(_configuration.Options.NumberOfReplicas) + .Priority(5))); + } +} diff --git a/src/Exceptionless.Core/Repositories/Interfaces/IRateNotificationRuleRepository.cs b/src/Exceptionless.Core/Repositories/Interfaces/IRateNotificationRuleRepository.cs new file mode 100644 index 0000000000..f24ef6961f --- /dev/null +++ b/src/Exceptionless.Core/Repositories/Interfaces/IRateNotificationRuleRepository.cs @@ -0,0 +1,13 @@ +using Exceptionless.Core.Models; +using Foundatio.Repositories; +using Foundatio.Repositories.Models; + +namespace Exceptionless.Core.Repositories; + +public interface IRateNotificationRuleRepository : IRepositoryOwnedByOrganizationAndProject +{ + Task> GetByProjectIdAndUserIdAsync(string projectId, string userId, CommandOptionsDescriptor? options = null); + Task> GetEnabledByProjectIdAsync(string projectId, CommandOptionsDescriptor? options = null); + Task CountByProjectIdAndUserIdAsync(string projectId, string userId); + Task RemoveByProjectIdAndUserIdAsync(string projectId, string userId); +} diff --git a/src/Exceptionless.Core/Repositories/RateNotificationRuleRepository.cs b/src/Exceptionless.Core/Repositories/RateNotificationRuleRepository.cs new file mode 100644 index 0000000000..91998a0cdf --- /dev/null +++ b/src/Exceptionless.Core/Repositories/RateNotificationRuleRepository.cs @@ -0,0 +1,63 @@ +using Exceptionless.Core.Models; +using Exceptionless.Core.Repositories.Configuration; +using Exceptionless.Core.Validation; +using Foundatio.Repositories; +using Foundatio.Repositories.Models; +using Nest; + +namespace Exceptionless.Core.Repositories; + +public sealed class RateNotificationRuleRepository : RepositoryOwnedByOrganizationAndProject, IRateNotificationRuleRepository +{ + public RateNotificationRuleRepository(ExceptionlessElasticConfiguration configuration, MiniValidationValidator validator, AppOptions options) + : base(configuration.RateNotificationRules, validator, options) + { + } + + public Task> GetByProjectIdAndUserIdAsync(string projectId, string userId, CommandOptionsDescriptor? options = null) + { + ArgumentException.ThrowIfNullOrEmpty(projectId); + ArgumentException.ThrowIfNullOrEmpty(userId); + + return FindAsync(q => q + .Project(projectId) + .FieldEquals(r => r.UserId, userId) + .SortAscending(r => r.Name.Suffix("keyword")), options); + } + + public Task> GetEnabledByProjectIdAsync(string projectId, CommandOptionsDescriptor? options = null) + { + ArgumentException.ThrowIfNullOrEmpty(projectId); + + return FindAsync(q => q + .Project(projectId) + .FieldEquals(r => r.IsEnabled, true) + .FieldEquals(r => r.IsDeleted, false), options); + } + + public async Task CountByProjectIdAndUserIdAsync(string projectId, string userId) + { + ArgumentException.ThrowIfNullOrEmpty(projectId); + ArgumentException.ThrowIfNullOrEmpty(userId); + + return await CountAsync(q => q + .Project(projectId) + .FieldEquals(r => r.UserId, userId)); + } + + public async Task RemoveByProjectIdAndUserIdAsync(string projectId, string userId) + { + ArgumentException.ThrowIfNullOrEmpty(projectId); + ArgumentException.ThrowIfNullOrEmpty(userId); + + var results = await FindAsync(q => q + .Project(projectId) + .FieldEquals(r => r.UserId, userId), o => o.PageLimit(1000)); + + if (results.Total is 0) + return 0; + + await RemoveAsync(results.Documents); + return results.Total; + } +} From f2bc457e452c9ca1776dab63832331930b29f696 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sun, 31 May 2026 07:41:58 -0500 Subject: [PATCH 04/14] feat: add RateCounterService and pipeline action for event rate tracking RateCounterService uses ICacheClient 1-minute bucket counters to track event rates per signal/project/stack. 075_UpdateRateCountersAction [Priority(75)] increments counters during event ingestion for all matching signals. --- .../Pipeline/075_UpdateRateCountersAction.cs | 110 ++++++++++++++++++ .../Services/RateCounterService.cs | 100 ++++++++++++++++ .../Services/RateNotificationRuleCache.cs | 53 +++++++++ 3 files changed, 263 insertions(+) create mode 100644 src/Exceptionless.Core/Pipeline/075_UpdateRateCountersAction.cs create mode 100644 src/Exceptionless.Core/Services/RateCounterService.cs create mode 100644 src/Exceptionless.Core/Services/RateNotificationRuleCache.cs diff --git a/src/Exceptionless.Core/Pipeline/075_UpdateRateCountersAction.cs b/src/Exceptionless.Core/Pipeline/075_UpdateRateCountersAction.cs new file mode 100644 index 0000000000..75381a2afc --- /dev/null +++ b/src/Exceptionless.Core/Pipeline/075_UpdateRateCountersAction.cs @@ -0,0 +1,110 @@ +using Exceptionless.Core.Extensions; +using Exceptionless.Core.Models; +using Exceptionless.Core.Plugins.EventProcessor; +using Exceptionless.Core.Services; +using Exceptionless.Core.Utility; +using Microsoft.Extensions.Logging; +using System.Text.Json; + +namespace Exceptionless.Core.Pipeline; + +[Priority(75)] +public class UpdateRateCountersAction : EventPipelineActionBase +{ + private readonly RateNotificationRuleCache _ruleCache; + private readonly RateCounterService _counterService; + private readonly UserAgentParser _parser; + private readonly JsonSerializerOptions _jsonOptions; + + public UpdateRateCountersAction( + RateNotificationRuleCache ruleCache, + RateCounterService counterService, + UserAgentParser parser, + JsonSerializerOptions jsonOptions, + AppOptions options, + ILoggerFactory loggerFactory) : base(options, loggerFactory) + { + _ruleCache = ruleCache; + _counterService = counterService; + _parser = parser; + _jsonOptions = jsonOptions; + ContinueOnError = true; + } + + public override async Task ProcessAsync(EventContext ctx) + { + // Premium gate — rate notifications require premium features + if (!ctx.Organization.HasPremiumFeatures) + return; + + // Stack must allow notifications + if (ctx.Stack is null || !ctx.Stack.AllowNotifications) + return; + + // Load enabled rules for this project + var rules = await _ruleCache.GetEnabledRulesAsync(ctx.Event.ProjectId); + if (rules.Count == 0) + return; + + // Bot check — same pattern as EventNotificationsJob + var request = ctx.Event.GetRequestInfo(_jsonOptions); + if (!String.IsNullOrEmpty(request?.UserAgent)) + { + var botPatterns = ctx.Project.Configuration.Settings + .GetStringCollection(SettingsDictionary.KnownKeys.UserAgentBotPatterns).ToList(); + var info = await _parser.ParseAsync(request.UserAgent); + if (info is not null && info.Device.IsSpider || request.UserAgent.AnyWildcardMatches(botPatterns)) + { + _logger.LogTrace("Skipping rate counter update for bot user agent: {UserAgent}", request.UserAgent); + return; + } + } + + // Build the set of signals matched by this event + bool isError = ctx.Event.IsError(); + bool isCritical = ctx.Event.IsCritical(); + var matchedSignals = new HashSet(); + + // AllEvents always matches + matchedSignals.Add(RateNotificationSignal.AllEvents); + + if (isError) + matchedSignals.Add(RateNotificationSignal.Errors); + + if (isError && isCritical) + matchedSignals.Add(RateNotificationSignal.CriticalErrors); + + if (ctx.IsNew && isError) + matchedSignals.Add(RateNotificationSignal.NewErrors); + + if (ctx.IsRegression) + matchedSignals.Add(RateNotificationSignal.Regressions); + + // Increment counters for each matching rule + foreach (var rule in rules) + { + if (!matchedSignals.Contains(rule.Signal)) + continue; + + // For Stack subject, only match if this event belongs to the rule's stack + if (rule.Subject == RateNotificationSubject.Stack) + { + if (String.IsNullOrEmpty(rule.StackId) || !String.Equals(ctx.Event.StackId, rule.StackId, StringComparison.Ordinal)) + continue; + } + + string counterKey = BuildCounterKey(rule); + await _counterService.IncrementAsync(counterKey); + } + } + + internal static string BuildCounterKey(RateNotificationRule rule) + { + return rule.Subject switch + { + RateNotificationSubject.Project => $"project:{rule.ProjectId}:signal:{rule.Signal}", + RateNotificationSubject.Stack => $"project:{rule.ProjectId}:stack:{rule.StackId}:signal:{rule.Signal}", + _ => $"project:{rule.ProjectId}:signal:{rule.Signal}" + }; + } +} diff --git a/src/Exceptionless.Core/Services/RateCounterService.cs b/src/Exceptionless.Core/Services/RateCounterService.cs new file mode 100644 index 0000000000..5294864e2e --- /dev/null +++ b/src/Exceptionless.Core/Services/RateCounterService.cs @@ -0,0 +1,100 @@ +using Foundatio.Caching; +using Microsoft.Extensions.Logging; + +namespace Exceptionless.Core.Services; + +/// +/// Cache-backed 1-minute bucket counter service for rate notification evaluation. +/// Keys: +/// rate:v1:count:{epochMinute}:{counterKey} — TTL 3h +/// rate:v1:active:{epochMinute} — TTL 3h (list of counter keys) +/// rate:v1:cooldown:{ruleId}:{subjectKey} — TTL = cooldown + 10min +/// +public class RateCounterService +{ + private readonly ICacheClient _cache; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + private static readonly TimeSpan BucketTtl = TimeSpan.FromHours(3); + + public RateCounterService(ICacheClient cache, TimeProvider timeProvider, ILoggerFactory loggerFactory) + { + _cache = cache; + _timeProvider = timeProvider; + _logger = loggerFactory.CreateLogger(); + } + + /// Increments the 1-minute bucket counter for the given counter key at the current UTC minute. + public async Task IncrementAsync(string counterKey, CancellationToken ct = default) + { + var now = _timeProvider.GetUtcNow().UtcDateTime; + long epochMinute = GetEpochMinute(now); + + string countKey = GetCountKey(epochMinute, counterKey); + await _cache.IncrementAsync(countKey, 1, BucketTtl); + + string activeKey = GetActiveKey(epochMinute); + await _cache.ListAddAsync(activeKey, counterKey, BucketTtl); + } + + /// Sums all 1-minute bucket counts for the given counter key in the range [fromUtc, toUtc]. + public async Task SumBucketsAsync(string counterKey, DateTime fromUtc, DateTime toUtc, CancellationToken ct = default) + { + long fromMinute = GetEpochMinute(fromUtc); + long toMinute = GetEpochMinute(toUtc); + + long total = 0; + for (long minute = fromMinute; minute <= toMinute; minute++) + { + string key = GetCountKey(minute, counterKey); + var value = await _cache.GetAsync(key); + if (value.HasValue) + total += value.Value; + } + + return total; + } + + /// Returns all counter keys that were active during the given minute. + public async Task> GetActiveCounterKeysAsync(DateTime minute, CancellationToken ct = default) + { + long epochMinute = GetEpochMinute(minute); + string activeKey = GetActiveKey(epochMinute); + + var result = await _cache.GetListAsync(activeKey); + if (!result.HasValue || result.Value is null) + return Array.Empty(); + + return result.Value.Where(k => k is not null).Distinct().ToList()!; + } + + /// Returns true if the rule/subject is currently on cooldown. + public Task IsOnCooldownAsync(string ruleId, string subjectKey, CancellationToken ct = default) + { + string key = GetCooldownKey(ruleId, subjectKey); + return _cache.ExistsAsync(key); + } + + /// Sets a cooldown for the given rule/subject combination. + public Task SetCooldownAsync(string ruleId, string subjectKey, TimeSpan duration, CancellationToken ct = default) + { + string key = GetCooldownKey(ruleId, subjectKey); + // TTL = duration + 10 minutes buffer + return _cache.SetAsync(key, true, duration.Add(TimeSpan.FromMinutes(10))); + } + + // ---- Key helpers ---- + + private static long GetEpochMinute(DateTime utc) + => (long)(utc - DateTime.UnixEpoch).TotalMinutes; + + private static string GetCountKey(long epochMinute, string counterKey) + => $"rate:v1:count:{epochMinute}:{counterKey}"; + + private static string GetActiveKey(long epochMinute) + => $"rate:v1:active:{epochMinute}"; + + private static string GetCooldownKey(string ruleId, string subjectKey) + => $"rate:v1:cooldown:{ruleId}:{subjectKey}"; +} diff --git a/src/Exceptionless.Core/Services/RateNotificationRuleCache.cs b/src/Exceptionless.Core/Services/RateNotificationRuleCache.cs new file mode 100644 index 0000000000..777b754ac7 --- /dev/null +++ b/src/Exceptionless.Core/Services/RateNotificationRuleCache.cs @@ -0,0 +1,53 @@ +using Exceptionless.Core.Models; +using Exceptionless.Core.Repositories; +using Foundatio.Caching; +using Foundatio.Repositories; +using Foundatio.Repositories.Models; +using Microsoft.Extensions.Logging; + +namespace Exceptionless.Core.Services; + +/// +/// Cache layer over IRateNotificationRuleRepository. +/// Cache key: rate:v1:rules:project:{projectId} TTL: 5 minutes +/// +public class RateNotificationRuleCache +{ + private readonly IRateNotificationRuleRepository _repository; + private readonly ICacheClient _cache; + private readonly ILogger _logger; + + private static readonly TimeSpan CacheTtl = TimeSpan.FromMinutes(5); + + public RateNotificationRuleCache(IRateNotificationRuleRepository repository, ICacheClient cache, ILoggerFactory loggerFactory) + { + _repository = repository; + _cache = cache; + _logger = loggerFactory.CreateLogger(); + } + + /// Returns all enabled, non-deleted rules for the given project (cached). + public async Task> GetEnabledRulesAsync(string projectId, CancellationToken ct = default) + { + string cacheKey = GetCacheKey(projectId); + + var cached = await _cache.GetAsync>(cacheKey); + if (cached.HasValue && cached.Value is not null) + return cached.Value; + + var results = await _repository.GetEnabledByProjectIdAsync(projectId, o => o.PageLimit(1000)); + var rules = results.Documents.ToList(); + + await _cache.SetAsync(cacheKey, rules, CacheTtl); + return rules; + } + + /// Invalidates the cache for the given project. + public Task InvalidateAsync(string projectId) + { + string cacheKey = GetCacheKey(projectId); + return _cache.RemoveAsync(cacheKey); + } + + private static string GetCacheKey(string projectId) => $"rate:v1:rules:project:{projectId}"; +} From fc1636f5ed8bbdadbbfd4278538ff6602c12b19b Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sun, 31 May 2026 07:42:06 -0500 Subject: [PATCH 05/14] feat: add RateNotificationEvaluatorJob and RateNotificationsJob EvaluatorJob runs minutely (cron) to check rules against counters and enqueue notifications. Implements the snooze back-alert fix: uses max(windowStart, snoozedUntilUtc) as the effective window start to prevent false alerts from traffic counted during a snooze period. RateNotificationsJob dequeues and sends notifications via IMailer. --- .../Jobs/RateNotificationEvaluatorJob.cs | 224 ++++++++++++++++++ .../Jobs/RateNotificationsJob.cs | 105 ++++++++ 2 files changed, 329 insertions(+) create mode 100644 src/Exceptionless.Core/Jobs/RateNotificationEvaluatorJob.cs create mode 100644 src/Exceptionless.Core/Jobs/RateNotificationsJob.cs diff --git a/src/Exceptionless.Core/Jobs/RateNotificationEvaluatorJob.cs b/src/Exceptionless.Core/Jobs/RateNotificationEvaluatorJob.cs new file mode 100644 index 0000000000..4f97835c30 --- /dev/null +++ b/src/Exceptionless.Core/Jobs/RateNotificationEvaluatorJob.cs @@ -0,0 +1,224 @@ +using Exceptionless.Core.Models; +using Exceptionless.Core.Pipeline; +using Exceptionless.Core.Queues.Models; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Services; +using Foundatio.Jobs; +using Foundatio.Lock; +using Foundatio.Queues; +using Foundatio.Repositories; +using Foundatio.Resilience; +using Microsoft.Extensions.Logging; + +namespace Exceptionless.Core.Jobs; + +[Job(Description = "Evaluates rate notification rules and enqueues notifications.", IsContinuous = false)] +public class RateNotificationEvaluatorJob : JobWithLockBase +{ + private readonly RateCounterService _counterService; + private readonly IRateNotificationRuleRepository _ruleRepository; + private readonly IOrganizationRepository _organizationRepository; + private readonly IQueue _notificationQueue; + private readonly ILockProvider _lockProvider; + + /// How far back to scan for active counter minutes. + private static readonly TimeSpan ScanWindow = TimeSpan.FromHours(2); + + public RateNotificationEvaluatorJob( + RateCounterService counterService, + IRateNotificationRuleRepository ruleRepository, + IOrganizationRepository organizationRepository, + IQueue notificationQueue, + ILockProvider lockProvider, + TimeProvider timeProvider, + IResiliencePolicyProvider resiliencePolicyProvider, + ILoggerFactory loggerFactory) : base(timeProvider, resiliencePolicyProvider, loggerFactory) + { + _counterService = counterService; + _ruleRepository = ruleRepository; + _organizationRepository = organizationRepository; + _notificationQueue = notificationQueue; + _lockProvider = lockProvider; + } + + protected override Task GetLockAsync(CancellationToken cancellationToken = default) + { + return _lockProvider.TryAcquireAsync(nameof(RateNotificationEvaluatorJob), TimeSpan.FromMinutes(2), cancellationToken); + } + + protected override async Task RunInternalAsync(JobContext context) + { + var now = _timeProvider.GetUtcNow().UtcDateTime; + var scanFrom = now.Subtract(ScanWindow); + + // Round scanFrom down to minute boundary; stop 1 minute before now (current bucket may be incomplete) + var fromMinute = new DateTime(scanFrom.Year, scanFrom.Month, scanFrom.Day, scanFrom.Hour, scanFrom.Minute, 0, DateTimeKind.Utc); + var toMinute = new DateTime(now.Year, now.Month, now.Day, now.Hour, now.Minute, 0, DateTimeKind.Utc).AddMinutes(-1); + + if (fromMinute > toMinute) + return JobResult.Success; + + _logger.LogInformation("Evaluating rate notification rules from {From} to {To}", fromMinute, toMinute); + + // Collect all unique counter keys across all minutes in the scan window + var allCounterKeys = new HashSet(); + for (var minute = fromMinute; minute <= toMinute; minute = minute.AddMinutes(1)) + { + var keys = await _counterService.GetActiveCounterKeysAsync(minute, context.CancellationToken); + foreach (var key in keys) + allCounterKeys.Add(key); + } + + if (allCounterKeys.Count == 0) + { + _logger.LogDebug("No active counter keys found in scan window"); + return JobResult.Success; + } + + // For each unique counter key, evaluate all matching rules + foreach (string counterKey in allCounterKeys) + { + if (context.CancellationToken.IsCancellationRequested) + return JobResult.Cancelled; + + await EvaluateCounterKeyAsync(counterKey, now, context.CancellationToken); + } + + _logger.LogInformation("Finished evaluating rate notification rules"); + return JobResult.Success; + } + + private async Task EvaluateCounterKeyAsync(string counterKey, DateTime now, CancellationToken ct) + { + // Parse projectId from counter key to load rules + string? projectId = ParseProjectIdFromCounterKey(counterKey); + if (projectId is null) + { + _logger.LogWarning("Unable to parse projectId from counter key: {CounterKey}", counterKey); + return; + } + + // Load enabled rules for project matching this counter key + var allProjectRules = await _ruleRepository.GetEnabledByProjectIdAsync(projectId, o => o.PageLimit(1000)); + + // Filter rules matching this counter key + var matchingRules = allProjectRules.Documents + .Where(r => CounterKeyMatchesRule(counterKey, r)) + .ToList(); + + if (matchingRules.Count == 0) + return; + + // Load org once per project for premium check + string? organizationId = matchingRules.First().OrganizationId; + var org = await _organizationRepository.GetByIdAsync(organizationId, o => o.Cache()); + if (org is null) + return; + + if (!org.HasPremiumFeatures) + return; + + foreach (var rule in matchingRules) + { + if (ct.IsCancellationRequested) + return; + + await EvaluateRuleAsync(rule, counterKey, now, ct); + } + } + + private async Task EvaluateRuleAsync(RateNotificationRule rule, string counterKey, DateTime now, CancellationToken ct) + { + if (!rule.IsEnabled) + return; + + // Skip if actively snoozed + if (rule.SnoozedUntilUtc.HasValue && rule.SnoozedUntilUtc.Value > now) + { + _logger.LogDebug("Skipping snoozed rule {RuleId} snoozed until {SnoozedUntil}", rule.Id, rule.SnoozedUntilUtc); + return; + } + + var windowStartUtc = now.Subtract(rule.Window); + + // SNOOZE BACK-ALERT FIX: + // If the rule was recently un-snoozed (SnoozedUntilUtc is set and in the past), use that as the + // effective window start to ignore traffic that occurred during the snooze period. + var effectiveWindowStartUtc = rule.SnoozedUntilUtc.HasValue && rule.SnoozedUntilUtc.Value > windowStartUtc + ? rule.SnoozedUntilUtc.Value + : windowStartUtc; + + var observedCount = await _counterService.SumBucketsAsync(counterKey, effectiveWindowStartUtc, now, ct); + + if (observedCount < rule.Threshold) + { + _logger.LogDebug("Rule {RuleId}: observed={Observed} < threshold={Threshold}, skipping", rule.Id, observedCount, rule.Threshold); + return; + } + + // Build subject key (for cooldown scoping) + string subjectKey = BuildSubjectKey(rule); + + // Check cooldown + if (await _counterService.IsOnCooldownAsync(rule.Id, subjectKey, ct)) + { + _logger.LogDebug("Rule {RuleId} is on cooldown for subject {SubjectKey}", rule.Id, subjectKey); + return; + } + + // Enqueue notification + await _notificationQueue.EnqueueAsync(new RateNotification + { + RuleId = rule.Id, + RuleVersion = rule.Version, + OrganizationId = rule.OrganizationId, + ProjectId = rule.ProjectId, + UserId = rule.UserId, + SubjectKey = subjectKey, + StackId = rule.StackId, + WindowStartUtc = effectiveWindowStartUtc, + WindowEndUtc = now, + ObservedCount = observedCount, + Threshold = rule.Threshold + }); + + // Set cooldown + await _counterService.SetCooldownAsync(rule.Id, subjectKey, rule.Cooldown, ct); + + // Update LastFiredUtc + rule.LastFiredUtc = now; + await _ruleRepository.SaveAsync(rule); + + _logger.LogInformation("Rate notification fired: rule={RuleId} project={ProjectId} observed={Observed} threshold={Threshold}", + rule.Id, rule.ProjectId, observedCount, rule.Threshold); + } + + /// Parses the projectId from a counter key of the form: project:{projectId}:... + private static string? ParseProjectIdFromCounterKey(string counterKey) + { + const string prefix = "project:"; + if (!counterKey.StartsWith(prefix)) + return null; + + int start = prefix.Length; + int end = counterKey.IndexOf(':', start); + if (end < 0) + return null; + + return counterKey[start..end]; + } + + /// Returns true if the counter key was generated by the given rule. + private static bool CounterKeyMatchesRule(string counterKey, RateNotificationRule rule) + { + string expected = UpdateRateCountersAction.BuildCounterKey(rule); + return String.Equals(counterKey, expected, StringComparison.Ordinal); + } + + private static string BuildSubjectKey(RateNotificationRule rule) + { + return rule.Subject == RateNotificationSubject.Stack && !String.IsNullOrEmpty(rule.StackId) + ? $"stack:{rule.StackId}" + : $"project:{rule.ProjectId}"; + } +} diff --git a/src/Exceptionless.Core/Jobs/RateNotificationsJob.cs b/src/Exceptionless.Core/Jobs/RateNotificationsJob.cs new file mode 100644 index 0000000000..253ec202de --- /dev/null +++ b/src/Exceptionless.Core/Jobs/RateNotificationsJob.cs @@ -0,0 +1,105 @@ +using Exceptionless.Core.Mail; +using Exceptionless.Core.Models; +using Exceptionless.Core.Queues.Models; +using Exceptionless.Core.Repositories; +using Foundatio.Jobs; +using Foundatio.Queues; +using Foundatio.Repositories; +using Foundatio.Resilience; +using Microsoft.Extensions.Logging; + +namespace Exceptionless.Core.Jobs; + +[Job(Description = "Delivers rate notification emails.", InitialDelay = "5s")] +public class RateNotificationsJob : QueueJobBase +{ + private readonly IMailer _mailer; + private readonly IRateNotificationRuleRepository _ruleRepository; + private readonly IProjectRepository _projectRepository; + private readonly IUserRepository _userRepository; + private readonly IStackRepository _stackRepository; + + public RateNotificationsJob( + IQueue queue, + IMailer mailer, + IRateNotificationRuleRepository ruleRepository, + IProjectRepository projectRepository, + IUserRepository userRepository, + IStackRepository stackRepository, + TimeProvider timeProvider, + IResiliencePolicyProvider resiliencePolicyProvider, + ILoggerFactory loggerFactory) : base(queue, timeProvider, resiliencePolicyProvider, loggerFactory) + { + _mailer = mailer; + _ruleRepository = ruleRepository; + _projectRepository = projectRepository; + _userRepository = userRepository; + _stackRepository = stackRepository; + } + + protected override async Task ProcessQueueEntryAsync(QueueEntryContext context) + { + var wi = context.QueueEntry.Value; + + // Load rule + var rule = await _ruleRepository.GetByIdAsync(wi.RuleId); + if (rule is null) + return JobResult.SuccessWithMessage($"Rate notification rule {wi.RuleId} not found; skipping."); + + if (!rule.IsEnabled) + return JobResult.SuccessWithMessage($"Rule {wi.RuleId} is disabled; skipping."); + + // Version check — rule was mutated after enqueueing + if (rule.Version != wi.RuleVersion) + { + _logger.LogInformation("Rule {RuleId} version mismatch: expected {Expected}, found {Actual}; skipping stale notification", + wi.RuleId, wi.RuleVersion, rule.Version); + return JobResult.Success; + } + + // Load project + var project = await _projectRepository.GetByIdAsync(rule.ProjectId, o => o.Cache()); + if (project is null) + return JobResult.SuccessWithMessage($"Project {rule.ProjectId} not found; skipping."); + + // Load user + var user = await _userRepository.GetByIdAsync(rule.UserId, o => o.Cache()); + if (user is null) + return JobResult.SuccessWithMessage($"User {rule.UserId} not found; skipping."); + + // User must still be a member of the org + if (!user.OrganizationIds.Contains(rule.OrganizationId)) + { + _logger.LogInformation("User {UserId} is no longer a member of org {OrgId}; skipping rate notification", rule.UserId, rule.OrganizationId); + return JobResult.Success; + } + + if (!user.IsEmailAddressVerified) + { + _logger.LogInformation("User {UserId} email not verified; skipping rate notification", rule.UserId); + return JobResult.Success; + } + + if (!user.EmailNotificationsEnabled) + { + _logger.LogInformation("User {UserId} has email notifications disabled; skipping rate notification", rule.UserId); + return JobResult.Success; + } + + // Load stack if this is a stack-scoped rule + Stack? stack = null; + if (rule.Subject == RateNotificationSubject.Stack && !String.IsNullOrEmpty(rule.StackId)) + { + stack = await _stackRepository.GetByIdAsync(rule.StackId, o => o.Cache()); + if (stack is null) + _logger.LogWarning("Stack {StackId} not found for rate notification rule {RuleId}", rule.StackId, rule.Id); + } + + await _mailer.SendRateNotificationAsync(user, project, rule, wi.ObservedCount, wi.WindowStartUtc, wi.WindowEndUtc, stack); + + _logger.LogInformation("Sent rate notification email: rule={RuleId} user={UserId} project={ProjectId} observed={Observed}", + rule.Id, rule.UserId, rule.ProjectId, wi.ObservedCount); + + return JobResult.Success; + } +} From bd51eed3d82842cafacb82997c4b22b5c2a4b851 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sun, 31 May 2026 07:42:12 -0500 Subject: [PATCH 06/14] feat: add SendRateNotificationAsync to IMailer and rate-notification email template Adds Handlebars HTML email template and mailer implementation. Template includes rule name, signal, threshold, observed count, project info, and snooze/manage links. --- .../Exceptionless.Core.csproj | 2 + src/Exceptionless.Core/Mail/IMailer.cs | 1 + src/Exceptionless.Core/Mail/Mailer.cs | 41 +++++++++++++++++++ .../Mail/Templates/rate-notification.html | 1 + .../Mail/CountingMailer.cs | 5 +++ tests/Exceptionless.Tests/Mail/NullMailer.cs | 5 +++ 6 files changed, 55 insertions(+) create mode 100644 src/Exceptionless.Core/Mail/Templates/rate-notification.html diff --git a/src/Exceptionless.Core/Exceptionless.Core.csproj b/src/Exceptionless.Core/Exceptionless.Core.csproj index 2ee31516e5..af25f9a2c5 100644 --- a/src/Exceptionless.Core/Exceptionless.Core.csproj +++ b/src/Exceptionless.Core/Exceptionless.Core.csproj @@ -6,6 +6,7 @@ + @@ -16,6 +17,7 @@ + diff --git a/src/Exceptionless.Core/Mail/IMailer.cs b/src/Exceptionless.Core/Mail/IMailer.cs index 79b3b7f39e..a8f1c1e2b8 100644 --- a/src/Exceptionless.Core/Mail/IMailer.cs +++ b/src/Exceptionless.Core/Mail/IMailer.cs @@ -10,6 +10,7 @@ public interface IMailer Task SendOrganizationNoticeAsync(User user, Organization organization, bool isOverMonthlyLimit, bool isOverHourlyLimit); Task SendOrganizationPaymentFailedAsync(User owner, Organization organization); Task SendProjectDailySummaryAsync(User user, Project project, IEnumerable? mostFrequent, IEnumerable? newest, DateTime startDate, bool hasSubmittedEvents, double count, double uniqueCount, double newCount, double fixedCount, int blockedCount, int tooBigCount, bool isFreePlan); + Task SendRateNotificationAsync(User user, Project project, RateNotificationRule rule, long observedCount, DateTime windowStart, DateTime windowEnd, Stack? stack = null); Task SendUserEmailVerifyAsync(User user); Task SendUserPasswordResetAsync(User user); } diff --git a/src/Exceptionless.Core/Mail/Mailer.cs b/src/Exceptionless.Core/Mail/Mailer.cs index b6498c1d45..16d6ba395e 100644 --- a/src/Exceptionless.Core/Mail/Mailer.cs +++ b/src/Exceptionless.Core/Mail/Mailer.cs @@ -281,6 +281,47 @@ public Task SendUserPasswordResetAsync(User user) }, template); } + public Task SendRateNotificationAsync(User user, Project project, RateNotificationRule rule, long observedCount, DateTime windowStart, DateTime windowEnd, Stack? stack = null) + { + const string template = "rate-notification"; + string windowDescription = FormatWindow(rule.Window); + + var data = new Dictionary { + { "Subject", $"Error rate exceeded: {rule.Name}" }, + { "BaseUrl", _appOptions.BaseURL }, + { "ProjectName", project.Name }, + { "ProjectId", project.Id }, + { "RuleName", rule.Name }, + { "ObservedCount", observedCount }, + { "Threshold", rule.Threshold }, + { "Window", windowDescription }, + { "Signal", rule.Signal.ToString() }, + { "Subject_Type", rule.Subject.ToString() }, + { "StackId", rule.StackId }, + { "StackTitle", stack?.Title }, + { "HasStack", stack is not null }, + { "WindowStartUtc", windowStart.ToString("u") }, + { "WindowEndUtc", windowEnd.ToString("u") }, + { "Cooldown", FormatWindow(rule.Cooldown) } + }; + + return QueueMessageAsync(new MailMessage + { + To = user.EmailAddress, + Subject = $"[{project.Name}] Error rate exceeded: {rule.Name}", + Body = RenderTemplate(template, data) + }, template); + } + + private static string FormatWindow(TimeSpan window) + { + if (window.TotalHours >= 1 && window.Minutes == 0) + return $"{(int)window.TotalHours}h"; + if (window.TotalMinutes >= 1 && window.Seconds == 0) + return $"{(int)window.TotalMinutes}min"; + return window.ToString(); + } + private string RenderTemplate(string name, IDictionary data) { var template = GetCompiledTemplate(name); diff --git a/src/Exceptionless.Core/Mail/Templates/rate-notification.html b/src/Exceptionless.Core/Mail/Templates/rate-notification.html new file mode 100644 index 0000000000..5b6da87dfb --- /dev/null +++ b/src/Exceptionless.Core/Mail/Templates/rate-notification.html @@ -0,0 +1 @@ +{{Subject}}
Exceptionless
 

Your rate notification rule “{{RuleName}}” has fired for the {{ProjectName}} project.

Events Observed
{{ObservedCount}} events (threshold: {{Threshold}}) in the last {{Window}}

Signal
{{Signal}}

Scope
{{Subject_Type}}{{#if HasStack}} — {{StackTitle}}{{/if}}

Window
{{WindowStartUtc}} – {{WindowEndUtc}}

{{#if HasStack}}View Stack {{else}}View Project Dashboard {{/if}}

This notification will not be sent again for another {{Cooldown}} (cooldown period).

diff --git a/tests/Exceptionless.Tests/Mail/CountingMailer.cs b/tests/Exceptionless.Tests/Mail/CountingMailer.cs index 96d5671c3f..1a6f5c1097 100644 --- a/tests/Exceptionless.Tests/Mail/CountingMailer.cs +++ b/tests/Exceptionless.Tests/Mail/CountingMailer.cs @@ -65,6 +65,11 @@ public Task SendUserPasswordResetAsync(User user) return Task.CompletedTask; } + public Task SendRateNotificationAsync(User user, Project project, RateNotificationRule rule, long observedCount, DateTime windowStart, DateTime windowEnd, Stack? stack = null) + { + return Task.CompletedTask; + } + public void Reset() { Interlocked.Exchange(ref _organizationNoticeCount, 0); diff --git a/tests/Exceptionless.Tests/Mail/NullMailer.cs b/tests/Exceptionless.Tests/Mail/NullMailer.cs index 826be1324d..bef6901698 100644 --- a/tests/Exceptionless.Tests/Mail/NullMailer.cs +++ b/tests/Exceptionless.Tests/Mail/NullMailer.cs @@ -44,4 +44,9 @@ public Task SendUserPasswordResetAsync(User user) { return Task.CompletedTask; } + + public Task SendRateNotificationAsync(User user, Project project, RateNotificationRule rule, long observedCount, DateTime windowStart, DateTime windowEnd, Stack? stack = null) + { + return Task.CompletedTask; + } } From 9692a4bc0874e8767a04ef2b5c2018b10847c986 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sun, 31 May 2026 07:42:19 -0500 Subject: [PATCH 07/14] feat: add RateNotificationRule API controller and DTOs REST CRUD + snooze/unsnooze endpoints under: /api/v2/users/{userId}/projects/{projectId}/rate-notifications Unsnooze sets SnoozedUntilUtc = now (not null) to prevent back-alerts after snooze expiry. Premium gate: non-premium forces IsEnabled=false. --- .../RateNotificationRuleController.cs | 387 ++++++++++++++++++ .../RateNotificationRuleModels.cs | 82 ++++ 2 files changed, 469 insertions(+) create mode 100644 src/Exceptionless.Web/Controllers/RateNotificationRuleController.cs create mode 100644 src/Exceptionless.Web/Models/RateNotification/RateNotificationRuleModels.cs diff --git a/src/Exceptionless.Web/Controllers/RateNotificationRuleController.cs b/src/Exceptionless.Web/Controllers/RateNotificationRuleController.cs new file mode 100644 index 0000000000..3a3998734d --- /dev/null +++ b/src/Exceptionless.Web/Controllers/RateNotificationRuleController.cs @@ -0,0 +1,387 @@ +using Exceptionless.Core.Authorization; +using Exceptionless.Core.Models; +using Exceptionless.Core.Queries.Validation; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Services; +using Exceptionless.Web.Controllers; +using Exceptionless.Web.Extensions; +using Exceptionless.Web.Models; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Foundatio.Repositories; + +namespace Exceptionless.App.Controllers.API; + +/// +/// Personal rate notification rule management. +/// +[Route(API_PREFIX + "/users/{userId:objectid}/projects/{projectId:objectid}/rate-notifications")] +[Authorize(Policy = AuthorizationRoles.UserPolicy)] +public class RateNotificationRuleController : ExceptionlessApiController +{ + private const int MaxRulesPerUserPerProject = 20; + + private static readonly TimeSpan[] ValidWindows = + [ + TimeSpan.FromMinutes(1), + TimeSpan.FromMinutes(5), + TimeSpan.FromMinutes(10), + TimeSpan.FromMinutes(15), + TimeSpan.FromMinutes(30), + TimeSpan.FromHours(1) + ]; + + private readonly IRateNotificationRuleRepository _ruleRepository; + private readonly IProjectRepository _projectRepository; + private readonly IOrganizationRepository _organizationRepository; + private readonly IStackRepository _stackRepository; + private readonly RateNotificationRuleCache _ruleCache; + + public RateNotificationRuleController( + IRateNotificationRuleRepository ruleRepository, + IProjectRepository projectRepository, + IOrganizationRepository organizationRepository, + IStackRepository stackRepository, + RateNotificationRuleCache ruleCache, + TimeProvider timeProvider, + ILoggerFactory loggerFactory) : base(timeProvider) + { + _ruleRepository = ruleRepository; + _projectRepository = projectRepository; + _organizationRepository = organizationRepository; + _stackRepository = stackRepository; + _ruleCache = ruleCache; + } + + /// Get all rate notification rules for a user/project. + [HttpGet] + public async Task>> GetAsync( + string userId, + string projectId, + int page = 1, + int limit = 25) + { + if (!CanManage(userId)) + return NotFound(); + + var project = await GetProjectAndCheckAccessAsync(projectId); + if (project is null) + return NotFound(); + + page = GetPage(page); + limit = GetLimit(limit); + + var results = await _ruleRepository.GetByProjectIdAndUserIdAsync(projectId, userId, o => o.PageNumber(page).PageLimit(limit)); + var viewModels = results.Documents.Select(MapToView).ToList(); + return OkWithResourceLinks(viewModels, results.HasMore && !NextPageExceedsSkipLimit(page, limit), page, results.Total); + } + + /// Create a rate notification rule. + [HttpPost] + [Consumes("application/json")] + [ProducesResponseType(StatusCodes.Status201Created)] + public async Task> PostAsync( + string userId, + string projectId, + [FromBody] NewRateNotificationRule model) + { + if (!CanManage(userId)) + return NotFound(); + + var project = await GetProjectAndCheckAccessAsync(projectId); + if (project is null) + return NotFound(); + + // Validate window + if (!ValidWindows.Contains(model.Window)) + return ValidationProblem(detail: $"Window must be one of: {String.Join(", ", ValidWindows.Select(w => w.ToString()))}"); + + // Validate cooldown >= window + if (model.Cooldown < model.Window) + return ValidationProblem(detail: "Cooldown must be greater than or equal to Window."); + + // Validate subject / stackId + if (model.Subject == RateNotificationSubject.Stack) + { + if (String.IsNullOrEmpty(model.StackId)) + return ValidationProblem(detail: "StackId is required when Subject is Stack."); + + var stack = await _stackRepository.GetByIdAsync(model.StackId, o => o.Cache()); + if (stack is null || !String.Equals(stack.ProjectId, projectId, StringComparison.Ordinal)) + return ValidationProblem(detail: "The specified StackId does not belong to this project."); + } + else if (!String.IsNullOrEmpty(model.StackId)) + { + return ValidationProblem(detail: "StackId must be empty when Subject is Project."); + } + + // Enforce max rules limit + long count = await _ruleRepository.CountByProjectIdAndUserIdAsync(projectId, userId); + if (count >= MaxRulesPerUserPerProject) + return ValidationProblem(detail: $"Maximum of {MaxRulesPerUserPerProject} rate notification rules per user per project."); + + // Premium gate — non-premium users can create rules but they start disabled + var org = await _organizationRepository.GetByIdAsync(project.OrganizationId, o => o.Cache()); + bool isEnabled = model.IsEnabled; + if (!org?.HasPremiumFeatures ?? false) + isEnabled = false; + + var now = _timeProvider.GetUtcNow().UtcDateTime; + var rule = new RateNotificationRule + { + OrganizationId = project.OrganizationId, + ProjectId = projectId, + UserId = userId, + Name = model.Name, + IsEnabled = isEnabled, + Signal = model.Signal, + Subject = model.Subject, + StackId = model.StackId, + Threshold = model.Threshold, + Window = model.Window, + Cooldown = model.Cooldown, + Version = 1, + CreatedUtc = now, + UpdatedUtc = now + }; + + rule = await _ruleRepository.AddAsync(rule, o => o.Cache()); + await _ruleCache.InvalidateAsync(projectId); + + return Created(new Uri(Url.Link("GetRateNotificationRuleById", new { userId, projectId, ruleId = rule.Id })!, UriKind.RelativeOrAbsolute), MapToView(rule)); + } + + /// Get a specific rate notification rule. + [HttpGet("{ruleId:objectid}", Name = "GetRateNotificationRuleById")] + public async Task> GetByIdAsync(string userId, string projectId, string ruleId) + { + if (!CanManage(userId)) + return NotFound(); + + var rule = await GetRuleAndCheckAccessAsync(ruleId, userId, projectId); + if (rule is null) + return NotFound(); + + return Ok(MapToView(rule)); + } + + /// Update a rate notification rule. + [HttpPut("{ruleId:objectid}")] + [Consumes("application/json")] + public async Task> PutAsync( + string userId, + string projectId, + string ruleId, + [FromBody] UpdateRateNotificationRule model) + { + if (!CanManage(userId)) + return NotFound(); + + var project = await GetProjectAndCheckAccessAsync(projectId); + if (project is null) + return NotFound(); + + var rule = await GetRuleAndCheckAccessAsync(ruleId, userId, projectId); + if (rule is null) + return NotFound(); + + // Apply updates + if (model.Name is not null) + rule.Name = model.Name; + + if (model.Signal.HasValue) + rule.Signal = model.Signal.Value; + + if (model.Subject.HasValue) + rule.Subject = model.Subject.Value; + + if (model.Threshold.HasValue) + rule.Threshold = model.Threshold.Value; + + // StackId update + var newStackId = model.StackId; + var newSubject = rule.Subject; + + if (newSubject == RateNotificationSubject.Stack) + { + if (String.IsNullOrEmpty(newStackId)) + return ValidationProblem(detail: "StackId is required when Subject is Stack."); + + var stack = await _stackRepository.GetByIdAsync(newStackId, o => o.Cache()); + if (stack is null || !String.Equals(stack.ProjectId, projectId, StringComparison.Ordinal)) + return ValidationProblem(detail: "The specified StackId does not belong to this project."); + + rule.StackId = newStackId; + } + else + { + rule.StackId = null; + } + + if (model.Window.HasValue) + { + if (!ValidWindows.Contains(model.Window.Value)) + return ValidationProblem(detail: $"Window must be one of: {String.Join(", ", ValidWindows.Select(w => w.ToString()))}"); + rule.Window = model.Window.Value; + } + + if (model.Cooldown.HasValue) + rule.Cooldown = model.Cooldown.Value; + + if (rule.Cooldown < rule.Window) + return ValidationProblem(detail: "Cooldown must be greater than or equal to Window."); + + if (model.IsEnabled.HasValue) + { + var org = await _organizationRepository.GetByIdAsync(project.OrganizationId, o => o.Cache()); + rule.IsEnabled = model.IsEnabled.Value && (org?.HasPremiumFeatures ?? false); + } + + rule.Version++; + rule.UpdatedUtc = _timeProvider.GetUtcNow().UtcDateTime; + + await _ruleRepository.SaveAsync(rule, o => o.Cache()); + await _ruleCache.InvalidateAsync(projectId); + + return Ok(MapToView(rule)); + } + + /// Delete a rate notification rule. + [HttpDelete("{ruleId:objectid}")] + public async Task DeleteAsync(string userId, string projectId, string ruleId) + { + if (!CanManage(userId)) + return NotFound(); + + var rule = await GetRuleAndCheckAccessAsync(ruleId, userId, projectId); + if (rule is null) + return NotFound(); + + await _ruleRepository.RemoveAsync(rule); + await _ruleCache.InvalidateAsync(projectId); + + return NoContent(); + } + + /// Snooze a rate notification rule. + [HttpPost("{ruleId:objectid}/snooze")] + [Consumes("application/json")] + public async Task> SnoozeAsync( + string userId, + string projectId, + string ruleId, + [FromBody] SnoozeRateNotificationRuleRequest request) + { + if (!CanManage(userId)) + return NotFound(); + + var rule = await GetRuleAndCheckAccessAsync(ruleId, userId, projectId); + if (rule is null) + return NotFound(); + + var now = _timeProvider.GetUtcNow().UtcDateTime; + + if (request.UntilUtc.HasValue) + { + rule.SnoozedUntilUtc = request.UntilUtc.Value; + } + else if (request.DurationSeconds.HasValue) + { + rule.SnoozedUntilUtc = now.AddSeconds(request.DurationSeconds.Value); + } + else + { + return ValidationProblem(detail: "Either DurationSeconds or UntilUtc must be provided."); + } + + rule.Version++; + rule.UpdatedUtc = now; + await _ruleRepository.SaveAsync(rule, o => o.Cache()); + await _ruleCache.InvalidateAsync(projectId); + + return Ok(MapToView(rule)); + } + + /// Unsnooze a rate notification rule. Sets SnoozedUntilUtc = now to establish a fresh baseline. + [HttpPost("{ruleId:objectid}/unsnooze")] + public async Task> UnsnoozeAsync(string userId, string projectId, string ruleId) + { + if (!CanManage(userId)) + return NotFound(); + + var rule = await GetRuleAndCheckAccessAsync(ruleId, userId, projectId); + if (rule is null) + return NotFound(); + + // Set to now (NOT null) so the evaluator uses now as the effective window start — no back-alert + rule.SnoozedUntilUtc = _timeProvider.GetUtcNow().UtcDateTime; + rule.Version++; + rule.UpdatedUtc = _timeProvider.GetUtcNow().UtcDateTime; + await _ruleRepository.SaveAsync(rule, o => o.Cache()); + await _ruleCache.InvalidateAsync(projectId); + + return Ok(MapToView(rule)); + } + + // ---- Helpers ---- + + private bool CanManage(string userId) + { + // User can manage their own rules; global admins can manage any user's rules + return String.Equals(CurrentUser.Id, userId, StringComparison.Ordinal) || Request.IsGlobalAdmin(); + } + + private async Task GetProjectAndCheckAccessAsync(string projectId) + { + var project = await _projectRepository.GetByIdAsync(projectId, o => o.Cache()); + if (project is null || !CanAccessOrganization(project.OrganizationId)) + return null; + return project; + } + + private async Task GetRuleAndCheckAccessAsync(string ruleId, string userId, string projectId) + { + var rule = await _ruleRepository.GetByIdAsync(ruleId); + if (rule is null) + return null; + + // Rule must belong to the specified user+project + if (!String.Equals(rule.UserId, userId, StringComparison.Ordinal)) + return null; + + if (!String.Equals(rule.ProjectId, projectId, StringComparison.Ordinal)) + return null; + + // Current user must be able to manage this rule + if (!CanManage(userId)) + return null; + + return rule; + } + + private ViewRateNotificationRule MapToView(RateNotificationRule rule) + { + var now = _timeProvider.GetUtcNow().UtcDateTime; + return new ViewRateNotificationRule + { + Id = rule.Id, + OrganizationId = rule.OrganizationId, + ProjectId = rule.ProjectId, + UserId = rule.UserId, + Version = rule.Version, + Name = rule.Name, + IsEnabled = rule.IsEnabled, + Signal = rule.Signal, + Subject = rule.Subject, + StackId = rule.StackId, + Threshold = rule.Threshold, + Window = rule.Window, + Cooldown = rule.Cooldown, + SnoozedUntilUtc = rule.SnoozedUntilUtc, + IsSnoozed = rule.SnoozedUntilUtc.HasValue && rule.SnoozedUntilUtc.Value > now, + LastFiredUtc = rule.LastFiredUtc, + CreatedUtc = rule.CreatedUtc, + UpdatedUtc = rule.UpdatedUtc + }; + } +} diff --git a/src/Exceptionless.Web/Models/RateNotification/RateNotificationRuleModels.cs b/src/Exceptionless.Web/Models/RateNotification/RateNotificationRuleModels.cs new file mode 100644 index 0000000000..0ff72cc38d --- /dev/null +++ b/src/Exceptionless.Web/Models/RateNotification/RateNotificationRuleModels.cs @@ -0,0 +1,82 @@ +using System.ComponentModel.DataAnnotations; +using Exceptionless.Core.Attributes; +using Exceptionless.Core.Models; + +namespace Exceptionless.Web.Models; + +public record NewRateNotificationRule +{ + [Required] + [MaxLength(100)] + public string Name { get; init; } = null!; + + public RateNotificationSignal Signal { get; init; } + + public RateNotificationSubject Subject { get; init; } + + [ObjectId] + public string? StackId { get; init; } + + [Range(1, int.MaxValue)] + public int Threshold { get; init; } = 10; + + public TimeSpan Window { get; init; } = TimeSpan.FromHours(1); + + public TimeSpan Cooldown { get; init; } = TimeSpan.FromHours(1); + + public bool IsEnabled { get; init; } = true; +} + +public record UpdateRateNotificationRule +{ + [MaxLength(100)] + public string? Name { get; init; } + + public RateNotificationSignal? Signal { get; init; } + + public RateNotificationSubject? Subject { get; init; } + + [ObjectId] + public string? StackId { get; init; } + + [Range(1, int.MaxValue)] + public int? Threshold { get; init; } + + public TimeSpan? Window { get; init; } + + public TimeSpan? Cooldown { get; init; } + + public bool? IsEnabled { get; init; } +} + +public record ViewRateNotificationRule +{ + public string Id { get; init; } = null!; + public string OrganizationId { get; init; } = null!; + public string ProjectId { get; init; } = null!; + public string UserId { get; init; } = null!; + public int Version { get; init; } + public string Name { get; init; } = null!; + public bool IsEnabled { get; init; } + public RateNotificationSignal Signal { get; init; } + public RateNotificationSubject Subject { get; init; } + public string? StackId { get; init; } + public int Threshold { get; init; } + public TimeSpan Window { get; init; } + public TimeSpan Cooldown { get; init; } + public DateTime? SnoozedUntilUtc { get; init; } + public bool IsSnoozed { get; init; } + public DateTime? LastFiredUtc { get; init; } + public DateTime CreatedUtc { get; init; } + public DateTime UpdatedUtc { get; init; } +} + +public record SnoozeRateNotificationRuleRequest +{ + /// Snooze duration in seconds. Mutually exclusive with UntilUtc. + [Range(1, int.MaxValue)] + public int? DurationSeconds { get; init; } + + /// Snooze until this UTC timestamp. Mutually exclusive with DurationSeconds. + public DateTime? UntilUtc { get; init; } +} From 1de73edcc99c57649b3815e17530fb32db75dc9a Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sun, 31 May 2026 07:42:26 -0500 Subject: [PATCH 08/14] feat: register rate notification services, jobs, and queues in DI - IRateNotificationRuleRepository, RateCounterService, RateNotificationRuleCache - RateNotificationsJob (queue delivery job) - RateNotificationEvaluatorJob (minutely distributed cron job) - RateNotification queue for in-process, Azure, Redis, SQS providers --- src/Exceptionless.Core/Bootstrapper.cs | 6 ++++++ src/Exceptionless.Insulation/Bootstrapper.cs | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/src/Exceptionless.Core/Bootstrapper.cs b/src/Exceptionless.Core/Bootstrapper.cs index 334f2b07ae..cebbaf3b6e 100644 --- a/src/Exceptionless.Core/Bootstrapper.cs +++ b/src/Exceptionless.Core/Bootstrapper.cs @@ -128,6 +128,7 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO services.AddSingleton(s => CreateQueue(s)); services.AddSingleton(s => CreateQueue(s)); services.AddSingleton(s => CreateQueue(s, TimeSpan.FromHours(1))); + services.AddSingleton(s => CreateQueue(s)); services.TryAddEnumerable(ServiceDescriptor.Singleton, WorkItemDuplicateDetectionQueueBehavior>()); @@ -163,6 +164,7 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); @@ -205,6 +207,8 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO services.AddSingleton(); services.AddStartupAction(); services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); @@ -275,12 +279,14 @@ public static void AddHostedJobs(IServiceCollection services, ILoggerFactory log services.AddJob(o => o.WaitForStartupActions()); services.AddJob(o => o.WaitForStartupActions()); services.AddJob(o => o.WaitForStartupActions()); + services.AddJob(o => o.WaitForStartupActions()); services.AddJob(o => o.WaitForStartupActions()); services.AddJob(o => o.WaitForStartupActions()); services.AddJob(o => o.WaitForStartupActions()); services.AddJob(o => o.WaitForStartupActions()); services.AddDistributedCronJob(Cron.Minutely()); + services.AddDistributedCronJob(Cron.Minutely()); services.AddDistributedCronJob("30 */4 * * *"); services.AddDistributedCronJob("45 */8 * * *"); services.AddDistributedCronJob(Cron.Daily(1)); diff --git a/src/Exceptionless.Insulation/Bootstrapper.cs b/src/Exceptionless.Insulation/Bootstrapper.cs index 6174ca2cc3..05ed57b454 100644 --- a/src/Exceptionless.Insulation/Bootstrapper.cs +++ b/src/Exceptionless.Insulation/Bootstrapper.cs @@ -89,6 +89,7 @@ private static IHealthChecksBuilder RegisterHealthChecks(IServiceCollection serv .AddAutoNamedCheck>("EventUserDescriptions", "AllJobs") .AddAutoNamedCheck>("EventNotifications", "AllJobs") .AddAutoNamedCheck>("WebHooks", "AllJobs") + .AddAutoNamedCheck>("RateNotifications", "AllJobs") .AddAutoNamedCheck>("AllJobs") .AddAutoNamedCheck>("WorkItem", "AllJobs") @@ -159,6 +160,7 @@ private static void RegisterQueue(IServiceCollection container, QueueOptions opt container.ReplaceSingleton(s => CreateAzureStorageQueue(s, options)); container.ReplaceSingleton(s => CreateAzureStorageQueue(s, options)); container.ReplaceSingleton(s => CreateAzureStorageQueue(s, options)); + container.ReplaceSingleton(s => CreateAzureStorageQueue(s, options)); container.ReplaceSingleton(s => CreateAzureStorageQueue(s, options)); container.ReplaceSingleton(s => CreateAzureStorageQueue(s, options, workItemTimeout: TimeSpan.FromHours(1))); } @@ -168,6 +170,7 @@ private static void RegisterQueue(IServiceCollection container, QueueOptions opt container.ReplaceSingleton(s => CreateRedisQueue(s, options, runMaintenanceTasks)); container.ReplaceSingleton(s => CreateRedisQueue(s, options, runMaintenanceTasks)); container.ReplaceSingleton(s => CreateRedisQueue(s, options, runMaintenanceTasks)); + container.ReplaceSingleton(s => CreateRedisQueue(s, options, runMaintenanceTasks)); container.ReplaceSingleton(s => CreateRedisQueue(s, options, runMaintenanceTasks)); container.ReplaceSingleton(s => CreateRedisQueue(s, options, runMaintenanceTasks, workItemTimeout: TimeSpan.FromHours(1))); } @@ -177,6 +180,7 @@ private static void RegisterQueue(IServiceCollection container, QueueOptions opt container.ReplaceSingleton(s => CreateSQSQueue(s, options)); container.ReplaceSingleton(s => CreateSQSQueue(s, options)); container.ReplaceSingleton(s => CreateSQSQueue(s, options)); + container.ReplaceSingleton(s => CreateSQSQueue(s, options)); container.ReplaceSingleton(s => CreateSQSQueue(s, options)); container.ReplaceSingleton(s => CreateSQSQueue(s, options, workItemTimeout: TimeSpan.FromHours(1))); } From 09c4b3f57b41f644bc72ab088cb0e0f6ab469dc9 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sun, 31 May 2026 07:42:34 -0500 Subject: [PATCH 09/14] test: add rate notification tests including snooze back-alert regression RateCounterServiceTests: 12 unit tests (no ES), including the critical regression: SumBucketsAsync_WithSnoozeFix_IgnoresTrafficDuringSnooze proves that events counted during a snooze period are excluded. RateNotificationEvaluatorJobTests: integration-level job tests including the back-alert scenario at job level (1 notification for Rule B only). RateNotificationRuleControllerTests: CRUD + snooze endpoint tests. --- .../RateNotificationRuleControllerTests.cs | 405 ++++++++++++++++++ .../Jobs/RateNotificationEvaluatorJobTests.cs | 221 ++++++++++ .../Services/RateCounterServiceTests.cs | 274 ++++++++++++ 3 files changed, 900 insertions(+) create mode 100644 tests/Exceptionless.Tests/Controllers/RateNotificationRuleControllerTests.cs create mode 100644 tests/Exceptionless.Tests/Jobs/RateNotificationEvaluatorJobTests.cs create mode 100644 tests/Exceptionless.Tests/Services/RateCounterServiceTests.cs diff --git a/tests/Exceptionless.Tests/Controllers/RateNotificationRuleControllerTests.cs b/tests/Exceptionless.Tests/Controllers/RateNotificationRuleControllerTests.cs new file mode 100644 index 0000000000..bdcf4a5a57 --- /dev/null +++ b/tests/Exceptionless.Tests/Controllers/RateNotificationRuleControllerTests.cs @@ -0,0 +1,405 @@ +using Exceptionless.Core.Models; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Utility; +using Exceptionless.Tests.Extensions; +using Exceptionless.Web.Models; +using Xunit; + +namespace Exceptionless.Tests.Controllers; + +public sealed class RateNotificationRuleControllerTests : IntegrationTestsBase +{ + private readonly IRateNotificationRuleRepository _ruleRepository; + private readonly IUserRepository _userRepository; + + public RateNotificationRuleControllerTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) + { + _ruleRepository = GetService(); + _userRepository = GetService(); + } + + protected override async Task ResetDataAsync() + { + await base.ResetDataAsync(); + var service = GetService(); + await service.CreateDataAsync(); + } + + // ---- Helper: get current user via /me endpoint ---- + private async Task GetTestOrganizationUserAsync() + { + var user = await SendRequestAsAsync(r => r + .AsTestOrganizationUser() + .AppendPath("users/me") + .StatusCodeShouldBeOk() + ); + Assert.NotNull(user); + return user; + } + + private async Task GetGlobalAdminUserAsync() + { + var user = await SendRequestAsAsync(r => r + .AsGlobalAdminUser() + .AppendPath("users/me") + .StatusCodeShouldBeOk() + ); + Assert.NotNull(user); + return user; + } + + private string RuleUrl(string userId, string projectId) => + $"users/{userId}/projects/{projectId}/rate-notifications"; + + private string RuleUrl(string userId, string projectId, string ruleId) => + $"users/{userId}/projects/{projectId}/rate-notifications/{ruleId}"; + + // ---- CRUD tests ---- + + [Fact] + public async Task GetAsync_AsOwnUser_ReturnsList() + { + var user = await GetTestOrganizationUserAsync(); + + var results = await SendRequestAsAsync>(r => r + .AsTestOrganizationUser() + .AppendPath(RuleUrl(user.Id, SampleDataService.TEST_PROJECT_ID)) + .StatusCodeShouldBeOk() + ); + + Assert.NotNull(results); + } + + [Fact] + public async Task GetAsync_AsDifferentUser_ReturnsNotFound() + { + // Using org user ID to try to access another user's rules + var adminUser = await GetGlobalAdminUserAsync(); + + await SendRequestAsync(r => r + .AsTestOrganizationUser() // Org user + .AppendPath(RuleUrl(adminUser.Id, SampleDataService.TEST_PROJECT_ID)) // accessing admin's rules + .StatusCodeShouldBeNotFound() + ); + } + + [Fact] + public async Task GetAsync_AsGlobalAdmin_CanAccessAnyUsersRules() + { + var orgUser = await GetTestOrganizationUserAsync(); + + var results = await SendRequestAsAsync>(r => r + .AsGlobalAdminUser() // admin accessing org user's rules + .AppendPath(RuleUrl(orgUser.Id, SampleDataService.TEST_PROJECT_ID)) + .StatusCodeShouldBeOk() + ); + + Assert.NotNull(results); + } + + [Fact] + public async Task PostAsync_ValidRule_CreatesRule() + { + var user = await GetTestOrganizationUserAsync(); + + var newRule = new NewRateNotificationRule + { + Name = "Error spike", + Signal = RateNotificationSignal.Errors, + Subject = RateNotificationSubject.Project, + Threshold = 10, + Window = TimeSpan.FromMinutes(5), + Cooldown = TimeSpan.FromMinutes(30), + IsEnabled = true + }; + + var created = await SendRequestAsAsync(r => r + .Post() + .AsTestOrganizationUser() + .AppendPath(RuleUrl(user.Id, SampleDataService.TEST_PROJECT_ID)) + .Content(newRule) + .StatusCodeShouldBeCreated() + ); + + Assert.NotNull(created); + Assert.Equal("Error spike", created.Name); + Assert.Equal(RateNotificationSignal.Errors, created.Signal); + Assert.Equal(10, created.Threshold); + } + + [Fact] + public async Task PostAsync_InvalidWindow_Returns422() + { + var user = await GetTestOrganizationUserAsync(); + + var newRule = new NewRateNotificationRule + { + Name = "Bad window", + Signal = RateNotificationSignal.Errors, + Subject = RateNotificationSubject.Project, + Threshold = 10, + Window = TimeSpan.FromMinutes(7), // invalid — not in allowed list + Cooldown = TimeSpan.FromMinutes(30), + IsEnabled = true + }; + + await SendRequestAsync(r => r + .Post() + .AsTestOrganizationUser() + .AppendPath(RuleUrl(user.Id, SampleDataService.TEST_PROJECT_ID)) + .Content(newRule) + .StatusCodeShouldBeUnprocessableEntity() + ); + } + + [Fact] + public async Task PostAsync_CooldownLessThanWindow_Returns422() + { + var user = await GetTestOrganizationUserAsync(); + + var newRule = new NewRateNotificationRule + { + Name = "Bad cooldown", + Signal = RateNotificationSignal.Errors, + Subject = RateNotificationSubject.Project, + Threshold = 10, + Window = TimeSpan.FromMinutes(30), + Cooldown = TimeSpan.FromMinutes(5), // less than window + IsEnabled = true + }; + + await SendRequestAsync(r => r + .Post() + .AsTestOrganizationUser() + .AppendPath(RuleUrl(user.Id, SampleDataService.TEST_PROJECT_ID)) + .Content(newRule) + .StatusCodeShouldBeUnprocessableEntity() + ); + } + + [Fact] + public async Task PostAsync_StackSubjectWithoutStackId_Returns422() + { + var user = await GetTestOrganizationUserAsync(); + + var newRule = new NewRateNotificationRule + { + Name = "Stack rule no id", + Signal = RateNotificationSignal.Errors, + Subject = RateNotificationSubject.Stack, // stack subject + StackId = null, // missing StackId + Threshold = 10, + Window = TimeSpan.FromMinutes(5), + Cooldown = TimeSpan.FromMinutes(30), + IsEnabled = true + }; + + await SendRequestAsync(r => r + .Post() + .AsTestOrganizationUser() + .AppendPath(RuleUrl(user.Id, SampleDataService.TEST_PROJECT_ID)) + .Content(newRule) + .StatusCodeShouldBeUnprocessableEntity() + ); + } + + [Fact] + public async Task GetByIdAsync_ExistingRule_ReturnsRule() + { + var user = await GetTestOrganizationUserAsync(); + + // Create first + var created = await SendRequestAsAsync(r => r + .Post() + .AsTestOrganizationUser() + .AppendPath(RuleUrl(user.Id, SampleDataService.TEST_PROJECT_ID)) + .Content(new NewRateNotificationRule + { + Name = "Get-by-id test", + Signal = RateNotificationSignal.Errors, + Subject = RateNotificationSubject.Project, + Threshold = 5, + Window = TimeSpan.FromMinutes(5), + Cooldown = TimeSpan.FromMinutes(30) + }) + .StatusCodeShouldBeCreated() + ); + + Assert.NotNull(created); + + // Fetch by ID + var fetched = await SendRequestAsAsync(r => r + .AsTestOrganizationUser() + .AppendPath(RuleUrl(user.Id, SampleDataService.TEST_PROJECT_ID, created.Id)) + .StatusCodeShouldBeOk() + ); + + Assert.NotNull(fetched); + Assert.Equal(created.Id, fetched.Id); + Assert.Equal("Get-by-id test", fetched.Name); + } + + [Fact] + public async Task DeleteAsync_ExistingRule_RemovesRule() + { + var user = await GetTestOrganizationUserAsync(); + + // Create + var created = await SendRequestAsAsync(r => r + .Post() + .AsTestOrganizationUser() + .AppendPath(RuleUrl(user.Id, SampleDataService.TEST_PROJECT_ID)) + .Content(new NewRateNotificationRule + { + Name = "Delete me", + Signal = RateNotificationSignal.Errors, + Subject = RateNotificationSubject.Project, + Threshold = 5, + Window = TimeSpan.FromMinutes(5), + Cooldown = TimeSpan.FromMinutes(30) + }) + .StatusCodeShouldBeCreated() + ); + + Assert.NotNull(created); + + // Delete + await SendRequestAsync(r => r + .Delete() + .AsTestOrganizationUser() + .AppendPath(RuleUrl(user.Id, SampleDataService.TEST_PROJECT_ID, created.Id)) + .StatusCodeShouldBeNoContent() + ); + + // Confirm deleted + var rule = await _ruleRepository.GetByIdAsync(created.Id); + Assert.Null(rule); + } + + [Fact] + public async Task SnoozeAsync_ValidDuration_SetsSnooze() + { + var user = await GetTestOrganizationUserAsync(); + + // Create a rule + var created = await SendRequestAsAsync(r => r + .Post() + .AsTestOrganizationUser() + .AppendPath(RuleUrl(user.Id, SampleDataService.TEST_PROJECT_ID)) + .Content(new NewRateNotificationRule + { + Name = "Snooze test", + Signal = RateNotificationSignal.Errors, + Subject = RateNotificationSubject.Project, + Threshold = 5, + Window = TimeSpan.FromMinutes(5), + Cooldown = TimeSpan.FromMinutes(30) + }) + .StatusCodeShouldBeCreated() + ); + + Assert.NotNull(created); + + // Snooze for 1 hour + var snoozed = await SendRequestAsAsync(r => r + .Post() + .AsTestOrganizationUser() + .AppendPath($"{RuleUrl(user.Id, SampleDataService.TEST_PROJECT_ID, created.Id)}/snooze") + .Content(new SnoozeRateNotificationRuleRequest { DurationSeconds = 3600 }) + .StatusCodeShouldBeOk() + ); + + Assert.NotNull(snoozed); + Assert.True(snoozed.IsSnoozed, "Rule should be snoozed after snooze request."); + Assert.NotNull(snoozed.SnoozedUntilUtc); + } + + [Fact] + public async Task UnsnoozeAsync_SnoozedRule_SetsSnoozedUntilToNow() + { + var user = await GetTestOrganizationUserAsync(); + + // Create + snooze + var created = await SendRequestAsAsync(r => r + .Post() + .AsTestOrganizationUser() + .AppendPath(RuleUrl(user.Id, SampleDataService.TEST_PROJECT_ID)) + .Content(new NewRateNotificationRule + { + Name = "Unsnooze test", + Signal = RateNotificationSignal.Errors, + Subject = RateNotificationSubject.Project, + Threshold = 5, + Window = TimeSpan.FromMinutes(5), + Cooldown = TimeSpan.FromMinutes(30) + }) + .StatusCodeShouldBeCreated() + ); + + Assert.NotNull(created); + + await SendRequestAsync(r => r + .Post() + .AsTestOrganizationUser() + .AppendPath($"{RuleUrl(user.Id, SampleDataService.TEST_PROJECT_ID, created.Id)}/snooze") + .Content(new SnoozeRateNotificationRuleRequest { DurationSeconds = 3600 }) + .StatusCodeShouldBeOk() + ); + + // Unsnooze — sets SnoozedUntilUtc = now, so IsSnoozed = false + var unsnoozed = await SendRequestAsAsync(r => r + .Post() + .AsTestOrganizationUser() + .AppendPath($"{RuleUrl(user.Id, SampleDataService.TEST_PROJECT_ID, created.Id)}/unsnooze") + .StatusCodeShouldBeOk() + ); + + Assert.NotNull(unsnoozed); + Assert.False(unsnoozed.IsSnoozed, "Rule should NOT be actively snoozed after unsnooze (SnoozedUntilUtc = now)."); + // SnoozedUntilUtc is still set (to now) — not null. This is the fresh baseline mechanism. + Assert.NotNull(unsnoozed.SnoozedUntilUtc); + } + + [Fact] + public async Task PostAsync_ExceedsMaxRulesPerUser_Returns422() + { + var user = await GetTestOrganizationUserAsync(); + + // Create 20 rules (the maximum) + for (int i = 0; i < 20; i++) + { + await SendRequestAsync(r => r + .Post() + .AsTestOrganizationUser() + .AppendPath(RuleUrl(user.Id, SampleDataService.TEST_PROJECT_ID)) + .Content(new NewRateNotificationRule + { + Name = $"Rule {i + 1}", + Signal = RateNotificationSignal.Errors, + Subject = RateNotificationSubject.Project, + Threshold = 5, + Window = TimeSpan.FromMinutes(5), + Cooldown = TimeSpan.FromMinutes(30) + }) + .StatusCodeShouldBeCreated() + ); + } + + // 21st rule should fail + await SendRequestAsync(r => r + .Post() + .AsTestOrganizationUser() + .AppendPath(RuleUrl(user.Id, SampleDataService.TEST_PROJECT_ID)) + .Content(new NewRateNotificationRule + { + Name = "Rule 21", + Signal = RateNotificationSignal.Errors, + Subject = RateNotificationSubject.Project, + Threshold = 5, + Window = TimeSpan.FromMinutes(5), + Cooldown = TimeSpan.FromMinutes(30) + }) + .StatusCodeShouldBeUnprocessableEntity() + ); + } +} diff --git a/tests/Exceptionless.Tests/Jobs/RateNotificationEvaluatorJobTests.cs b/tests/Exceptionless.Tests/Jobs/RateNotificationEvaluatorJobTests.cs new file mode 100644 index 0000000000..641702c01f --- /dev/null +++ b/tests/Exceptionless.Tests/Jobs/RateNotificationEvaluatorJobTests.cs @@ -0,0 +1,221 @@ +using Exceptionless.Core.Jobs; +using Exceptionless.Core.Models; +using Exceptionless.Core.Queues.Models; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Services; +using Exceptionless.Core.Utility; +using Foundatio.Queues; +using Xunit; + +namespace Exceptionless.Tests.Jobs; + +public class RateNotificationEvaluatorJobTests : IntegrationTestsBase +{ + private readonly RateNotificationEvaluatorJob _job; + private readonly RateCounterService _counterService; + private readonly IRateNotificationRuleRepository _ruleRepository; + private readonly IOrganizationRepository _orgRepository; + private readonly IProjectRepository _projectRepository; + private readonly IQueue _notificationQueue; + + public RateNotificationEvaluatorJobTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) + { + _job = GetService(); + _counterService = GetService(); + _ruleRepository = GetService(); + _orgRepository = GetService(); + _projectRepository = GetService(); + _notificationQueue = GetService>(); + } + + protected override async Task ResetDataAsync() + { + await base.ResetDataAsync(); + var service = GetService(); + await service.CreateDataAsync(); + } + + private RateNotificationRule BuildRule(string? projectId = null, RateNotificationSignal signal = RateNotificationSignal.Errors, int threshold = 10, string? window = null, string? cooldown = null) + { + var now = TimeProvider.GetUtcNow().UtcDateTime; + return new RateNotificationRule + { + OrganizationId = SampleDataService.TEST_ORG_ID, + ProjectId = projectId ?? SampleDataService.TEST_PROJECT_ID, + UserId = "507f1f77bcf86cd799439011", + Name = "Test Rule", + IsEnabled = true, + Signal = signal, + Subject = RateNotificationSubject.Project, + Threshold = threshold, + Window = window is not null ? TimeSpan.Parse(window) : TimeSpan.FromMinutes(5), + Cooldown = cooldown is not null ? TimeSpan.Parse(cooldown) : TimeSpan.FromMinutes(10), + Version = 1, + CreatedUtc = now, + UpdatedUtc = now + }; + } + + private string BuildCounterKey(RateNotificationRule rule) => + $"project:{rule.ProjectId}:signal:{rule.Signal}"; + + [Fact] + public async Task RunAsync_WhenThresholdCrossed_EnqueuesNotification() + { + var ct = TestContext.Current.CancellationToken; + // Arrange: start at event time (2 min before "now"), then advance forward + var now = new DateTime(2024, 6, 1, 12, 0, 0, DateTimeKind.Utc); + TimeProvider.SetUtcNow(now.AddMinutes(-2)); + + var rule = await _ruleRepository.AddAsync(BuildRule(threshold: 5)); + string counterKey = BuildCounterKey(rule); + + // Simulate 10 events within the 5-minute window + for (int i = 0; i < 10; i++) + await _counterService.IncrementAsync(counterKey, ct); + + TimeProvider.Advance(TimeSpan.FromMinutes(2)); + + // Act + await _job.RunAsync(ct); + + // Assert — notification enqueued + var stats = await _notificationQueue.GetQueueStatsAsync(); + Assert.True(stats.Enqueued > 0, "Expected a RateNotification to be enqueued when threshold is crossed."); + } + + [Fact] + public async Task RunAsync_WhenBelowThreshold_DoesNotEnqueue() + { + var ct = TestContext.Current.CancellationToken; + // Arrange: start at event time, advance to "now" + var now = new DateTime(2024, 6, 1, 12, 0, 0, DateTimeKind.Utc); + TimeProvider.SetUtcNow(now.AddMinutes(-2)); + + var rule = await _ruleRepository.AddAsync(BuildRule(threshold: 50)); + string counterKey = BuildCounterKey(rule); + + // Only 5 events — well below threshold of 50 + for (int i = 0; i < 5; i++) + await _counterService.IncrementAsync(counterKey, ct); + + TimeProvider.Advance(TimeSpan.FromMinutes(2)); + long queueBefore = (await _notificationQueue.GetQueueStatsAsync()).Enqueued; + + // Act + await _job.RunAsync(ct); + + // Assert — no new notification + long queueAfter = (await _notificationQueue.GetQueueStatsAsync()).Enqueued; + Assert.Equal(queueBefore, queueAfter); + } + + [Fact] + public async Task RunAsync_WhenOnCooldown_DoesNotEnqueueAgain() + { + var ct = TestContext.Current.CancellationToken; + // Arrange: start at event time, advance to "now", then set cooldown + var now = new DateTime(2024, 6, 1, 12, 0, 0, DateTimeKind.Utc); + TimeProvider.SetUtcNow(now.AddMinutes(-2)); + + var rule = await _ruleRepository.AddAsync(BuildRule(threshold: 5)); + string counterKey = BuildCounterKey(rule); + string subjectKey = $"project:{rule.ProjectId}"; + + // Add enough events to cross threshold + for (int i = 0; i < 10; i++) + await _counterService.IncrementAsync(counterKey, ct); + + TimeProvider.Advance(TimeSpan.FromMinutes(2)); + + // Put rule on cooldown (at "now") + await _counterService.SetCooldownAsync(rule.Id, subjectKey, TimeSpan.FromHours(1), ct); + + long queueBefore = (await _notificationQueue.GetQueueStatsAsync()).Enqueued; + + // Act + await _job.RunAsync(ct); + + // Assert — still on cooldown, nothing extra enqueued + long queueAfter = (await _notificationQueue.GetQueueStatsAsync()).Enqueued; + Assert.Equal(queueBefore, queueAfter); + } + + [Fact] + public async Task RunAsync_WhenActivelySnoozed_SkipsEvaluation() + { + var ct = TestContext.Current.CancellationToken; + // Arrange: start at event time, advance to "now" + var now = new DateTime(2024, 6, 1, 12, 0, 0, DateTimeKind.Utc); + TimeProvider.SetUtcNow(now.AddMinutes(-2)); + + var ruleData = BuildRule(threshold: 5); + ruleData.SnoozedUntilUtc = now.AddHours(2); // snoozed for 2 more hours + var rule = await _ruleRepository.AddAsync(ruleData); + string counterKey = BuildCounterKey(rule); + + // Add events above threshold + for (int i = 0; i < 20; i++) + await _counterService.IncrementAsync(counterKey, ct); + + TimeProvider.Advance(TimeSpan.FromMinutes(2)); + long queueBefore = (await _notificationQueue.GetQueueStatsAsync()).Enqueued; + + // Act + await _job.RunAsync(ct); + + // Assert — snoozed rule does not fire + long queueAfter = (await _notificationQueue.GetQueueStatsAsync()).Enqueued; + Assert.Equal(queueBefore, queueAfter); + } + + /// + /// CRITICAL REGRESSION: Snooze back-alert prevention at job level. + /// + /// Setup: Rule A was snoozed until T-2min (snooze recently expired). + /// 15 events were counted at T-3min (during the snooze period). + /// Rule B (identical but not snoozed) gets the same traffic. + /// + /// Expected: + /// Rule B fires: full 5-min window sees 15 events at or above threshold. + /// Rule A does NOT fire: effective window is [T-2min, now], so 0 events (prevented). + /// + [Fact] + public async Task RunAsync_SnoozeBackAlert_SnoozedRuleIgnoresTrafficDuringSnoozeWindow() + { + var ct = TestContext.Current.CancellationToken; + // Arrange: start at T-3min (events arrive during snooze), then advance to "now" + var now = new DateTime(2024, 6, 1, 12, 0, 0, DateTimeKind.Utc); + TimeProvider.SetUtcNow(now.AddMinutes(-3)); + + // Rule A: snoozed until T-2min (snooze has just expired at "now") + var ruleAData = BuildRule(threshold: 10); + ruleAData.SnoozedUntilUtc = now.AddMinutes(-2); + var ruleA = await _ruleRepository.AddAsync(ruleAData); + + // Rule B: not snoozed — should fire normally + var ruleB = await _ruleRepository.AddAsync(BuildRule(threshold: 10)); + + // Both rules watch the same signal/counter key + string counterKey = BuildCounterKey(ruleA); + Assert.Equal(counterKey, BuildCounterKey(ruleB)); + + // Simulate 15 events at T-3min (during Rule A's snooze window) + for (int i = 0; i < 15; i++) + await _counterService.IncrementAsync(counterKey, ct); + + TimeProvider.Advance(TimeSpan.FromMinutes(3)); + long queueBefore = (await _notificationQueue.GetQueueStatsAsync()).Enqueued; + + // Act + await _job.RunAsync(ct); + + // Assert + long queueAfter = (await _notificationQueue.GetQueueStatsAsync()).Enqueued; + long newNotifications = queueAfter - queueBefore; + + // Rule B should have fired (1 notification), Rule A should NOT have fired + // So exactly 1 notification should be enqueued. + Assert.Equal(1, newNotifications); + } +} diff --git a/tests/Exceptionless.Tests/Services/RateCounterServiceTests.cs b/tests/Exceptionless.Tests/Services/RateCounterServiceTests.cs new file mode 100644 index 0000000000..8d8a53bbbf --- /dev/null +++ b/tests/Exceptionless.Tests/Services/RateCounterServiceTests.cs @@ -0,0 +1,274 @@ +using Exceptionless.Core.Services; +using Exceptionless.Tests.Utility; +using Foundatio.Caching; +using Microsoft.Extensions.Logging.Abstractions; +using Xunit; + +namespace Exceptionless.Tests.Services; + +/// +/// Unit tests for RateCounterService, including the critical snooze back-alert regression test. +/// Uses an in-memory cache client and ProxyTimeProvider — no Elasticsearch required. +/// +public class RateCounterServiceTests +{ + private const string CounterKey = "project:P1:signal:Errors"; + private const string RuleId = "rule-001"; + private const string SubjectKey = "project:P1"; + + private static (RateCounterService service, ProxyTimeProvider timeProvider, InMemoryCacheClient cache) Create() + { + var timeProvider = new ProxyTimeProvider(); + var cache = new InMemoryCacheClient(new InMemoryCacheClientOptions + { + LoggerFactory = NullLoggerFactory.Instance + }); + var service = new RateCounterService(cache, timeProvider, NullLoggerFactory.Instance); + return (service, timeProvider, cache); + } + + // ------------------------------------------------------------------------- + // CRITICAL REGRESSION TEST: Snooze back-alert prevention + // ------------------------------------------------------------------------- + + /// + /// Verifies that when a rule was snoozed and the snooze recently expired, the + /// evaluator's SumBucketsAsync call uses max(windowStart, snoozedUntil) as the + /// effective window start — so traffic counted during the snooze window does NOT + /// trigger the rule. + /// + /// Without the snooze fix: Rule A would see the 15 events counted 3 minutes ago + /// (inside the 5-minute window) and fire incorrectly. + /// + /// With the snooze fix: Rule A uses snoozedUntilUtc as the effective lower boundary, + /// so it only counts events after the snooze expired (0 events) and does NOT fire. + /// + [Fact] + public async Task SumBucketsAsync_WithSnoozeFix_IgnoresTrafficDuringSnooze() + { + var ct = TestContext.Current.CancellationToken; + var (service, timeProvider, _) = Create(); + + // Start at T-3min (the time the snoozed events occurred) + var baseTime = new DateTime(2024, 1, 15, 12, 0, 0, DateTimeKind.Utc); + var eventTime = baseTime.AddMinutes(-3); + timeProvider.SetUtcNow(eventTime); + + for (int i = 0; i < 15; i++) + await service.IncrementAsync(CounterKey, ct); + + // Advance to "now" (T+0) + timeProvider.Advance(TimeSpan.FromMinutes(3)); + + var now = baseTime; + var windowDuration = TimeSpan.FromMinutes(5); + var windowStartUtc = now.Subtract(windowDuration); // T-5min + + // Rule A: snoozed until T-2min (snooze recently expired) + var snoozedUntilUtc = now.AddMinutes(-2); // T-2min + + // Without snooze fix: sum from T-5min to now = 15 events (fires incorrectly) + long withoutFix = await service.SumBucketsAsync(CounterKey, windowStartUtc, now, ct); + + // With snooze fix: effective window start = max(T-5min, T-2min) = T-2min + // Events were counted at T-3min which is BEFORE T-2min, so they're excluded + var effectiveWindowStart = snoozedUntilUtc > windowStartUtc ? snoozedUntilUtc : windowStartUtc; + long withFix = await service.SumBucketsAsync(CounterKey, effectiveWindowStart, now, ct); + + // Without fix: 15 events (would fire incorrectly) + Assert.Equal(15, withoutFix); + + // With fix: 0 events (correctly prevents back-alert) + Assert.Equal(0, withFix); + } + + /// + /// Verifies that Rule B (not snoozed) DOES fire for the same traffic. + /// Companion test to the snooze regression — proves the snooze fix is selective. + /// + [Fact] + public async Task SumBucketsAsync_NonSnoozedRule_CountsAllTrafficInWindow() + { + var ct = TestContext.Current.CancellationToken; + var (service, timeProvider, _) = Create(); + + // Start at T-3min, then advance forward + var baseTime = new DateTime(2024, 1, 15, 12, 0, 0, DateTimeKind.Utc); + var eventTime = baseTime.AddMinutes(-3); + timeProvider.SetUtcNow(eventTime); + + for (int i = 0; i < 15; i++) + await service.IncrementAsync(CounterKey, ct); + + timeProvider.Advance(TimeSpan.FromMinutes(3)); + + var now = baseTime; + var windowStartUtc = now.AddMinutes(-5); // full 5min window + + // Non-snoozed rule: uses full window + long count = await service.SumBucketsAsync(CounterKey, windowStartUtc, now, ct); + + // All 15 events are in the 5-minute window => threshold=10 => fires + Assert.Equal(15, count); + } + + // ------------------------------------------------------------------------- + // Bucket increment and sum tests + // ------------------------------------------------------------------------- + + [Fact] + public async Task IncrementAsync_SingleIncrement_CreatesCountBucket() + { + var ct = TestContext.Current.CancellationToken; + var (service, timeProvider, _) = Create(); + var now = new DateTime(2024, 1, 15, 10, 0, 0, DateTimeKind.Utc); + timeProvider.SetUtcNow(now); + + await service.IncrementAsync(CounterKey, ct); + + long count = await service.SumBucketsAsync(CounterKey, now.AddMinutes(-1), now, ct); + Assert.Equal(1, count); + } + + [Fact] + public async Task IncrementAsync_MultipleIncrements_AccumulatesCount() + { + var ct = TestContext.Current.CancellationToken; + var (service, timeProvider, _) = Create(); + var now = new DateTime(2024, 1, 15, 10, 0, 0, DateTimeKind.Utc); + timeProvider.SetUtcNow(now); + + for (int i = 0; i < 7; i++) + await service.IncrementAsync(CounterKey, ct); + + long count = await service.SumBucketsAsync(CounterKey, now.AddMinutes(-1), now, ct); + Assert.Equal(7, count); + } + + [Fact] + public async Task SumBucketsAsync_AcrossMultipleMinutes_SumsAllBuckets() + { + var ct = TestContext.Current.CancellationToken; + var (service, timeProvider, _) = Create(); + var now = new DateTime(2024, 1, 15, 10, 5, 0, DateTimeKind.Utc); + + // Add 3 events at T-4min + timeProvider.SetUtcNow(now.AddMinutes(-4)); + for (int i = 0; i < 3; i++) + await service.IncrementAsync(CounterKey, ct); + + // Add 5 events at T-2min + timeProvider.SetUtcNow(now.AddMinutes(-2)); + for (int i = 0; i < 5; i++) + await service.IncrementAsync(CounterKey, ct); + + // Add 2 events at T-0 + timeProvider.SetUtcNow(now); + for (int i = 0; i < 2; i++) + await service.IncrementAsync(CounterKey, ct); + + long count = await service.SumBucketsAsync(CounterKey, now.AddMinutes(-5), now, ct); + Assert.Equal(10, count); + } + + [Fact] + public async Task SumBucketsAsync_EmptyWindow_ReturnsZero() + { + var ct = TestContext.Current.CancellationToken; + var (service, timeProvider, _) = Create(); + var now = new DateTime(2024, 1, 15, 10, 0, 0, DateTimeKind.Utc); + timeProvider.SetUtcNow(now); + + long count = await service.SumBucketsAsync(CounterKey, now.AddMinutes(-5), now, ct); + Assert.Equal(0, count); + } + + [Fact] + public async Task SumBucketsAsync_EventsOutsideWindow_NotCounted() + { + var ct = TestContext.Current.CancellationToken; + var (service, timeProvider, _) = Create(); + var now = new DateTime(2024, 1, 15, 10, 10, 0, DateTimeKind.Utc); + + // Add events 10 minutes ago — outside a 5-minute window + timeProvider.SetUtcNow(now.AddMinutes(-10)); + for (int i = 0; i < 20; i++) + await service.IncrementAsync(CounterKey, ct); + + timeProvider.SetUtcNow(now); + long count = await service.SumBucketsAsync(CounterKey, now.AddMinutes(-5), now, ct); + Assert.Equal(0, count); + } + + // ------------------------------------------------------------------------- + // Active counter key tracking + // ------------------------------------------------------------------------- + + [Fact] + public async Task GetActiveCounterKeysAsync_AfterIncrement_ReturnsKey() + { + var ct = TestContext.Current.CancellationToken; + var (service, timeProvider, _) = Create(); + var now = new DateTime(2024, 1, 15, 10, 0, 0, DateTimeKind.Utc); + timeProvider.SetUtcNow(now); + + await service.IncrementAsync(CounterKey, ct); + + var keys = await service.GetActiveCounterKeysAsync(now, ct); + Assert.Contains(CounterKey, keys); + } + + [Fact] + public async Task GetActiveCounterKeysAsync_DifferentMinute_ReturnsEmpty() + { + var ct = TestContext.Current.CancellationToken; + var (service, timeProvider, _) = Create(); + var now = new DateTime(2024, 1, 15, 10, 5, 0, DateTimeKind.Utc); + timeProvider.SetUtcNow(now); + + await service.IncrementAsync(CounterKey, ct); + + // Ask for a different minute + var keys = await service.GetActiveCounterKeysAsync(now.AddMinutes(-3), ct); + Assert.DoesNotContain(CounterKey, keys); + } + + // ------------------------------------------------------------------------- + // Cooldown tests + // ------------------------------------------------------------------------- + + [Fact] + public async Task IsOnCooldownAsync_WhenNoCooldownSet_ReturnsFalse() + { + var ct = TestContext.Current.CancellationToken; + var (service, _, _) = Create(); + bool onCooldown = await service.IsOnCooldownAsync(RuleId, SubjectKey, ct); + Assert.False(onCooldown); + } + + [Fact] + public async Task IsOnCooldownAsync_AfterSetCooldown_ReturnsTrue() + { + var ct = TestContext.Current.CancellationToken; + var (service, _, _) = Create(); + await service.SetCooldownAsync(RuleId, SubjectKey, TimeSpan.FromHours(1), ct); + bool onCooldown = await service.IsOnCooldownAsync(RuleId, SubjectKey, ct); + Assert.True(onCooldown); + } + + [Fact] + public async Task SetCooldownAsync_DifferentRules_IndependentCooldowns() + { + var ct = TestContext.Current.CancellationToken; + var (service, _, _) = Create(); + const string ruleId2 = "rule-002"; + + await service.SetCooldownAsync(RuleId, SubjectKey, TimeSpan.FromHours(1), ct); + + bool rule1OnCooldown = await service.IsOnCooldownAsync(RuleId, SubjectKey, ct); + bool rule2OnCooldown = await service.IsOnCooldownAsync(ruleId2, SubjectKey, ct); + + Assert.True(rule1OnCooldown); + Assert.False(rule2OnCooldown); + } +} From 43254b075d80d14334f986b314679c2e95d3cd6e Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sun, 31 May 2026 07:42:37 -0500 Subject: [PATCH 10/14] chore: update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 5cd0861747..9c8213d1bc 100644 --- a/.gitignore +++ b/.gitignore @@ -84,3 +84,4 @@ debug-storybook.log .devcontainer/devcontainer-lock.json *.lscache +.gstack/ From 2d93a42e02d843c4ed11e7e91be348e345fa00e4 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sun, 31 May 2026 07:48:29 -0500 Subject: [PATCH 11/14] feat: add rate-notifications Svelte 5 frontend feature module - types.ts: ViewRateNotificationRule, NewRateNotificationRule, UpdateRateNotificationRule, SnoozeRateNotificationRuleRequest - api.svelte.ts: TanStack Query wrappers for list, create, update, delete, snooze, unsnooze - components/rate-notification-rule-list.svelte: list with enable toggle, snooze badge, delete confirm, premium gate - components/rate-notification-rule-form.svelte: create/edit form with validation, premium gate, noise warning - index.ts: barrel export Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../features/rate-notifications/api.svelte.ts | 210 +++++++++++++ .../rate-notification-rule-form.svelte | 284 ++++++++++++++++++ .../rate-notification-rule-list.svelte | 217 +++++++++++++ .../lib/features/rate-notifications/index.ts | 5 + .../lib/features/rate-notifications/types.ts | 78 +++++ 5 files changed, 794 insertions(+) create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/api.svelte.ts create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/components/rate-notification-rule-form.svelte create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/components/rate-notification-rule-list.svelte create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/index.ts create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/types.ts diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/api.svelte.ts new file mode 100644 index 0000000000..f0224d51d2 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/api.svelte.ts @@ -0,0 +1,210 @@ +import type { + NewRateNotificationRule, + SnoozeRateNotificationRuleRequest, + UpdateRateNotificationRule, + ViewRateNotificationRule +} from '$features/rate-notifications/types'; + +import { accessToken } from '$features/auth/index.svelte'; +import { type FetchClientResponse, type ProblemDetails, useFetchClient } from '@exceptionless/fetchclient'; +import { createMutation, createQuery, QueryClient, useQueryClient } from '@tanstack/svelte-query'; + +export const queryKeys = { + create: (userId: string | undefined, projectId: string | undefined) => + [...queryKeys.list(userId, projectId), 'create'] as const, + delete: (userId: string | undefined, projectId: string | undefined, ruleId: string | undefined) => + [...queryKeys.id(userId, projectId, ruleId), 'delete'] as const, + id: (userId: string | undefined, projectId: string | undefined, ruleId: string | undefined) => + [...queryKeys.list(userId, projectId), ruleId] as const, + list: (userId: string | undefined, projectId: string | undefined) => + ['RateNotificationRule', userId, projectId] as const, + snooze: (userId: string | undefined, projectId: string | undefined, ruleId: string | undefined) => + [...queryKeys.id(userId, projectId, ruleId), 'snooze'] as const, + unsnooze: (userId: string | undefined, projectId: string | undefined, ruleId: string | undefined) => + [...queryKeys.id(userId, projectId, ruleId), 'unsnooze'] as const, + update: (userId: string | undefined, projectId: string | undefined, ruleId: string | undefined) => + [...queryKeys.id(userId, projectId, ruleId), 'update'] as const +}; + +export async function invalidateRateNotificationQueries( + queryClient: QueryClient, + userId: string | undefined, + projectId: string | undefined, + ruleId?: string | undefined +) { + if (ruleId) { + await queryClient.invalidateQueries({ queryKey: queryKeys.id(userId, projectId, ruleId) }); + } + await queryClient.invalidateQueries({ queryKey: queryKeys.list(userId, projectId) }); +} + +function ruleRoute(userId: string, projectId: string, ruleId?: string): string { + const base = `users/${userId}/projects/${projectId}/rate-notifications`; + return ruleId ? `${base}/${ruleId}` : base; +} + +// ---- List ---- + +export interface GetRuleListRequest { + params?: { limit?: number; page?: number }; + route: { projectId: string | undefined; userId: string | undefined }; +} + +export function getRateNotificationRulesQuery(request: GetRuleListRequest) { + const queryClient = useQueryClient(); + + return createQuery, ProblemDetails>(() => ({ + enabled: () => !!accessToken.current && !!request.route.userId && !!request.route.projectId, + onSuccess: (data: FetchClientResponse) => { + data.data?.forEach((rule) => { + queryClient.setQueryData(queryKeys.id(request.route.userId, request.route.projectId, rule.id), rule); + }); + }, + queryClient, + queryFn: async ({ signal }: { signal: AbortSignal }) => { + const client = useFetchClient(); + const response = await client.getJSON( + ruleRoute(request.route.userId!, request.route.projectId!), + { params: { limit: 50, ...request.params }, signal } + ); + return response; + }, + queryKey: [...queryKeys.list(request.route.userId, request.route.projectId), { params: request.params }] + })); +} + +// ---- Create ---- + +export interface CreateRuleRequest { + route: { projectId: string | undefined; userId: string | undefined }; +} + +export function createRateNotificationRule(request: CreateRuleRequest) { + const queryClient = useQueryClient(); + + return createMutation(() => ({ + enabled: () => !!accessToken.current && !!request.route.userId && !!request.route.projectId, + mutationFn: async (body: NewRateNotificationRule) => { + const client = useFetchClient(); + const response = await client.postJSON( + ruleRoute(request.route.userId!, request.route.projectId!), + body + ); + return response.data!; + }, + mutationKey: queryKeys.create(request.route.userId, request.route.projectId), + onSuccess: (rule: ViewRateNotificationRule) => { + queryClient.setQueryData(queryKeys.id(request.route.userId, request.route.projectId, rule.id), rule); + queryClient.invalidateQueries({ queryKey: queryKeys.list(request.route.userId, request.route.projectId) }); + } + })); +} + +// ---- Update ---- + +export interface UpdateRuleRequest { + route: { projectId: string | undefined; ruleId: string | undefined; userId: string | undefined }; +} + +export function updateRateNotificationRule(request: UpdateRuleRequest) { + const queryClient = useQueryClient(); + + return createMutation(() => ({ + enabled: () => + !!accessToken.current && !!request.route.userId && !!request.route.projectId && !!request.route.ruleId, + mutationFn: async (body: UpdateRateNotificationRule) => { + const client = useFetchClient(); + const response = await client.putJSON( + ruleRoute(request.route.userId!, request.route.projectId!, request.route.ruleId!), + body + ); + return response.data!; + }, + mutationKey: queryKeys.update(request.route.userId, request.route.projectId, request.route.ruleId), + onSuccess: (rule: ViewRateNotificationRule) => { + queryClient.setQueryData(queryKeys.id(request.route.userId, request.route.projectId, rule.id), rule); + queryClient.invalidateQueries({ queryKey: queryKeys.list(request.route.userId, request.route.projectId) }); + } + })); +} + +// ---- Delete ---- + +export interface DeleteRuleRequest { + route: { projectId: string | undefined; ruleId: string | undefined; userId: string | undefined }; +} + +export function deleteRateNotificationRule(request: DeleteRuleRequest) { + const queryClient = useQueryClient(); + + return createMutation(() => ({ + enabled: () => + !!accessToken.current && !!request.route.userId && !!request.route.projectId && !!request.route.ruleId, + mutationFn: async () => { + const client = useFetchClient(); + await client.delete(ruleRoute(request.route.userId!, request.route.projectId!, request.route.ruleId!), { + expectedStatusCodes: [204] + }); + }, + mutationKey: queryKeys.delete(request.route.userId, request.route.projectId, request.route.ruleId), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: queryKeys.list(request.route.userId, request.route.projectId) }); + } + })); +} + +// ---- Snooze ---- + +export interface SnoozeRuleRequest { + route: { projectId: string | undefined; ruleId: string | undefined; userId: string | undefined }; +} + +export function snoozeRateNotificationRule(request: SnoozeRuleRequest) { + const queryClient = useQueryClient(); + + return createMutation(() => ({ + enabled: () => + !!accessToken.current && !!request.route.userId && !!request.route.projectId && !!request.route.ruleId, + mutationFn: async (body: SnoozeRateNotificationRuleRequest) => { + const client = useFetchClient(); + await client.postJSON( + `${ruleRoute(request.route.userId!, request.route.projectId!, request.route.ruleId!)}/snooze`, + body, + { expectedStatusCodes: [200, 204] } + ); + }, + mutationKey: queryKeys.snooze(request.route.userId, request.route.projectId, request.route.ruleId), + onSuccess: () => { + queryClient.invalidateQueries({ + queryKey: queryKeys.id(request.route.userId, request.route.projectId, request.route.ruleId) + }); + queryClient.invalidateQueries({ queryKey: queryKeys.list(request.route.userId, request.route.projectId) }); + } + })); +} + +// ---- Unsnooze ---- + +export function unsnoozeRateNotificationRule(request: SnoozeRuleRequest) { + const queryClient = useQueryClient(); + + return createMutation(() => ({ + enabled: () => + !!accessToken.current && !!request.route.userId && !!request.route.projectId && !!request.route.ruleId, + mutationFn: async () => { + const client = useFetchClient(); + await client.postJSON( + `${ruleRoute(request.route.userId!, request.route.projectId!, request.route.ruleId!)}/unsnooze`, + {}, + { expectedStatusCodes: [200, 204] } + ); + }, + mutationKey: queryKeys.unsnooze(request.route.userId, request.route.projectId, request.route.ruleId), + onSuccess: () => { + queryClient.invalidateQueries({ + queryKey: queryKeys.id(request.route.userId, request.route.projectId, request.route.ruleId) + }); + queryClient.invalidateQueries({ queryKey: queryKeys.list(request.route.userId, request.route.projectId) }); + } + })); +} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/components/rate-notification-rule-form.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/components/rate-notification-rule-form.svelte new file mode 100644 index 0000000000..554deaad06 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/components/rate-notification-rule-form.svelte @@ -0,0 +1,284 @@ + + +
{ e.preventDefault(); handleSubmit(); }}> + {#if !hasPremiumFeatures} + + + + Upgrade now to enable and create rate notification rules. + + + {/if} + + + + This rule may be noisy. Use a cooldown to avoid repeated emails. + + + +
+ + + {#if nameError} +

{nameError}

+ {/if} +
+ + +
+ + + + {SIGNAL_LABELS[signal]} + + + {#each Object.entries(SIGNAL_LABELS) as [value, label] (value)} + {label} + {/each} + + +
+ + +
+ + + + {subject} + + + Project + Stack + + +
+ + + {#if subject === 'Stack'} +
+ + + {#if stackIdError} +

{stackIdError}

+ {/if} +
+ {/if} + + +
+ + + {#if thresholdError} +

{thresholdError}

+ {/if} +
+ + +
+ + + + {WINDOW_OPTIONS.find((o) => o.value === window)?.label ?? window} + + + {#each WINDOW_OPTIONS as option (option.value)} + {option.label} + {/each} + + +
+ + +
+ + + + {WINDOW_OPTIONS.find((o) => o.value === cooldown)?.label ?? cooldown} + + + {#each WINDOW_OPTIONS as option (option.value)} + {option.label} + {/each} + + 2 hours + 4 hours + 8 hours + 24 hours + + + {#if cooldownError} +

{cooldownError}

+ {/if} + Further notifications for this rule are suppressed during the cooldown period. +
+ + +
+ + +
+ + {#if formError} +

{formError}

+ {/if} + +
+ {#if onCancel} + + {/if} + +
+
diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/components/rate-notification-rule-list.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/components/rate-notification-rule-list.svelte new file mode 100644 index 0000000000..917994e2a5 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/components/rate-notification-rule-list.svelte @@ -0,0 +1,217 @@ + + +
+ + + {#if !hasPremiumFeatures} + + + + Upgrade now to enable personal rate notifications! + + + {/if} + + {#if listQuery.isLoading} +
+ {#each { length: 2 } as _} +
+ {/each} +
+ {:else if rules.length === 0} +
+ +

No rate notification rules yet.

+ {#if hasPremiumFeatures} + + {/if} +
+ {:else} +
+ {#each rules as rule (rule.id)} +
+
+
+ +
+ {SIGNAL_LABELS[rule.signal]} + + ≥{rule.threshold} in {formatWindow(rule.window)} + + {#if rule.is_snoozed} + + + Snoozed + + {/if} + {#if rule.subject === 'Stack' && rule.stack_id} + Stack-scoped + {/if} +
+
+
+ + toggleEnabled(rule, checked)} + aria-label={rule.is_enabled ? 'Disable rule' : 'Enable rule'} + /> + +
+
+
+ {/each} +
+ + {#if hasPremiumFeatures && rules.length < MAX_RULES_PER_PROJECT} + + {/if} + {/if} +
+ + + !open && (confirmDeleteRuleId = undefined)}> + + + Delete rule? + + This action cannot be undone. The rate notification rule will be permanently deleted. + + + + + + + + diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/index.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/index.ts new file mode 100644 index 0000000000..8e3c411379 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/index.ts @@ -0,0 +1,5 @@ +// Rate notifications feature barrel export +export { default as RateNotificationRuleForm } from './components/rate-notification-rule-form.svelte'; +export { default as RateNotificationRuleList } from './components/rate-notification-rule-list.svelte'; +export * from './api.svelte'; +export * from './types'; diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/types.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/types.ts new file mode 100644 index 0000000000..7f56f5bbbd --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/types.ts @@ -0,0 +1,78 @@ +// Types matching the RateNotificationRule API DTOs + +export type RateNotificationSignal = 'AllEvents' | 'Errors' | 'CriticalErrors' | 'NewErrors' | 'Regressions'; +export type RateNotificationSubject = 'Project' | 'Stack'; + +export interface ViewRateNotificationRule { + id: string; + organization_id: string; + project_id: string; + user_id: string; + version: number; + name: string; + is_enabled: boolean; + signal: RateNotificationSignal; + subject: RateNotificationSubject; + stack_id?: string; + threshold: number; + /** ISO 8601 duration string (e.g. "00:05:00") */ + window: string; + /** ISO 8601 duration string */ + cooldown: string; + snoozed_until_utc?: string; + last_fired_utc?: string; + created_utc: string; + updated_utc: string; + /** Computed: snoozed_until_utc is in the future */ + is_snoozed: boolean; +} + +export interface NewRateNotificationRule { + name: string; + signal: RateNotificationSignal; + subject: RateNotificationSubject; + stack_id?: string; + threshold: number; + /** ISO 8601 duration string (e.g. "00:05:00") */ + window: string; + /** ISO 8601 duration string */ + cooldown: string; + is_enabled: boolean; +} + +export interface UpdateRateNotificationRule { + name?: string; + signal?: RateNotificationSignal; + subject?: RateNotificationSubject; + stack_id?: string; + threshold?: number; + window?: string; + cooldown?: string; + is_enabled?: boolean; +} + +export interface SnoozeRateNotificationRuleRequest { + duration_seconds?: number; + until_utc?: string; +} + +/** Friendly labels for signal enum values */ +export const SIGNAL_LABELS: Record = { + AllEvents: 'All Events', + CriticalErrors: 'Critical Errors', + Errors: 'Errors', + NewErrors: 'New Errors', + Regressions: 'Regressions' +}; + +/** Allowed window durations (as ISO 8601) mapped to friendly labels */ +export const WINDOW_OPTIONS: { label: string; value: string }[] = [ + { label: '1 minute', value: '00:01:00' }, + { label: '5 minutes', value: '00:05:00' }, + { label: '10 minutes', value: '00:10:00' }, + { label: '15 minutes', value: '00:15:00' }, + { label: '30 minutes', value: '00:30:00' }, + { label: '1 hour', value: '01:00:00' } +]; + +export const MAX_RULES_PER_PROJECT = 20; From 143cfea9da8889fdece5b31dd360c0bcd7668116 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sun, 31 May 2026 07:58:01 -0500 Subject: [PATCH 12/14] fix: use ImmediateConsistency on AddAsync to prevent rule-count bypass and test false positives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Controller: AddAsync(rule, o => o.Cache().ImmediateConsistency()) ensures the 20-rule-per-project limit is enforced even under rapid concurrent creation. Without this, ES doesn't refresh before CountByProjectIdAndUserIdAsync, returning stale count=0 and allowing unlimited rules. - Tests: add Foundatio.Repositories using + ImmediateConsistency() on all _ruleRepository.AddAsync calls in RateNotificationEvaluatorJobTests so that GetEnabledByProjectIdAsync sees freshly indexed rules. Previously the 3 passing tests were false positives (rules invisible → no enqueue = expected) and the 2 failing tests were false negatives (rules invisible → no enqueue ≠ expected). All 29 rate-notification tests now pass: 12/12 RateCounterServiceTests 12/12 RateNotificationRuleControllerTests 5/5 RateNotificationEvaluatorJobTests Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Controllers/RateNotificationRuleController.cs | 2 +- .../Jobs/RateNotificationEvaluatorJobTests.cs | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/Exceptionless.Web/Controllers/RateNotificationRuleController.cs b/src/Exceptionless.Web/Controllers/RateNotificationRuleController.cs index 3a3998734d..c4bf00aa0d 100644 --- a/src/Exceptionless.Web/Controllers/RateNotificationRuleController.cs +++ b/src/Exceptionless.Web/Controllers/RateNotificationRuleController.cs @@ -145,7 +145,7 @@ public async Task> PostAsync( UpdatedUtc = now }; - rule = await _ruleRepository.AddAsync(rule, o => o.Cache()); + rule = await _ruleRepository.AddAsync(rule, o => o.Cache().ImmediateConsistency()); await _ruleCache.InvalidateAsync(projectId); return Created(new Uri(Url.Link("GetRateNotificationRuleById", new { userId, projectId, ruleId = rule.Id })!, UriKind.RelativeOrAbsolute), MapToView(rule)); diff --git a/tests/Exceptionless.Tests/Jobs/RateNotificationEvaluatorJobTests.cs b/tests/Exceptionless.Tests/Jobs/RateNotificationEvaluatorJobTests.cs index 641702c01f..6a55b4653c 100644 --- a/tests/Exceptionless.Tests/Jobs/RateNotificationEvaluatorJobTests.cs +++ b/tests/Exceptionless.Tests/Jobs/RateNotificationEvaluatorJobTests.cs @@ -5,6 +5,7 @@ using Exceptionless.Core.Services; using Exceptionless.Core.Utility; using Foundatio.Queues; +using Foundatio.Repositories; using Xunit; namespace Exceptionless.Tests.Jobs; @@ -67,7 +68,7 @@ public async Task RunAsync_WhenThresholdCrossed_EnqueuesNotification() var now = new DateTime(2024, 6, 1, 12, 0, 0, DateTimeKind.Utc); TimeProvider.SetUtcNow(now.AddMinutes(-2)); - var rule = await _ruleRepository.AddAsync(BuildRule(threshold: 5)); + var rule = await _ruleRepository.AddAsync(BuildRule(threshold: 5), o => o.ImmediateConsistency()); string counterKey = BuildCounterKey(rule); // Simulate 10 events within the 5-minute window @@ -92,7 +93,7 @@ public async Task RunAsync_WhenBelowThreshold_DoesNotEnqueue() var now = new DateTime(2024, 6, 1, 12, 0, 0, DateTimeKind.Utc); TimeProvider.SetUtcNow(now.AddMinutes(-2)); - var rule = await _ruleRepository.AddAsync(BuildRule(threshold: 50)); + var rule = await _ruleRepository.AddAsync(BuildRule(threshold: 50), o => o.ImmediateConsistency()); string counterKey = BuildCounterKey(rule); // Only 5 events — well below threshold of 50 @@ -118,7 +119,7 @@ public async Task RunAsync_WhenOnCooldown_DoesNotEnqueueAgain() var now = new DateTime(2024, 6, 1, 12, 0, 0, DateTimeKind.Utc); TimeProvider.SetUtcNow(now.AddMinutes(-2)); - var rule = await _ruleRepository.AddAsync(BuildRule(threshold: 5)); + var rule = await _ruleRepository.AddAsync(BuildRule(threshold: 5), o => o.ImmediateConsistency()); string counterKey = BuildCounterKey(rule); string subjectKey = $"project:{rule.ProjectId}"; @@ -151,7 +152,7 @@ public async Task RunAsync_WhenActivelySnoozed_SkipsEvaluation() var ruleData = BuildRule(threshold: 5); ruleData.SnoozedUntilUtc = now.AddHours(2); // snoozed for 2 more hours - var rule = await _ruleRepository.AddAsync(ruleData); + var rule = await _ruleRepository.AddAsync(ruleData, o => o.ImmediateConsistency()); string counterKey = BuildCounterKey(rule); // Add events above threshold @@ -191,10 +192,10 @@ public async Task RunAsync_SnoozeBackAlert_SnoozedRuleIgnoresTrafficDuringSnooze // Rule A: snoozed until T-2min (snooze has just expired at "now") var ruleAData = BuildRule(threshold: 10); ruleAData.SnoozedUntilUtc = now.AddMinutes(-2); - var ruleA = await _ruleRepository.AddAsync(ruleAData); + var ruleA = await _ruleRepository.AddAsync(ruleAData, o => o.ImmediateConsistency()); // Rule B: not snoozed — should fire normally - var ruleB = await _ruleRepository.AddAsync(BuildRule(threshold: 10)); + var ruleB = await _ruleRepository.AddAsync(BuildRule(threshold: 10), o => o.ImmediateConsistency()); // Both rules watch the same signal/counter key string counterKey = BuildCounterKey(ruleA); From 0cbe8299cd706e1a6ffd23c44676e5f9a224d4d3 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sun, 31 May 2026 08:23:20 -0500 Subject: [PATCH 13/14] fix: string-enum serialization + mutation-init + UI integration for rate notifications MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs fixed: 1. Enum serialization mismatch: RateNotificationSignal/Subject were bare int enums; API rejected frontend's string values ('Errors', 'Project') with 400. Fix: add JsonStringEnumConverter (STJ) + StringEnumConverter (Newtonsoft) to both enums, matching the StackStatus pattern used elsewhere in the codebase. API now returns and accepts string enum names. 2. Mutation created inside event handler: createRateNotificationRule/ updateRateNotificationRule were called inside handleSubmit(), which runs outside Svelte's synchronous component init context — TanStack Query context not available there. Fix: create both mutations at init with reactive getter props, call mutateAsync only inside the handler. Also: integrated RateNotificationRuleList + RateNotificationRuleForm into account/notifications page, fixed type errors in both components (Select.Root type='single', Alert variant, parseSeconds array safety). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Models/Enums/RateNotificationSignal.cs | 5 ++ .../Models/Enums/RateNotificationSubject.cs | 5 ++ .../rate-notification-rule-form.svelte | 40 ++++++++---- .../rate-notification-rule-list.svelte | 5 +- .../(app)/account/notifications/+page.svelte | 65 +++++++++++++++++++ 5 files changed, 105 insertions(+), 15 deletions(-) diff --git a/src/Exceptionless.Core/Models/Enums/RateNotificationSignal.cs b/src/Exceptionless.Core/Models/Enums/RateNotificationSignal.cs index e45a6f276f..2e3e9e620b 100644 --- a/src/Exceptionless.Core/Models/Enums/RateNotificationSignal.cs +++ b/src/Exceptionless.Core/Models/Enums/RateNotificationSignal.cs @@ -1,5 +1,10 @@ +using System.Text.Json.Serialization; +using Newtonsoft.Json.Converters; + namespace Exceptionless.Core.Models; +[JsonConverter(typeof(JsonStringEnumConverter))] +[Newtonsoft.Json.JsonConverter(typeof(StringEnumConverter))] public enum RateNotificationSignal { AllEvents = 0, diff --git a/src/Exceptionless.Core/Models/Enums/RateNotificationSubject.cs b/src/Exceptionless.Core/Models/Enums/RateNotificationSubject.cs index 44bca330a3..0e3f353c04 100644 --- a/src/Exceptionless.Core/Models/Enums/RateNotificationSubject.cs +++ b/src/Exceptionless.Core/Models/Enums/RateNotificationSubject.cs @@ -1,5 +1,10 @@ +using System.Text.Json.Serialization; +using Newtonsoft.Json.Converters; + namespace Exceptionless.Core.Models; +[JsonConverter(typeof(JsonStringEnumConverter))] +[Newtonsoft.Json.JsonConverter(typeof(StringEnumConverter))] public enum RateNotificationSubject { Project = 0, diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/components/rate-notification-rule-form.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/components/rate-notification-rule-form.svelte index 554deaad06..cca7f22d0c 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/components/rate-notification-rule-form.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/components/rate-notification-rule-form.svelte @@ -15,6 +15,7 @@ import { createRateNotificationRule, updateRateNotificationRule } from '$features/rate-notifications/api.svelte'; import { SIGNAL_LABELS, WINDOW_OPTIONS } from '$features/rate-notifications/types'; import type { RateNotificationSignal, RateNotificationSubject } from '$features/rate-notifications/types'; + // NOTE: mutations must be created at initialization, not inside event handlers (Svelte/TanStack context requirement) interface Props { hasPremiumFeatures?: boolean; @@ -47,10 +48,12 @@ const thresholdError = $derived(threshold < 1 ? 'Threshold must be at least 1.' : undefined); const stackIdError = $derived(subject === 'Stack' && !stackId.trim() ? 'Stack ID is required when subject is Stack.' : undefined); - // Check cooldown >= window function parseSeconds(iso: string): number { - const [h, m, s] = iso.split(':').map(Number); - return (h || 0) * 3600 + (m || 0) * 60 + (s || 0); + const parts = iso.split(':'); + const h = parseInt(parts[0] ?? '0', 10); + const m = parseInt(parts[1] ?? '0', 10); + const s = parseInt(parts[2] ?? '0', 10); + return h * 3600 + m * 60 + s; } const cooldownError = $derived( parseSeconds(cooldown) < parseSeconds(window) ? 'Cooldown must be at least as long as the window.' : undefined @@ -72,6 +75,21 @@ } }); + const createMutation = createRateNotificationRule({ + route: { + get projectId() { return projectId; }, + get userId() { return userId; } + } + }); + + const updateMutation = updateRateNotificationRule({ + route: { + get projectId() { return rule?.project_id; }, + get ruleId() { return rule?.id; }, + get userId() { return rule?.user_id; } + } + }); + async function handleSubmit() { if (hasErrors || saving) return; @@ -90,9 +108,7 @@ threshold, window }; - const updated = await updateRateNotificationRule({ - route: { projectId: rule.project_id, ruleId: rule.id, userId: rule.user_id } - }).mutateAsync(body); + const updated = await updateMutation.mutateAsync(body); toast.success('Rule updated.'); onSaved?.(updated); } else { @@ -106,7 +122,7 @@ threshold, window }; - const created = await createRateNotificationRule({ route: { projectId, userId } }).mutateAsync(body); + const created = await createMutation.mutateAsync(body); toast.success('Rule created.'); onSaved?.(created); } @@ -130,7 +146,7 @@ {/if} - + This rule may be noisy. Use a cooldown to avoid repeated emails. @@ -155,7 +171,7 @@
- + {SIGNAL_LABELS[signal]} @@ -170,7 +186,7 @@
- + {subject} @@ -220,7 +236,7 @@
- + {WINDOW_OPTIONS.find((o) => o.value === window)?.label ?? window} @@ -235,7 +251,7 @@
- + {WINDOW_OPTIONS.find((o) => o.value === cooldown)?.label ?? cooldown} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/components/rate-notification-rule-list.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/components/rate-notification-rule-list.svelte index 917994e2a5..afee9f76d5 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/components/rate-notification-rule-list.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/components/rate-notification-rule-list.svelte @@ -93,11 +93,10 @@ } function formatWindow(isoWindow: string): string { - // Parse HH:MM:SS to friendly string const parts = isoWindow.split(':'); if (parts.length < 3) return isoWindow; - const h = parseInt(parts[0], 10); - const m = parseInt(parts[1], 10); + const h = parseInt(parts[0] ?? '0', 10); + const m = parseInt(parts[1] ?? '0', 10); if (h > 0) return `${h}h`; if (m === 1) return '1 min'; return `${m} min`; diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/account/notifications/+page.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/account/notifications/+page.svelte index 5c660266e7..c23b8fe355 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/account/notifications/+page.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/account/notifications/+page.svelte @@ -1,14 +1,18 @@
@@ -201,5 +230,41 @@ {emailNotificationsEnabled} hasPremiumFeatures={selectedProject.has_premium_features} /> + +
+

Rate Notifications

+ Get notified when event rates for this project exceed your custom thresholds. +
+ + {/if}
+ + !open && closeRateRuleDialog()}> + + + {editingRateRule ? 'Edit Rate Notification Rule' : 'Create Rate Notification Rule'} + + {editingRateRule ? 'Update the rule settings below.' : 'Configure when you want to receive an email notification based on event rates.'} + + + {#if selectedProject} + + {/if} + + From 568f8fc3eef6cdb0d1a90f0b7e8883143a808bdf Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sun, 31 May 2026 08:24:38 -0500 Subject: [PATCH 14/14] style: address bot feedback - combine nested ifs and use Where() in pipeline action Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Pipeline/075_UpdateRateCountersAction.cs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/Exceptionless.Core/Pipeline/075_UpdateRateCountersAction.cs b/src/Exceptionless.Core/Pipeline/075_UpdateRateCountersAction.cs index 75381a2afc..edbe7c4707 100644 --- a/src/Exceptionless.Core/Pipeline/075_UpdateRateCountersAction.cs +++ b/src/Exceptionless.Core/Pipeline/075_UpdateRateCountersAction.cs @@ -81,17 +81,12 @@ public override async Task ProcessAsync(EventContext ctx) matchedSignals.Add(RateNotificationSignal.Regressions); // Increment counters for each matching rule - foreach (var rule in rules) + foreach (var rule in rules.Where(r => matchedSignals.Contains(r.Signal))) { - if (!matchedSignals.Contains(rule.Signal)) - continue; - // For Stack subject, only match if this event belongs to the rule's stack - if (rule.Subject == RateNotificationSubject.Stack) - { - if (String.IsNullOrEmpty(rule.StackId) || !String.Equals(ctx.Event.StackId, rule.StackId, StringComparison.Ordinal)) - continue; - } + if (rule.Subject == RateNotificationSubject.Stack && + (String.IsNullOrEmpty(rule.StackId) || !String.Equals(ctx.Event.StackId, rule.StackId, StringComparison.Ordinal))) + continue; string counterKey = BuildCounterKey(rule); await _counterService.IncrementAsync(counterKey);