diff --git a/.gitignore b/.gitignore index 5cd0861747..9c8213d1bc 100644 --- a/.gitignore +++ b/.gitignore @@ -84,3 +84,4 @@ debug-storybook.log .devcontainer/devcontainer-lock.json *.lscache +.gstack/ diff --git a/openspec/changes/add-personal-rate-notifications/design.md b/openspec/changes/add-personal-rate-notifications/design.md new file mode 100644 index 0000000000..416337c9dc --- /dev/null +++ b/openspec/changes/add-personal-rate-notifications/design.md @@ -0,0 +1,429 @@ +# Design: Add Personal Rate Notifications + +## Current Architecture Summary + +- `Project` stores `NotificationSettings` as a `Dictionary` keyed by user ID or integration key. +- Existing `NotificationSettings` are simple booleans: `ReportNewErrors`, `ReportCriticalErrors`, `ReportEventRegressions`, `ReportNewEvents`, `ReportCriticalEvents`, `SendDailySummary`. +- `QueueNotificationAction` (priority 70) enqueues `EventNotification` for existing per-event notifications based on project notification settings. +- `EventNotificationsJob` loads event/project/user and sends Slack/email with existing throttles. +- `Mailer` queues `MailMessage` and `MailMessageJob` sends it. +- `UsageService` already uses cache-backed 5-minute bucket counters for usage tracking via `ICacheClient`. +- Production insulation (`Exceptionless.Insulation`) replaces in-memory cache/queues with Redis/Azure/SQS. + +## Scope + +v1 is personal rate notifications only. No organization-level rules, no webhooks, no digests. It also follows the existing premium-only occurrence-notification model instead of introducing a new free notification channel. + +## Data Model + +### RateNotificationRule + +```csharp +public class RateNotificationRule : IOwnedByOrganizationAndProjectWithIdentity +{ + public string Id { get; set; } + public string OrganizationId { get; set; } + public string ProjectId { get; set; } + public string UserId { get; set; } + public int Version { get; set; } + public string Name { get; set; } + public bool IsEnabled { get; set; } + public RateNotificationSignal Signal { get; set; } + public RateNotificationSubject Subject { get; set; } + public string? StackId { get; set; } + public int Threshold { get; set; } + public TimeSpan Window { get; set; } + public TimeSpan Cooldown { get; set; } + public DateTime? SnoozedUntilUtc { get; set; } + public DateTime CreatedUtc { get; set; } + public DateTime UpdatedUtc { get; set; } + public bool IsDeleted { get; set; } +} +``` + +### Enums + +```csharp +public enum RateNotificationSignal +{ + AllEvents, + Errors, + CriticalErrors, + NewErrors, + Regressions +} + +public enum RateNotificationSubject +{ + Project, + Stack +} +``` + +### Design decisions + +- Rules are stored separately from `Project.NotificationSettings`. +- Existing `NotificationSettings` should not be expanded because rate rules are richer and independently mutable. +- No tag/environment/time-of-day/external-recipient fields in v1. +- `SnoozedUntilUtc` is also the rule's resume boundary. Manual unsnooze sets it to the current UTC time instead of clearing it so the evaluator can ignore activity gathered during the muted period. + +## Repository + +### IRateNotificationRuleRepository + +```csharp +public interface IRateNotificationRuleRepository : IRepositoryOwnedByOrganizationAndProject +{ + Task> GetByProjectIdAndUserIdAsync(string projectId, string userId, CommandOptionsDescriptor? options = null); + Task> GetEnabledByProjectIdAsync(string projectId, CommandOptionsDescriptor? options = null); + Task CountByProjectIdAndUserIdAsync(string projectId, string userId); +} +``` + +### RateNotificationRuleRepository + +Elasticsearch-backed repository following existing patterns (e.g., `StackRepository`, `WebHookRepository`). + +## API + +### Routes + +| Method | Route | Description | +|--------|-------|-------------| +| GET | `/api/v2/users/{userId}/projects/{projectId}/rate-notifications` | List user's rules for project | +| POST | `/api/v2/users/{userId}/projects/{projectId}/rate-notifications` | Create rule | +| GET | `/api/v2/users/{userId}/projects/{projectId}/rate-notifications/{ruleId}` | Get rule | +| PUT | `/api/v2/users/{userId}/projects/{projectId}/rate-notifications/{ruleId}` | Update rule | +| DELETE | `/api/v2/users/{userId}/projects/{projectId}/rate-notifications/{ruleId}` | Delete rule | +| POST | `/api/v2/users/{userId}/projects/{projectId}/rate-notifications/{ruleId}/snooze` | Snooze rule | +| POST | `/api/v2/users/{userId}/projects/{projectId}/rate-notifications/{ruleId}/unsnooze` | Unsnooze rule | + +### Authorization + +- Current user can manage their own rules. +- Global admin can manage any user's rules. +- User must have access to the project's organization. +- External recipients are not supported in v1. + +### Premium gating + +- Rate notifications follow the current occurrence-notification premium model. +- Rules may remain persisted across plan downgrade, but countering, evaluation, delivery, and Svelte create/enable controls MUST treat the feature as unavailable until premium is restored. + +### Validation + +- `threshold > 0` +- `window` must be one of: 1m, 5m, 10m, 15m, 30m, 1h +- `cooldown` must be at least the window duration +- Recommended default cooldown: 30m +- Project subject must not specify `stack_id` +- Stack subject must specify `stack_id` +- `stack_id` must belong to the same project +- User cannot exceed 20 rules per project +- Rule name must be non-empty and ≤ 100 characters + +## Rule Index + +### RateNotificationRuleIndex service + +Purpose: + +- Load enabled rules for a project. +- Cache compiled counter definitions briefly. +- Ensure event pipeline can cheaply determine which counters to increment. +- Invalidate project rule index on create/update/delete/snooze/unsnooze. + +**Important:** The event pipeline must not increment every possible counter. It must increment only counters required by enabled rules for that project. + +Cache key: `rate:v1:rules:project:{projectId}` + +## Counter Architecture + +### RateCounterService + +Uses 1-minute UTC buckets. + +### Cache keys + +``` +rate:v1:count:{epochMinute}:{counterKey} +rate:v1:active:{epochMinute} +rate:v1:cooldown:{ruleId}:{subjectKey} +rate:v1:rules:project:{projectId} +``` + +### Counter key examples + +``` +project:{projectId}:signal:Errors +project:{projectId}:signal:CriticalErrors +project:{projectId}:signal:NewErrors +project:{projectId}:signal:Regressions +project:{projectId}:signal:AllEvents +project:{projectId}:stack:{stackId}:signal:Errors +project:{projectId}:stack:{stackId}:signal:CriticalErrors +project:{projectId}:stack:{stackId}:signal:NewErrors +project:{projectId}:stack:{stackId}:signal:Regressions +project:{projectId}:stack:{stackId}:signal:AllEvents +``` + +### TTL guidance + +- Counter bucket TTL: 3 hours +- Active bucket TTL: 3 hours +- Cooldown TTL: cooldown duration + 10 minutes + +### Signal matching + +| Signal | Matches when | +|--------|-------------| +| AllEvents | Any event | +| Errors | `ev.IsError()` | +| CriticalErrors | `ev.IsError() && ev.IsCritical()` | +| NewErrors | `ctx.IsNew && ev.IsError()` | +| Regressions | `ctx.IsRegression` | + +## Event Pipeline Action + +### UpdateRateCountersAction + +- Runs after stack assignment (priority > 70, e.g., 75) +- Exits fast when the organization does not have premium features +- Loads `RateNotificationRuleIndex` for project +- Exits fast if no enabled rules +- Skips events on stacks where `!ctx.Stack.AllowNotifications` +- Skips canceled/discarded events that would not produce occurrence notifications +- Skips requests already marked as bots by request-info enrichment +- Matches event against compiled counter definitions +- Increments matching counters via `RateCounterService` +- Adds counter key to active bucket list/set +- Never sends notifications directly +- Never queries Elasticsearch per event + +## Evaluator Job + +### RateNotificationEvaluatorJob + +- Runs periodically (recommended: every 60 seconds) +- Acquires distributed lock so only one evaluator runs per cluster +- Inspects recently active counters from active bucket sets +- Skips organizations without premium features +- Sums buckets for each rule's configured window +- Uses `max(windowStartUtc, rule.SnoozedUntilUtc)` as the lower bound when a snooze boundary falls inside the evaluation window so a rule resumes from a fresh baseline +- Skips disabled rules +- Skips snoozed rules (where `SnoozedUntilUtc > now`) +- Compares observed count ≥ threshold +- Enforces cooldown per rule + subject +- Enqueues `RateNotification` work item on threshold crossing +- Sets cooldown key when enqueue succeeds +- Logs fired/skipped reasons with structured context + +This v1 does NOT need a full Normal/Pending/Firing/Recovering state machine. Simple threshold + cooldown + snooze is sufficient. + +### Snooze semantics + +- Snooze suppresses delivery immediately. +- When a snooze expires or a user manually unsnoozes a rule, the rule resumes from a fresh baseline. +- Activity observed entirely during the snooze window MUST NOT trigger an immediate post-snooze alert, even when another enabled rule kept the shared counter hot. + +## Queue Model + +### RateNotification + +```csharp +public class RateNotification +{ + public string RuleId { get; set; } + public int RuleVersion { get; set; } + public string OrganizationId { get; set; } + public string ProjectId { get; set; } + public string UserId { get; set; } + public string SubjectKey { get; set; } + public string? StackId { get; set; } + public DateTime WindowStartUtc { get; set; } + public DateTime WindowEndUtc { get; set; } + public long ObservedCount { get; set; } + public int Threshold { get; set; } +} +``` + +## Delivery Job + +### RateNotificationsJob + +- Loads rule by ID +- Loads project +- Loads user +- Loads stack for stack-scoped rules so email copy can include stack title and deep link +- Validates: + - Rule still exists + - Rule is enabled + - Rule version matches or is compatible + - User belongs to organization + - User email is verified + - User email notifications are enabled + - Project/org still exists +- Sends email through `IMailer` +- Skips with structured logs when validation fails +- Does not send Slack/webhooks in v1 (marked as future work) + +## Lifecycle cleanup + +- Remove rate notification rules when a user loses organization access. +- Remove rate notification rules when a project or organization is deleted. +- Invalidate cached rule indexes when cleanup runs so orphaned rules stop consuming evaluator work immediately. +- Reuse the same work-item cleanup pattern already used for `Project.NotificationSettings` updates where practical. + +## Email + +### IMailer.SendRateNotificationAsync + +```csharp +Task SendRateNotificationAsync(User user, Project project, RateNotificationRule rule, long observedCount, DateTime windowStart, DateTime windowEnd, Stack? stack); +``` + +Email includes: + +- Rule name +- Project name +- Observed count +- Threshold +- Window +- Subject type (project or stack) +- Stack title (when subject is stack and available) +- Link to project or stack +- Cooldown explanation +- No "everything is fine" messaging + +**Example subject:** `[ProjectName] Error rate exceeded` + +**Example body:** + +``` +Rule: Production error storm +Observed: 241 errors in 5 minutes +Threshold: 100 errors in 5 minutes +Cooldown: Further notifications for this rule are suppressed for 30 minutes. +``` + +## Frontend + +### Feature module + +`src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/` + +UI supports: + +- List rules for current user/project +- Create rule +- Edit rule +- Delete rule +- Enable/disable rule +- Snooze/unsnooze rule +- Disabled/upgrade state when the organization lacks premium features + +### Form fields + +- Name +- Signal (dropdown: All Events, Errors, Critical Errors, New Errors, Regressions) +- Subject (Project or Stack) +- Stack selector (shown when Subject = Stack) +- Threshold (number) +- Window (dropdown: 1m, 5m, 10m, 15m, 30m, 1h) +- Cooldown (duration, minimum = window) +- Enabled (toggle) + +### Not built in v1 + +- Rule history tab +- Delivery history +- Action builder +- Webhook builder +- Slack builder +- Digest UI +- Quiet hours UI +- Organization inheritance UI +- Preview charts + +### Noise warning + +Display when creating/editing: "This rule may be noisy. Use a cooldown to avoid repeated emails." + +## Metrics and Logging + +### Metrics + +- `rate_notification.rules.loaded` +- `rate_notification.counters.incremented` +- `rate_notification.evaluator.runs` +- `rate_notification.evaluator.rules_evaluated` +- `rate_notification.evaluator.notifications_enqueued` +- `rate_notification.delivery.sent` +- `rate_notification.delivery.skipped` + +### Structured log fields + +- `RuleId` +- `ProjectId` +- `UserId` +- `ObservedCount` +- `Threshold` +- `Reason` (skipped/fired) + +## Bootstrap / DI + +- Register `IRateNotificationRuleRepository` / `RateNotificationRuleRepository` +- Register `RateNotificationRuleIndex` +- Register `RateCounterService` +- Register `RateNotificationEvaluatorJob` +- Register `IQueue` notification queue +- Register `RateNotificationsJob` (delivery) +- Update `Exceptionless.Insulation` queue registration for Redis/Azure/SQS providers +- Add queue health check if existing patterns support it + +## Testing Strategy + +### Unit tests + +- Rule validation (threshold, window, cooldown, subject/stack consistency, name) +- Counter key builder +- Counter increments +- Bucket summing +- Signal matching +- Premium gating +- `AllowNotifications` / bot suppression checks +- Cooldown behavior +- Snooze behavior +- Fresh-baseline behavior after snooze expiry or manual unsnooze +- Evaluator threshold crossing logic + +### Integration tests + +- User can CRUD own rules +- User cannot manage another user's rules +- Global admin can manage another user's rules +- Stack rule rejects stack from another project +- Non-premium organizations do not counter, evaluate, or deliver rate notifications +- Events on stacks with `AllowNotifications = false` do not increment rate counters +- Bot-marked requests do not increment rate counters +- Evaluator enqueues notification when threshold crossed +- Evaluator does not enqueue below threshold +- Evaluator respects cooldown +- Evaluator respects snooze +- Activity gathered during snooze does not fire immediately when the rule resumes +- Delivery skips disabled rule +- Delivery skips unverified email +- Delivery skips user not in org +- Delivery sends email for valid rule +- Delivery loads stack context for stack-scoped emails +- Membership/project/org cleanup removes orphaned rules + +### Not tested in v1 + +- Action execution +- Webhooks +- Slack +- Digests +- No-data alerts +- Backtesting +- Rule history UI diff --git a/openspec/changes/add-personal-rate-notifications/proposal.md b/openspec/changes/add-personal-rate-notifications/proposal.md new file mode 100644 index 0000000000..6f215db58f --- /dev/null +++ b/openspec/changes/add-personal-rate-notifications/proposal.md @@ -0,0 +1,126 @@ +# Proposal: Add Personal Rate Notifications + +## Summary + +Add personal, rate-based project and stack notifications. Users can configure rules like: + +- "Email me when this project has more than 100 errors in 5 minutes." +- "Email me when this stack occurs more than 20 times in 10 minutes." + +The implementation is intentionally small and focused: + +- Cheap cache-backed counters (1-minute UTC buckets) +- Asynchronous evaluator job +- Email delivery +- Cooldown and snooze to reduce noise +- Premium-gated runtime behavior that matches existing occurrence notifications +- Existing notification suppression semantics so muted traffic does not reappear as rate noise + +This feature is informed by [issue #177](https://github.com/exceptionless/Exceptionless/issues/177), but intentionally does not implement the full notification wishlist. Issue #177's core goal — notifications should keep users informed without overwhelming them — is addressed through mandatory cooldowns, snooze support, and cache-only hot paths. + +## Classification + +- **Type:** Feature +- **Affected areas:** + - Backend/API + - Event pipeline + - Cache/Redis + - Queue jobs + - Email + - Svelte UI + - Tests +- **OpenSpec justification:** + - New API endpoints + - New persisted rule model + - New event-pipeline counter behavior + - New evaluator/delivery jobs + - Cross-cutting notification behavior + - User-facing notification settings + +## Goals + +- Notify users when project or stack activity crosses a configured threshold. +- Detect high-volume repeated errors that current new/error/regression notifications can miss. +- Reduce notification noise versus per-event email streams. +- Keep event ingestion cheap. +- Support horizontal scaling with distributed cache and queues. +- Preserve current premium-only occurrence-notification behavior. +- Honor existing notification suppression so ignored, snoozed, discarded, and fixed stacks — and bot traffic already excluded from occurrence emails — do not generate rate alerts. +- Validate user/project/org state before sending. +- Support snoozing a noisy rule. +- Resume from a fresh baseline after snooze instead of back-alerting on traffic that happened while the rule was muted. +- Provide enough logging, metrics, and tests to trust the system. + +## Non-goals + +- Anomaly detection or machine learning +- Percent-change alerts +- Arbitrary query/filter language +- Tag rules +- Environment rules +- Time-of-day rules +- Quiet hours / quiet days +- Daily/weekly/monthly summary changes +- Digest emails +- No-data alerts +- Recovery/resolved notifications +- Notification grouping/digesting +- Generic webhook actions +- Slack actions (marked as future work) +- PagerDuty/OpsGenie integrations +- External recipients who are not Exceptionless users +- Mutating automated actions +- Action execution engine +- Durable action execution +- Rule history UI +- Delivery history UI +- In-app notification center +- Billing/overage notification changes +- Email sender/from-address overhaul +- Queue/system health notification UI +- Reduce-noise in-app callouts + +## Deliberate Cutbacks + +Issue #177 is broad — it covers configurable rules based on type, tags, time of day, number of exceptions, environment, snoozing, periodic digest emails, reduced-noise behavior, in-app notices, and third-party integrations. + +This change intentionally implements only **personal rate notifications** — the minimum useful product: + +> "When this project or stack exceeds X matching events in Y minutes, email me, but not more than once per cooldown, and let me snooze the rule." + +The following are explicitly deferred: + +- Digests and periodic summaries +- Quiet hours and time-of-day logic +- Advanced conditional rules (tags, environment, arbitrary filters) +- External recipients and non-user notification targets +- Webhooks, Slack, PagerDuty, and other delivery channels +- Mutating or automated actions +- Organization-level rules and inheritance + +The first release should prove the cheap counter architecture and noise-control model before adding advanced alerting features. + +## Compatibility Risks + +| Risk | Mitigation | +|------|-----------| +| Existing project notification settings remain unchanged | Rate rules are stored separately; `Project.NotificationSettings` is not modified | +| Existing `EventNotificationsJob` behavior remains unchanged | Rate counters are a new pipeline action; existing notification queueing is unaffected | +| Existing `DailySummaryJob` behavior remains unchanged | No changes to daily summary logic | +| Existing Slack/webhook integrations are not changed | New delivery path is email-only; existing `WebHookNotification` queue untouched | +| Existing premium-only occurrence notification behavior could drift | Countering, evaluation, delivery, and Svelte enablement follow the same premium gating model as existing occurrence notifications | +| Rate counters depend on distributed cache in production | In-memory cache/queues remain development-only; production requires Redis/Azure providers | +| Muted stacks or bot traffic could reappear as new rate noise | Countering honors `Stack.AllowNotifications`, canceled/discarded contexts, and request-info bot markers before incrementing counters | +| Snooze could defer a notification instead of suppressing it | Evaluation resumes from a fresh baseline using the snooze boundary so activity gathered during snooze does not fire immediately on resume | +| Notification noise could increase if defaults are bad | Mandatory cooldowns, validation, and max rules per project (20) | +| Rules may become stale if user/project/org state changes | Delivery job re-validates rule, user, project, and org state before sending | +| Orphaned rules could be indexed and evaluated forever | Add cleanup on user membership changes and project/org deletion, plus cache invalidation for removed rules | + +## Rollback Plan + +1. Disable the rate notification evaluator job. +2. Disable the `UpdateRateCountersAction` in the event pipeline. +3. Existing event notifications and daily summaries continue to operate unchanged. +4. Delete or ignore persisted rate notification rules if needed. +5. Remove new UI route/feature module. +6. Remove new queues and cache keys if rolling back fully. diff --git a/openspec/changes/add-personal-rate-notifications/specs/rate-notifications/spec.md b/openspec/changes/add-personal-rate-notifications/specs/rate-notifications/spec.md new file mode 100644 index 0000000000..5c71129d33 --- /dev/null +++ b/openspec/changes/add-personal-rate-notifications/specs/rate-notifications/spec.md @@ -0,0 +1,410 @@ +# Spec: Rate Notifications + +## ADDED Requirements + +### Requirement: User can list their project rate notification rules + +The API MUST allow authenticated users to retrieve their own rate notification rules for a specific project. + +#### Scenario: Authenticated user lists own rules + +Given authenticated user with access to project +When GET `/api/v2/users/{userId}/projects/{projectId}/rate-notifications` +Then return only that user's rules for that project. + +### Requirement: User can create a project rate notification rule + +The API MUST allow authenticated users to create a rate notification rule scoped to a project with a threshold, window, cooldown, signal, and subject. + +#### Scenario: Valid project rule is persisted + +Given authenticated user with access to project +When POST valid rule with threshold, window, cooldown, signal, subject=Project +Then rule is persisted with organization_id, project_id, user_id, threshold, window, cooldown, signal, subject. + +### Requirement: User can create a stack rate notification rule + +The API MUST allow authenticated users to create a rate notification rule scoped to a specific stack within a project. + +#### Scenario: Valid stack rule is persisted + +Given authenticated user with access to project +And stack belongs to project +When POST subject=Stack and stack_id +Then rule is persisted. + +### Requirement: Invalid stack scope is rejected + +The API MUST reject a rule targeting a stack that does not belong to the specified project. + +#### Scenario: Stack from another project is rejected + +Given stack does not belong to project +When user creates stack rule +Then response is 400 or 404. + +### Requirement: User cannot manage another user's rules + +The API MUST deny non-admin users access to rate notification rules belonging to other users. + +#### Scenario: Non-admin user is denied access to other user's rules + +Given non-admin user A +When they request user B's rate notification rules +Then response is 404 or 403. + +### Requirement: Global admin can manage another user's rules + +The API MUST allow global administrators to create, read, update, and delete rate notification rules on behalf of any user. + +#### Scenario: Admin accesses another user's rules + +Given global admin +When they manage another user's rate notification rules +Then request succeeds. + +### Requirement: Rule validation prevents noisy/invalid rules + +The API MUST enforce validation constraints to prevent misconfigured or excessively noisy rules. + +#### Scenario: Threshold must be positive + +Given threshold <= 0 +When user creates rule +Then response is 400 with validation error. + +#### Scenario: Unsupported window is rejected + +Given window is not one of 1m, 5m, 10m, 15m, 30m, 1h +When user creates rule +Then response is 400 with validation error. + +#### Scenario: Cooldown shorter than window is rejected + +Given cooldown < window +When user creates rule +Then response is 400 with validation error. + +#### Scenario: Project subject with stack_id is rejected + +Given subject = Project and stack_id is set +When user creates rule +Then response is 400 with validation error. + +#### Scenario: Stack subject without stack_id is rejected + +Given subject = Stack and stack_id is null +When user creates rule +Then response is 400 with validation error. + +#### Scenario: Empty name is rejected + +Given name is empty +When user creates rule +Then response is 400 with validation error. + +#### Scenario: Name exceeding 100 characters is rejected + +Given name length > 100 +When user creates rule +Then response is 400 with validation error. + +#### Scenario: Exceeding max rules per project is rejected + +Given user already has 20 rules for project +When user creates another rule +Then response is 400 with validation error. + +### Requirement: Event pipeline increments matching configured counters + +The event pipeline MUST increment the corresponding rate counter in the current UTC minute bucket when an event matches an enabled rule's signal. + +#### Scenario: Matching event increments counter + +Given enabled project rule exists +When matching event is processed +Then matching rate counter for current UTC minute bucket is incremented. + +### Requirement: Event pipeline skips projects without enabled rules + +The event pipeline MUST NOT perform any counter operations for projects with no enabled rate notification rules. + +#### Scenario: No rules means no counter work + +Given no enabled rate notification rules for project +When event is processed +Then no rate counter is incremented. + +### Requirement: Rate notifications honor premium feature gating + +Rate notifications MUST follow the existing premium-only occurrence-notification model and MUST NOT become a free notification channel. + +#### Scenario: Non-premium organizations do not activate rate notifications + +Given project organization does not have premium features +When matching events are processed or the evaluator runs +Then no rate counters are incremented +And no rate notification work item is enqueued +And no rate notification email is sent. + +### Requirement: Event pipeline increments only required counters + +The pipeline MUST only increment counters that are required by at least one enabled rule, not all possible signal counters. + +#### Scenario: Only configured signal counters are incremented + +Given project has only an Errors rule +When critical error event is processed +Then Errors counter is incremented +And unrelated counters are not incremented unless required by configured rules. + +### Requirement: Stack counters are stack-specific + +Stack-scoped counters MUST only be incremented by events on the specific stack referenced by the rule. + +#### Scenario: Event on different stack does not increment + +Given stack rule exists for stack A +When event occurs on stack B +Then stack A counter is not incremented. + +### Requirement: Rate notifications honor existing occurrence-notification suppression + +Rate notifications MUST respect existing occurrence-notification suppression semantics so muted traffic does not reappear as rate noise. + +#### Scenario: Events on muted stacks do not increment counters + +Given the event stack has `AllowNotifications = false` +When the event is processed +Then no rate counter is incremented for that event. + +#### Scenario: Bot-marked requests do not increment counters + +Given request enrichment has marked the event request as a bot +When the event is processed +Then no rate counter is incremented for that event. + +### Requirement: Evaluator enqueues notification when threshold is crossed + +The evaluator job MUST enqueue a RateNotification work item when the sum of counter buckets for a rule's window meets or exceeds the threshold. + +#### Scenario: Threshold reached triggers enqueue + +Given rule threshold is 100 errors in 5 minutes +And matching counters sum to 100 or more +When evaluator runs +Then RateNotification work item is enqueued. + +### Requirement: Evaluator does not enqueue below threshold + +The evaluator MUST NOT enqueue a notification when the observed event count is below the rule threshold. + +#### Scenario: Below threshold is silent + +Given rule threshold is 100 +And observed count is 99 +When evaluator runs +Then no work item is enqueued. + +### Requirement: Cooldown suppresses repeated sends + +The evaluator MUST NOT enqueue a new notification for a rule until the cooldown period from the previous firing has expired. + +#### Scenario: Active cooldown prevents re-enqueue + +Given rule has fired +And cooldown has not expired +When threshold is crossed again +Then no new work item is enqueued. + +### Requirement: Snooze suppresses sends + +The evaluator MUST NOT fire a snoozed rule until the snooze period expires. + +#### Scenario: Snoozed rule does not fire + +Given rule snoozed_until_utc is in the future +When threshold is crossed +Then no work item is enqueued. + +### Requirement: Snooze resumes from a fresh baseline + +When a snoozed rule resumes, the evaluator MUST ignore activity observed entirely during the snooze window so the rule does not back-alert immediately after unsnooze or natural expiry. + +#### Scenario: Unsnoozing does not immediately fire on snoozed activity + +Given a rule was snoozed while matching events continued +And the shared subject counter remained active for another enabled rule +When the user unsnoozes the rule +Then the evaluator does not enqueue a rate notification until new post-unsnooze activity crosses the threshold. + +#### Scenario: Snooze expiry does not immediately fire on snoozed activity + +Given a rule remained snoozed until its snooze window expired +And matching activity during the snooze window crossed the threshold +When the evaluator next runs after the snooze expires +Then the evaluator does not enqueue a rate notification until new post-expiry activity crosses the threshold. + +### Requirement: Disabled rules do not fire + +The evaluator MUST skip disabled rules during evaluation. + +#### Scenario: Disabled rule is skipped + +Given rule is disabled +When threshold is crossed +Then no work item is enqueued. + +### Requirement: No healthy/no-activity emails + +The system MUST NOT send notifications indicating that everything is fine or that no activity occurred. + +#### Scenario: Below threshold produces no notification + +Given threshold is not crossed +When evaluator runs +Then no healthy/no-activity notification is sent. + +### Requirement: Delivery sends email for valid notification + +The delivery job MUST send a rate notification email when the rule, project, user, and email state are all valid. + +#### Scenario: Valid state delivers email + +Given RateNotification work item +And rule/project/user are valid +And user email is verified +And user email notifications are enabled +When delivery job processes item +Then rate notification email is queued. + +### Requirement: Delivery skips invalid user state + +The delivery job MUST skip sending and log structured reasons when any precondition is not met. + +#### Scenario: Unverified email is skipped + +Given user email is unverified +When delivery job processes item +Then notification is skipped with log. + +#### Scenario: Notifications disabled is skipped + +Given user email notifications are disabled +When delivery job processes item +Then notification is skipped with log. + +#### Scenario: User not in organization is skipped + +Given user no longer belongs to organization +When delivery job processes item +Then notification is skipped with log. + +#### Scenario: Deleted project is skipped + +Given project is deleted +When delivery job processes item +Then notification is skipped with log. + +#### Scenario: Deleted rule is skipped + +Given rule is deleted +When delivery job processes item +Then notification is skipped with log. + +#### Scenario: Disabled rule is skipped during delivery + +Given rule is disabled +When delivery job processes item +Then notification is skipped with log. + +#### Scenario: Stale rule version is skipped + +Given rule version does not match work item +When delivery job processes item +Then notification is skipped with log. + +### Requirement: Email includes actionable context + +Rate notification emails MUST include all information the user needs to understand and act on the alert. + +#### Scenario: Email body contains all required fields + +Given notification email is queued +Then email includes rule name, project name, observed count, threshold, window, subject type, stack title (when applicable), link to project or stack, and cooldown explanation. + +#### Scenario: Stack-scoped email includes stack context + +Given a stack-scoped rate notification email is queued +When the delivery job loads the stack context +Then the email includes the stack title and a deep link to the stack. + +### Requirement: User can manage project rate rules in Svelte UI + +The Svelte UI MUST provide a full CRUD interface for rate notification rules within project settings. + +#### Scenario: Full CRUD and controls available + +Given user opens project notification settings +Then they can list, create, edit, delete, enable/disable, snooze, and unsnooze rate rules. + +#### Scenario: Non-premium organizations show rate notifications as unavailable + +Given the organization does not have premium features +When the user opens project notification settings +Then the UI shows the feature as unavailable +And the user cannot create or enable active rate notification rules. + +### Requirement: UI avoids advanced/deferred features + +The v1 UI MUST NOT expose features that are deferred to future iterations. + +#### Scenario: Deferred features are not exposed + +Given user opens rate notification management UI +Then UI does not expose digests, webhooks, automated actions, quiet hours, arbitrary filters, external recipients, or no-data alerts. + +### Requirement: Rate notification hot path is cache-only + +The event pipeline counter path MUST use only cache operations and MUST NOT query Elasticsearch per event. + +#### Scenario: No Elasticsearch per event + +Given event is processed +Then rate notification countering does not query Elasticsearch per event. + +### Requirement: Production requires distributed cache/queue + +Production deployments MUST use provider-backed (Redis/Azure/SQS) cache and queues for rate notification infrastructure. + +#### Scenario: Production uses provider-backed infrastructure + +Given production deployment +Then rate notification counters and queues use provider-backed cache/queues, not in-memory providers. + +### Requirement: Rollback does not affect existing notifications + +Disabling rate notification components MUST NOT impact existing event notification or daily summary behavior. + +#### Scenario: Disabling rate notifications leaves existing behavior intact + +Given rate notification evaluator/action is disabled +Then existing event notifications and daily summaries continue to operate unchanged. + +### Requirement: Rule lifecycle cleanup removes orphaned rules + +The system MUST remove or invalidate orphaned rate notification rules when the owning membership, project, or organization is deleted. + +#### Scenario: Membership removal cleans up user rules + +Given a user is removed from the organization +When cleanup runs +Then that user's rate notification rules for the organization are removed or invalidated +And cached rule indexes are invalidated. + +#### Scenario: Project deletion cleans up project rules + +Given a project is deleted +When cleanup runs +Then rate notification rules for that project are removed or invalidated +And cached rule indexes are invalidated. diff --git a/openspec/changes/add-personal-rate-notifications/tasks.md b/openspec/changes/add-personal-rate-notifications/tasks.md new file mode 100644 index 0000000000..2744258d0a --- /dev/null +++ b/openspec/changes/add-personal-rate-notifications/tasks.md @@ -0,0 +1,163 @@ +# Tasks: Add Personal Rate Notifications + +## Backend - Models and Repositories + +- [ ] **Add RateNotificationRule model and enums** + - Create `RateNotificationRule` model with all fields (Id, OrganizationId, ProjectId, UserId, Version, Name, IsEnabled, Signal, Subject, StackId, Threshold, Window, Cooldown, SnoozedUntilUtc, CreatedUtc, UpdatedUtc, IsDeleted) + - Create `RateNotificationSignal` enum (AllEvents, Errors, CriticalErrors, NewErrors, Regressions) + - Create `RateNotificationSubject` enum (Project, Stack) + +- [ ] **Add IRateNotificationRuleRepository and implementation** + - Create interface extending `IRepositoryOwnedByOrganizationAndProject` + - Add methods: `GetByProjectIdAndUserIdAsync`, `GetEnabledByProjectIdAsync`, `CountByProjectIdAndUserIdAsync` + - Create Elasticsearch-backed implementation following existing repository patterns + +- [ ] **Register repository in DI** + - Register `IRateNotificationRuleRepository` / `RateNotificationRuleRepository` in `Bootstrapper.cs` + +## Backend - Countering and Evaluation + +- [ ] **Add RateNotificationRuleIndex service** + - Load enabled rules for a project from repository + - Cache compiled counter definitions with short TTL + - Invalidate on rule create/update/delete/snooze/unsnooze + - Cache key: `rate:v1:rules:project:{projectId}` + +- [ ] **Add RateCounterService** + - Implement 1-minute UTC bucket counters via `ICacheClient` + - Methods: increment counter, sum buckets for window, check/set cooldown + - Counter key format: `rate:v1:count:{epochMinute}:{counterKey}` + - Active bucket tracking: `rate:v1:active:{epochMinute}` + - Cooldown key format: `rate:v1:cooldown:{ruleId}:{subjectKey}` + - TTLs: counter/active = 3h, cooldown = cooldown + 10m + +- [ ] **Add UpdateRateCountersAction (event pipeline)** + - Priority after stack assignment (e.g., 75) + - Exit fast if organization lacks premium features + - Load rule index for project; exit fast if no enabled rules + - Skip events on stacks where `AllowNotifications` is false + - Skip canceled/discarded events and requests already marked as bots + - Match event against compiled counter definitions (signal matching) + - Increment matching counters; add to active bucket set + - Never query Elasticsearch per event; never send notifications directly + +- [ ] **Add RateNotificationEvaluatorJob** + - Periodic job (60s interval) with distributed lock + - Skip organizations without premium features + - Inspect recently active counters + - Sum buckets for each rule's window using a fresh baseline after snooze/unsnooze + - Skip disabled/snoozed rules + - Compare observed ≥ threshold; enforce cooldown per rule+subject + - Enqueue `RateNotification` work item on threshold crossing + - Set cooldown key on successful enqueue + - Structured logging for fired/skipped reasons + +- [ ] **Add RateNotification queue model** + - Fields: RuleId, RuleVersion, OrganizationId, ProjectId, UserId, SubjectKey, StackId, WindowStartUtc, WindowEndUtc, ObservedCount, Threshold + +- [ ] **Register counter/evaluator services in DI** + - Register `RateNotificationRuleIndex`, `RateCounterService`, `RateNotificationEvaluatorJob` + - Register `IQueue` in Core and Insulation bootstrappers + +## Backend - Delivery + +- [ ] **Add RateNotificationsJob (delivery)** + - Load rule, project, user + - Load stack for stack-scoped rules + - Validate: rule exists, enabled, version compatible, user in org, email verified, notifications enabled, project/org exists + - Send email via `IMailer.SendRateNotificationAsync` + - Skip with structured logs on validation failure + +- [ ] **Add IMailer.SendRateNotificationAsync** + - Add method to `IMailer` interface and `Mailer` implementation + - Email includes: rule name, project name, observed count, threshold, window, subject type, stack title (when applicable), link, cooldown explanation + - Subject format: `[ProjectName] Error rate exceeded` + - No "everything is fine" messaging + +- [ ] **Update Insulation queue registration** + - Register `IQueue` for Redis/Azure/SQS providers in `Exceptionless.Insulation/Bootstrapper.cs` + +## Backend - API + +- [ ] **Add RateNotificationRuleController** + - Routes: GET list, POST create, GET by id, PUT update, DELETE, POST snooze, POST unsnooze + - Authorization: current user manages own rules; global admin manages any user's rules; user must access project org + - Validation: threshold > 0, supported windows, cooldown ≥ window, subject/stack consistency, max 20 rules per user per project, name non-empty and ≤ 100 chars + - Preserve persisted rules across plan downgrade, but do not allow the runtime or Svelte UI to treat non-premium orgs as active rate-notification senders + +- [ ] **Add request/response DTOs** + - Create/update request models with validation attributes + - Snooze request model (duration or until timestamp) + +- [ ] **Update tests/http files** + - Add HTTP sample requests for all rate notification endpoints + +## Frontend + +- [ ] **Add rate-notifications feature module** + - Create `src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/` + - Models/types matching API DTOs + - TanStack Query API wrappers (list, create, update, delete, snooze, unsnooze) + +- [ ] **Add rate notification rule list component** + - List rules for current user/project + - Show enable/disable toggle, snooze status + - Show premium-gated disabled state / upgrade messaging when org lacks premium features + - Delete confirmation + +- [ ] **Add rate notification rule form component** + - Fields: Name, Signal, Subject, Stack selector, Threshold, Window, Cooldown, Enabled + - Validation matching backend constraints + - Noise warning copy: "This rule may be noisy. Use a cooldown to avoid repeated emails." + +- [ ] **Integrate into project settings** + - Add rate notifications section/tab in project notification settings + - Route and navigation entry + +## Tests + +- [ ] **Add unit tests** + - Rule validation (threshold, window, cooldown, subject/stack, name) + - Counter key builder + - Counter increments and bucket summing + - Signal matching logic + - Premium gating + - `AllowNotifications` / bot suppression + - Cooldown behavior + - Snooze behavior + - Fresh-baseline behavior after snooze/unsnooze + - Evaluator threshold crossing + +- [ ] **Add integration tests** + - User CRUD own rules (list, create, get, update, delete) + - User cannot manage another user's rules + - Global admin can manage another user's rules + - Stack rule rejects stack from another project + - Non-premium organizations do not counter, evaluate, or deliver rate notifications + - Events on stacks with `AllowNotifications = false` do not increment rate counters + - Bot-marked requests do not increment rate counters + - Evaluator enqueues when threshold crossed + - Evaluator does not enqueue below threshold + - Evaluator respects cooldown + - Evaluator respects snooze + - Activity gathered during snooze does not fire immediately when the rule resumes + - Delivery skips disabled rule + - Delivery skips unverified email + - Delivery skips user not in org + - Delivery sends email for valid rule + - Delivery loads stack context for stack-scoped emails + - Membership/project/org cleanup removes orphaned rules + +- [ ] **Add lifecycle cleanup** + - Remove/invalidate rules when user membership changes + - Remove/invalidate rules when project or organization is deleted + +## Validation + +- [ ] **Run OpenSpec validation** + - `openspec validate add-personal-rate-notifications --strict` + +- [ ] **Run relevant builds and tests** + - `dotnet build` + - `dotnet test` + - Frontend build and lint diff --git a/src/Exceptionless.Core/Bootstrapper.cs b/src/Exceptionless.Core/Bootstrapper.cs index 334f2b07ae..cebbaf3b6e 100644 --- a/src/Exceptionless.Core/Bootstrapper.cs +++ b/src/Exceptionless.Core/Bootstrapper.cs @@ -128,6 +128,7 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO services.AddSingleton(s => CreateQueue(s)); services.AddSingleton(s => CreateQueue(s)); services.AddSingleton(s => CreateQueue(s, TimeSpan.FromHours(1))); + services.AddSingleton(s => CreateQueue(s)); services.TryAddEnumerable(ServiceDescriptor.Singleton, WorkItemDuplicateDetectionQueueBehavior>()); @@ -163,6 +164,7 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); @@ -205,6 +207,8 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO services.AddSingleton(); services.AddStartupAction(); services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); @@ -275,12 +279,14 @@ public static void AddHostedJobs(IServiceCollection services, ILoggerFactory log services.AddJob(o => o.WaitForStartupActions()); services.AddJob(o => o.WaitForStartupActions()); services.AddJob(o => o.WaitForStartupActions()); + services.AddJob(o => o.WaitForStartupActions()); services.AddJob(o => o.WaitForStartupActions()); services.AddJob(o => o.WaitForStartupActions()); services.AddJob(o => o.WaitForStartupActions()); services.AddJob(o => o.WaitForStartupActions()); services.AddDistributedCronJob(Cron.Minutely()); + services.AddDistributedCronJob(Cron.Minutely()); services.AddDistributedCronJob("30 */4 * * *"); services.AddDistributedCronJob("45 */8 * * *"); services.AddDistributedCronJob(Cron.Daily(1)); diff --git a/src/Exceptionless.Core/Exceptionless.Core.csproj b/src/Exceptionless.Core/Exceptionless.Core.csproj index 2ee31516e5..af25f9a2c5 100644 --- a/src/Exceptionless.Core/Exceptionless.Core.csproj +++ b/src/Exceptionless.Core/Exceptionless.Core.csproj @@ -6,6 +6,7 @@ + @@ -16,6 +17,7 @@ + diff --git a/src/Exceptionless.Core/Jobs/RateNotificationEvaluatorJob.cs b/src/Exceptionless.Core/Jobs/RateNotificationEvaluatorJob.cs new file mode 100644 index 0000000000..4f97835c30 --- /dev/null +++ b/src/Exceptionless.Core/Jobs/RateNotificationEvaluatorJob.cs @@ -0,0 +1,224 @@ +using Exceptionless.Core.Models; +using Exceptionless.Core.Pipeline; +using Exceptionless.Core.Queues.Models; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Services; +using Foundatio.Jobs; +using Foundatio.Lock; +using Foundatio.Queues; +using Foundatio.Repositories; +using Foundatio.Resilience; +using Microsoft.Extensions.Logging; + +namespace Exceptionless.Core.Jobs; + +[Job(Description = "Evaluates rate notification rules and enqueues notifications.", IsContinuous = false)] +public class RateNotificationEvaluatorJob : JobWithLockBase +{ + private readonly RateCounterService _counterService; + private readonly IRateNotificationRuleRepository _ruleRepository; + private readonly IOrganizationRepository _organizationRepository; + private readonly IQueue _notificationQueue; + private readonly ILockProvider _lockProvider; + + /// How far back to scan for active counter minutes. + private static readonly TimeSpan ScanWindow = TimeSpan.FromHours(2); + + public RateNotificationEvaluatorJob( + RateCounterService counterService, + IRateNotificationRuleRepository ruleRepository, + IOrganizationRepository organizationRepository, + IQueue notificationQueue, + ILockProvider lockProvider, + TimeProvider timeProvider, + IResiliencePolicyProvider resiliencePolicyProvider, + ILoggerFactory loggerFactory) : base(timeProvider, resiliencePolicyProvider, loggerFactory) + { + _counterService = counterService; + _ruleRepository = ruleRepository; + _organizationRepository = organizationRepository; + _notificationQueue = notificationQueue; + _lockProvider = lockProvider; + } + + protected override Task GetLockAsync(CancellationToken cancellationToken = default) + { + return _lockProvider.TryAcquireAsync(nameof(RateNotificationEvaluatorJob), TimeSpan.FromMinutes(2), cancellationToken); + } + + protected override async Task RunInternalAsync(JobContext context) + { + var now = _timeProvider.GetUtcNow().UtcDateTime; + var scanFrom = now.Subtract(ScanWindow); + + // Round scanFrom down to minute boundary; stop 1 minute before now (current bucket may be incomplete) + var fromMinute = new DateTime(scanFrom.Year, scanFrom.Month, scanFrom.Day, scanFrom.Hour, scanFrom.Minute, 0, DateTimeKind.Utc); + var toMinute = new DateTime(now.Year, now.Month, now.Day, now.Hour, now.Minute, 0, DateTimeKind.Utc).AddMinutes(-1); + + if (fromMinute > toMinute) + return JobResult.Success; + + _logger.LogInformation("Evaluating rate notification rules from {From} to {To}", fromMinute, toMinute); + + // Collect all unique counter keys across all minutes in the scan window + var allCounterKeys = new HashSet(); + for (var minute = fromMinute; minute <= toMinute; minute = minute.AddMinutes(1)) + { + var keys = await _counterService.GetActiveCounterKeysAsync(minute, context.CancellationToken); + foreach (var key in keys) + allCounterKeys.Add(key); + } + + if (allCounterKeys.Count == 0) + { + _logger.LogDebug("No active counter keys found in scan window"); + return JobResult.Success; + } + + // For each unique counter key, evaluate all matching rules + foreach (string counterKey in allCounterKeys) + { + if (context.CancellationToken.IsCancellationRequested) + return JobResult.Cancelled; + + await EvaluateCounterKeyAsync(counterKey, now, context.CancellationToken); + } + + _logger.LogInformation("Finished evaluating rate notification rules"); + return JobResult.Success; + } + + private async Task EvaluateCounterKeyAsync(string counterKey, DateTime now, CancellationToken ct) + { + // Parse projectId from counter key to load rules + string? projectId = ParseProjectIdFromCounterKey(counterKey); + if (projectId is null) + { + _logger.LogWarning("Unable to parse projectId from counter key: {CounterKey}", counterKey); + return; + } + + // Load enabled rules for project matching this counter key + var allProjectRules = await _ruleRepository.GetEnabledByProjectIdAsync(projectId, o => o.PageLimit(1000)); + + // Filter rules matching this counter key + var matchingRules = allProjectRules.Documents + .Where(r => CounterKeyMatchesRule(counterKey, r)) + .ToList(); + + if (matchingRules.Count == 0) + return; + + // Load org once per project for premium check + string? organizationId = matchingRules.First().OrganizationId; + var org = await _organizationRepository.GetByIdAsync(organizationId, o => o.Cache()); + if (org is null) + return; + + if (!org.HasPremiumFeatures) + return; + + foreach (var rule in matchingRules) + { + if (ct.IsCancellationRequested) + return; + + await EvaluateRuleAsync(rule, counterKey, now, ct); + } + } + + private async Task EvaluateRuleAsync(RateNotificationRule rule, string counterKey, DateTime now, CancellationToken ct) + { + if (!rule.IsEnabled) + return; + + // Skip if actively snoozed + if (rule.SnoozedUntilUtc.HasValue && rule.SnoozedUntilUtc.Value > now) + { + _logger.LogDebug("Skipping snoozed rule {RuleId} snoozed until {SnoozedUntil}", rule.Id, rule.SnoozedUntilUtc); + return; + } + + var windowStartUtc = now.Subtract(rule.Window); + + // SNOOZE BACK-ALERT FIX: + // If the rule was recently un-snoozed (SnoozedUntilUtc is set and in the past), use that as the + // effective window start to ignore traffic that occurred during the snooze period. + var effectiveWindowStartUtc = rule.SnoozedUntilUtc.HasValue && rule.SnoozedUntilUtc.Value > windowStartUtc + ? rule.SnoozedUntilUtc.Value + : windowStartUtc; + + var observedCount = await _counterService.SumBucketsAsync(counterKey, effectiveWindowStartUtc, now, ct); + + if (observedCount < rule.Threshold) + { + _logger.LogDebug("Rule {RuleId}: observed={Observed} < threshold={Threshold}, skipping", rule.Id, observedCount, rule.Threshold); + return; + } + + // Build subject key (for cooldown scoping) + string subjectKey = BuildSubjectKey(rule); + + // Check cooldown + if (await _counterService.IsOnCooldownAsync(rule.Id, subjectKey, ct)) + { + _logger.LogDebug("Rule {RuleId} is on cooldown for subject {SubjectKey}", rule.Id, subjectKey); + return; + } + + // Enqueue notification + await _notificationQueue.EnqueueAsync(new RateNotification + { + RuleId = rule.Id, + RuleVersion = rule.Version, + OrganizationId = rule.OrganizationId, + ProjectId = rule.ProjectId, + UserId = rule.UserId, + SubjectKey = subjectKey, + StackId = rule.StackId, + WindowStartUtc = effectiveWindowStartUtc, + WindowEndUtc = now, + ObservedCount = observedCount, + Threshold = rule.Threshold + }); + + // Set cooldown + await _counterService.SetCooldownAsync(rule.Id, subjectKey, rule.Cooldown, ct); + + // Update LastFiredUtc + rule.LastFiredUtc = now; + await _ruleRepository.SaveAsync(rule); + + _logger.LogInformation("Rate notification fired: rule={RuleId} project={ProjectId} observed={Observed} threshold={Threshold}", + rule.Id, rule.ProjectId, observedCount, rule.Threshold); + } + + /// Parses the projectId from a counter key of the form: project:{projectId}:... + private static string? ParseProjectIdFromCounterKey(string counterKey) + { + const string prefix = "project:"; + if (!counterKey.StartsWith(prefix)) + return null; + + int start = prefix.Length; + int end = counterKey.IndexOf(':', start); + if (end < 0) + return null; + + return counterKey[start..end]; + } + + /// Returns true if the counter key was generated by the given rule. + private static bool CounterKeyMatchesRule(string counterKey, RateNotificationRule rule) + { + string expected = UpdateRateCountersAction.BuildCounterKey(rule); + return String.Equals(counterKey, expected, StringComparison.Ordinal); + } + + private static string BuildSubjectKey(RateNotificationRule rule) + { + return rule.Subject == RateNotificationSubject.Stack && !String.IsNullOrEmpty(rule.StackId) + ? $"stack:{rule.StackId}" + : $"project:{rule.ProjectId}"; + } +} diff --git a/src/Exceptionless.Core/Jobs/RateNotificationsJob.cs b/src/Exceptionless.Core/Jobs/RateNotificationsJob.cs new file mode 100644 index 0000000000..253ec202de --- /dev/null +++ b/src/Exceptionless.Core/Jobs/RateNotificationsJob.cs @@ -0,0 +1,105 @@ +using Exceptionless.Core.Mail; +using Exceptionless.Core.Models; +using Exceptionless.Core.Queues.Models; +using Exceptionless.Core.Repositories; +using Foundatio.Jobs; +using Foundatio.Queues; +using Foundatio.Repositories; +using Foundatio.Resilience; +using Microsoft.Extensions.Logging; + +namespace Exceptionless.Core.Jobs; + +[Job(Description = "Delivers rate notification emails.", InitialDelay = "5s")] +public class RateNotificationsJob : QueueJobBase +{ + private readonly IMailer _mailer; + private readonly IRateNotificationRuleRepository _ruleRepository; + private readonly IProjectRepository _projectRepository; + private readonly IUserRepository _userRepository; + private readonly IStackRepository _stackRepository; + + public RateNotificationsJob( + IQueue queue, + IMailer mailer, + IRateNotificationRuleRepository ruleRepository, + IProjectRepository projectRepository, + IUserRepository userRepository, + IStackRepository stackRepository, + TimeProvider timeProvider, + IResiliencePolicyProvider resiliencePolicyProvider, + ILoggerFactory loggerFactory) : base(queue, timeProvider, resiliencePolicyProvider, loggerFactory) + { + _mailer = mailer; + _ruleRepository = ruleRepository; + _projectRepository = projectRepository; + _userRepository = userRepository; + _stackRepository = stackRepository; + } + + protected override async Task ProcessQueueEntryAsync(QueueEntryContext context) + { + var wi = context.QueueEntry.Value; + + // Load rule + var rule = await _ruleRepository.GetByIdAsync(wi.RuleId); + if (rule is null) + return JobResult.SuccessWithMessage($"Rate notification rule {wi.RuleId} not found; skipping."); + + if (!rule.IsEnabled) + return JobResult.SuccessWithMessage($"Rule {wi.RuleId} is disabled; skipping."); + + // Version check — rule was mutated after enqueueing + if (rule.Version != wi.RuleVersion) + { + _logger.LogInformation("Rule {RuleId} version mismatch: expected {Expected}, found {Actual}; skipping stale notification", + wi.RuleId, wi.RuleVersion, rule.Version); + return JobResult.Success; + } + + // Load project + var project = await _projectRepository.GetByIdAsync(rule.ProjectId, o => o.Cache()); + if (project is null) + return JobResult.SuccessWithMessage($"Project {rule.ProjectId} not found; skipping."); + + // Load user + var user = await _userRepository.GetByIdAsync(rule.UserId, o => o.Cache()); + if (user is null) + return JobResult.SuccessWithMessage($"User {rule.UserId} not found; skipping."); + + // User must still be a member of the org + if (!user.OrganizationIds.Contains(rule.OrganizationId)) + { + _logger.LogInformation("User {UserId} is no longer a member of org {OrgId}; skipping rate notification", rule.UserId, rule.OrganizationId); + return JobResult.Success; + } + + if (!user.IsEmailAddressVerified) + { + _logger.LogInformation("User {UserId} email not verified; skipping rate notification", rule.UserId); + return JobResult.Success; + } + + if (!user.EmailNotificationsEnabled) + { + _logger.LogInformation("User {UserId} has email notifications disabled; skipping rate notification", rule.UserId); + return JobResult.Success; + } + + // Load stack if this is a stack-scoped rule + Stack? stack = null; + if (rule.Subject == RateNotificationSubject.Stack && !String.IsNullOrEmpty(rule.StackId)) + { + stack = await _stackRepository.GetByIdAsync(rule.StackId, o => o.Cache()); + if (stack is null) + _logger.LogWarning("Stack {StackId} not found for rate notification rule {RuleId}", rule.StackId, rule.Id); + } + + await _mailer.SendRateNotificationAsync(user, project, rule, wi.ObservedCount, wi.WindowStartUtc, wi.WindowEndUtc, stack); + + _logger.LogInformation("Sent rate notification email: rule={RuleId} user={UserId} project={ProjectId} observed={Observed}", + rule.Id, rule.UserId, rule.ProjectId, wi.ObservedCount); + + return JobResult.Success; + } +} diff --git a/src/Exceptionless.Core/Mail/IMailer.cs b/src/Exceptionless.Core/Mail/IMailer.cs index 79b3b7f39e..a8f1c1e2b8 100644 --- a/src/Exceptionless.Core/Mail/IMailer.cs +++ b/src/Exceptionless.Core/Mail/IMailer.cs @@ -10,6 +10,7 @@ public interface IMailer Task SendOrganizationNoticeAsync(User user, Organization organization, bool isOverMonthlyLimit, bool isOverHourlyLimit); Task SendOrganizationPaymentFailedAsync(User owner, Organization organization); Task SendProjectDailySummaryAsync(User user, Project project, IEnumerable? mostFrequent, IEnumerable? newest, DateTime startDate, bool hasSubmittedEvents, double count, double uniqueCount, double newCount, double fixedCount, int blockedCount, int tooBigCount, bool isFreePlan); + Task SendRateNotificationAsync(User user, Project project, RateNotificationRule rule, long observedCount, DateTime windowStart, DateTime windowEnd, Stack? stack = null); Task SendUserEmailVerifyAsync(User user); Task SendUserPasswordResetAsync(User user); } diff --git a/src/Exceptionless.Core/Mail/Mailer.cs b/src/Exceptionless.Core/Mail/Mailer.cs index b6498c1d45..16d6ba395e 100644 --- a/src/Exceptionless.Core/Mail/Mailer.cs +++ b/src/Exceptionless.Core/Mail/Mailer.cs @@ -281,6 +281,47 @@ public Task SendUserPasswordResetAsync(User user) }, template); } + public Task SendRateNotificationAsync(User user, Project project, RateNotificationRule rule, long observedCount, DateTime windowStart, DateTime windowEnd, Stack? stack = null) + { + const string template = "rate-notification"; + string windowDescription = FormatWindow(rule.Window); + + var data = new Dictionary { + { "Subject", $"Error rate exceeded: {rule.Name}" }, + { "BaseUrl", _appOptions.BaseURL }, + { "ProjectName", project.Name }, + { "ProjectId", project.Id }, + { "RuleName", rule.Name }, + { "ObservedCount", observedCount }, + { "Threshold", rule.Threshold }, + { "Window", windowDescription }, + { "Signal", rule.Signal.ToString() }, + { "Subject_Type", rule.Subject.ToString() }, + { "StackId", rule.StackId }, + { "StackTitle", stack?.Title }, + { "HasStack", stack is not null }, + { "WindowStartUtc", windowStart.ToString("u") }, + { "WindowEndUtc", windowEnd.ToString("u") }, + { "Cooldown", FormatWindow(rule.Cooldown) } + }; + + return QueueMessageAsync(new MailMessage + { + To = user.EmailAddress, + Subject = $"[{project.Name}] Error rate exceeded: {rule.Name}", + Body = RenderTemplate(template, data) + }, template); + } + + private static string FormatWindow(TimeSpan window) + { + if (window.TotalHours >= 1 && window.Minutes == 0) + return $"{(int)window.TotalHours}h"; + if (window.TotalMinutes >= 1 && window.Seconds == 0) + return $"{(int)window.TotalMinutes}min"; + return window.ToString(); + } + private string RenderTemplate(string name, IDictionary data) { var template = GetCompiledTemplate(name); diff --git a/src/Exceptionless.Core/Mail/Templates/rate-notification.html b/src/Exceptionless.Core/Mail/Templates/rate-notification.html new file mode 100644 index 0000000000..5b6da87dfb --- /dev/null +++ b/src/Exceptionless.Core/Mail/Templates/rate-notification.html @@ -0,0 +1 @@ +{{Subject}}
Exceptionless
 

Your rate notification rule “{{RuleName}}” has fired for the {{ProjectName}} project.

Events Observed
{{ObservedCount}} events (threshold: {{Threshold}}) in the last {{Window}}

Signal
{{Signal}}

Scope
{{Subject_Type}}{{#if HasStack}} — {{StackTitle}}{{/if}}

Window
{{WindowStartUtc}} – {{WindowEndUtc}}

{{#if HasStack}}View Stack {{else}}View Project Dashboard {{/if}}

This notification will not be sent again for another {{Cooldown}} (cooldown period).

diff --git a/src/Exceptionless.Core/Models/Enums/RateNotificationSignal.cs b/src/Exceptionless.Core/Models/Enums/RateNotificationSignal.cs new file mode 100644 index 0000000000..2e3e9e620b --- /dev/null +++ b/src/Exceptionless.Core/Models/Enums/RateNotificationSignal.cs @@ -0,0 +1,15 @@ +using System.Text.Json.Serialization; +using Newtonsoft.Json.Converters; + +namespace Exceptionless.Core.Models; + +[JsonConverter(typeof(JsonStringEnumConverter))] +[Newtonsoft.Json.JsonConverter(typeof(StringEnumConverter))] +public enum RateNotificationSignal +{ + AllEvents = 0, + Errors = 1, + CriticalErrors = 2, + NewErrors = 3, + Regressions = 4 +} diff --git a/src/Exceptionless.Core/Models/Enums/RateNotificationSubject.cs b/src/Exceptionless.Core/Models/Enums/RateNotificationSubject.cs new file mode 100644 index 0000000000..0e3f353c04 --- /dev/null +++ b/src/Exceptionless.Core/Models/Enums/RateNotificationSubject.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; +using Newtonsoft.Json.Converters; + +namespace Exceptionless.Core.Models; + +[JsonConverter(typeof(JsonStringEnumConverter))] +[Newtonsoft.Json.JsonConverter(typeof(StringEnumConverter))] +public enum RateNotificationSubject +{ + Project = 0, + Stack = 1 +} diff --git a/src/Exceptionless.Core/Models/Queues/RateNotification.cs b/src/Exceptionless.Core/Models/Queues/RateNotification.cs new file mode 100644 index 0000000000..55e81254ef --- /dev/null +++ b/src/Exceptionless.Core/Models/Queues/RateNotification.cs @@ -0,0 +1,16 @@ +namespace Exceptionless.Core.Queues.Models; + +public class RateNotification +{ + public required string RuleId { get; set; } + public required int RuleVersion { get; set; } + public required string OrganizationId { get; set; } + public required string ProjectId { get; set; } + public required string UserId { get; set; } + public required string SubjectKey { get; set; } + public string? StackId { get; set; } + public required DateTime WindowStartUtc { get; set; } + public required DateTime WindowEndUtc { get; set; } + public required long ObservedCount { get; set; } + public required int Threshold { get; set; } +} diff --git a/src/Exceptionless.Core/Models/RateNotificationRule.cs b/src/Exceptionless.Core/Models/RateNotificationRule.cs new file mode 100644 index 0000000000..0ffa6ad49a --- /dev/null +++ b/src/Exceptionless.Core/Models/RateNotificationRule.cs @@ -0,0 +1,55 @@ +using System.ComponentModel.DataAnnotations; +using Exceptionless.Core.Attributes; +using Foundatio.Repositories.Models; + +namespace Exceptionless.Core.Models; + +public class RateNotificationRule : IOwnedByOrganizationAndProjectWithIdentity, IHaveDates +{ + [ObjectId] + public string Id { get; set; } = null!; + + [Required] + [ObjectId] + public string OrganizationId { get; set; } = null!; + + [Required] + [ObjectId] + public string ProjectId { get; set; } = null!; + + [Required] + [ObjectId] + public string UserId { get; set; } = null!; + + public int Version { get; set; } = 1; + + [Required] + [MaxLength(100)] + public string Name { get; set; } = null!; + + public bool IsEnabled { get; set; } = true; + + public RateNotificationSignal Signal { get; set; } + + public RateNotificationSubject Subject { get; set; } + + [ObjectId] + public string? StackId { get; set; } + + [Range(1, int.MaxValue)] + public int Threshold { get; set; } = 10; + + public TimeSpan Window { get; set; } = TimeSpan.FromHours(1); + + public TimeSpan Cooldown { get; set; } = TimeSpan.FromHours(1); + + public DateTime? SnoozedUntilUtc { get; set; } + + public DateTime? LastFiredUtc { get; set; } + + public bool IsDeleted { get; set; } + + public DateTime CreatedUtc { get; set; } + + public DateTime UpdatedUtc { get; set; } +} diff --git a/src/Exceptionless.Core/Pipeline/075_UpdateRateCountersAction.cs b/src/Exceptionless.Core/Pipeline/075_UpdateRateCountersAction.cs new file mode 100644 index 0000000000..edbe7c4707 --- /dev/null +++ b/src/Exceptionless.Core/Pipeline/075_UpdateRateCountersAction.cs @@ -0,0 +1,105 @@ +using Exceptionless.Core.Extensions; +using Exceptionless.Core.Models; +using Exceptionless.Core.Plugins.EventProcessor; +using Exceptionless.Core.Services; +using Exceptionless.Core.Utility; +using Microsoft.Extensions.Logging; +using System.Text.Json; + +namespace Exceptionless.Core.Pipeline; + +[Priority(75)] +public class UpdateRateCountersAction : EventPipelineActionBase +{ + private readonly RateNotificationRuleCache _ruleCache; + private readonly RateCounterService _counterService; + private readonly UserAgentParser _parser; + private readonly JsonSerializerOptions _jsonOptions; + + public UpdateRateCountersAction( + RateNotificationRuleCache ruleCache, + RateCounterService counterService, + UserAgentParser parser, + JsonSerializerOptions jsonOptions, + AppOptions options, + ILoggerFactory loggerFactory) : base(options, loggerFactory) + { + _ruleCache = ruleCache; + _counterService = counterService; + _parser = parser; + _jsonOptions = jsonOptions; + ContinueOnError = true; + } + + public override async Task ProcessAsync(EventContext ctx) + { + // Premium gate — rate notifications require premium features + if (!ctx.Organization.HasPremiumFeatures) + return; + + // Stack must allow notifications + if (ctx.Stack is null || !ctx.Stack.AllowNotifications) + return; + + // Load enabled rules for this project + var rules = await _ruleCache.GetEnabledRulesAsync(ctx.Event.ProjectId); + if (rules.Count == 0) + return; + + // Bot check — same pattern as EventNotificationsJob + var request = ctx.Event.GetRequestInfo(_jsonOptions); + if (!String.IsNullOrEmpty(request?.UserAgent)) + { + var botPatterns = ctx.Project.Configuration.Settings + .GetStringCollection(SettingsDictionary.KnownKeys.UserAgentBotPatterns).ToList(); + var info = await _parser.ParseAsync(request.UserAgent); + if (info is not null && info.Device.IsSpider || request.UserAgent.AnyWildcardMatches(botPatterns)) + { + _logger.LogTrace("Skipping rate counter update for bot user agent: {UserAgent}", request.UserAgent); + return; + } + } + + // Build the set of signals matched by this event + bool isError = ctx.Event.IsError(); + bool isCritical = ctx.Event.IsCritical(); + var matchedSignals = new HashSet(); + + // AllEvents always matches + matchedSignals.Add(RateNotificationSignal.AllEvents); + + if (isError) + matchedSignals.Add(RateNotificationSignal.Errors); + + if (isError && isCritical) + matchedSignals.Add(RateNotificationSignal.CriticalErrors); + + if (ctx.IsNew && isError) + matchedSignals.Add(RateNotificationSignal.NewErrors); + + if (ctx.IsRegression) + matchedSignals.Add(RateNotificationSignal.Regressions); + + // Increment counters for each matching rule + foreach (var rule in rules.Where(r => matchedSignals.Contains(r.Signal))) + { + // For Stack subject, only match if this event belongs to the rule's stack + if (rule.Subject == RateNotificationSubject.Stack && + (String.IsNullOrEmpty(rule.StackId) || !String.Equals(ctx.Event.StackId, rule.StackId, StringComparison.Ordinal))) + continue; + + string counterKey = BuildCounterKey(rule); + await _counterService.IncrementAsync(counterKey); + } + } + + internal static string BuildCounterKey(RateNotificationRule rule) + { + return rule.Subject switch + { + RateNotificationSubject.Project => $"project:{rule.ProjectId}:signal:{rule.Signal}", + RateNotificationSubject.Stack => $"project:{rule.ProjectId}:stack:{rule.StackId}:signal:{rule.Signal}", + _ => $"project:{rule.ProjectId}:signal:{rule.Signal}" + }; + } +} diff --git a/src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs b/src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs index 2158adeade..127008a6dc 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs @@ -44,6 +44,7 @@ ILoggerFactory loggerFactory AddIndex(Migrations = new MigrationIndex(this, _appOptions.ElasticsearchOptions.ScopePrefix + "migrations", appOptions.ElasticsearchOptions.NumberOfReplicas)); AddIndex(Organizations = new OrganizationIndex(this)); AddIndex(Projects = new ProjectIndex(this)); + AddIndex(RateNotificationRules = new RateNotificationRuleIndex(this)); AddIndex(SavedViews = new SavedViewIndex(this)); AddIndex(Tokens = new TokenIndex(this)); AddIndex(Users = new UserIndex(this)); @@ -72,6 +73,7 @@ public override void ConfigureGlobalQueryBuilders(ElasticQueryBuilder builder) public MigrationIndex Migrations { get; } public OrganizationIndex Organizations { get; } public ProjectIndex Projects { get; } + public RateNotificationRuleIndex RateNotificationRules { get; } public SavedViewIndex SavedViews { get; } public TokenIndex Tokens { get; } public UserIndex Users { get; } diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/RateNotificationRuleIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/RateNotificationRuleIndex.cs new file mode 100644 index 0000000000..aab51d9184 --- /dev/null +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/RateNotificationRuleIndex.cs @@ -0,0 +1,43 @@ +using Exceptionless.Core.Models; +using Foundatio.Repositories.Elasticsearch.Configuration; +using Foundatio.Repositories.Elasticsearch.Extensions; +using Nest; + +namespace Exceptionless.Core.Repositories.Configuration; + +public sealed class RateNotificationRuleIndex : VersionedIndex +{ + private readonly ExceptionlessElasticConfiguration _configuration; + + public RateNotificationRuleIndex(ExceptionlessElasticConfiguration configuration) + : base(configuration, configuration.Options.ScopePrefix + "rate-notification-rules", 1) + { + _configuration = configuration; + } + + public override TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + { + return map + .Dynamic(false) + .Properties(p => p + .SetupDefaults() + .Keyword(f => f.Name(e => e.OrganizationId)) + .Keyword(f => f.Name(e => e.ProjectId)) + .Keyword(f => f.Name(e => e.UserId)) + .Keyword(f => f.Name(e => e.StackId)) + .Keyword(f => f.Name(e => e.Signal)) + .Keyword(f => f.Name(e => e.Subject)) + .Boolean(f => f.Name(e => e.IsEnabled)) + .Boolean(f => f.Name(e => e.IsDeleted)) + .Text(f => f.Name(e => e.Name).AddKeywordField()) + ); + } + + public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + { + return base.ConfigureIndex(idx.Settings(s => s + .NumberOfShards(_configuration.Options.NumberOfShards) + .NumberOfReplicas(_configuration.Options.NumberOfReplicas) + .Priority(5))); + } +} diff --git a/src/Exceptionless.Core/Repositories/Interfaces/IRateNotificationRuleRepository.cs b/src/Exceptionless.Core/Repositories/Interfaces/IRateNotificationRuleRepository.cs new file mode 100644 index 0000000000..f24ef6961f --- /dev/null +++ b/src/Exceptionless.Core/Repositories/Interfaces/IRateNotificationRuleRepository.cs @@ -0,0 +1,13 @@ +using Exceptionless.Core.Models; +using Foundatio.Repositories; +using Foundatio.Repositories.Models; + +namespace Exceptionless.Core.Repositories; + +public interface IRateNotificationRuleRepository : IRepositoryOwnedByOrganizationAndProject +{ + Task> GetByProjectIdAndUserIdAsync(string projectId, string userId, CommandOptionsDescriptor? options = null); + Task> GetEnabledByProjectIdAsync(string projectId, CommandOptionsDescriptor? options = null); + Task CountByProjectIdAndUserIdAsync(string projectId, string userId); + Task RemoveByProjectIdAndUserIdAsync(string projectId, string userId); +} diff --git a/src/Exceptionless.Core/Repositories/RateNotificationRuleRepository.cs b/src/Exceptionless.Core/Repositories/RateNotificationRuleRepository.cs new file mode 100644 index 0000000000..91998a0cdf --- /dev/null +++ b/src/Exceptionless.Core/Repositories/RateNotificationRuleRepository.cs @@ -0,0 +1,63 @@ +using Exceptionless.Core.Models; +using Exceptionless.Core.Repositories.Configuration; +using Exceptionless.Core.Validation; +using Foundatio.Repositories; +using Foundatio.Repositories.Models; +using Nest; + +namespace Exceptionless.Core.Repositories; + +public sealed class RateNotificationRuleRepository : RepositoryOwnedByOrganizationAndProject, IRateNotificationRuleRepository +{ + public RateNotificationRuleRepository(ExceptionlessElasticConfiguration configuration, MiniValidationValidator validator, AppOptions options) + : base(configuration.RateNotificationRules, validator, options) + { + } + + public Task> GetByProjectIdAndUserIdAsync(string projectId, string userId, CommandOptionsDescriptor? options = null) + { + ArgumentException.ThrowIfNullOrEmpty(projectId); + ArgumentException.ThrowIfNullOrEmpty(userId); + + return FindAsync(q => q + .Project(projectId) + .FieldEquals(r => r.UserId, userId) + .SortAscending(r => r.Name.Suffix("keyword")), options); + } + + public Task> GetEnabledByProjectIdAsync(string projectId, CommandOptionsDescriptor? options = null) + { + ArgumentException.ThrowIfNullOrEmpty(projectId); + + return FindAsync(q => q + .Project(projectId) + .FieldEquals(r => r.IsEnabled, true) + .FieldEquals(r => r.IsDeleted, false), options); + } + + public async Task CountByProjectIdAndUserIdAsync(string projectId, string userId) + { + ArgumentException.ThrowIfNullOrEmpty(projectId); + ArgumentException.ThrowIfNullOrEmpty(userId); + + return await CountAsync(q => q + .Project(projectId) + .FieldEquals(r => r.UserId, userId)); + } + + public async Task RemoveByProjectIdAndUserIdAsync(string projectId, string userId) + { + ArgumentException.ThrowIfNullOrEmpty(projectId); + ArgumentException.ThrowIfNullOrEmpty(userId); + + var results = await FindAsync(q => q + .Project(projectId) + .FieldEquals(r => r.UserId, userId), o => o.PageLimit(1000)); + + if (results.Total is 0) + return 0; + + await RemoveAsync(results.Documents); + return results.Total; + } +} diff --git a/src/Exceptionless.Core/Services/RateCounterService.cs b/src/Exceptionless.Core/Services/RateCounterService.cs new file mode 100644 index 0000000000..5294864e2e --- /dev/null +++ b/src/Exceptionless.Core/Services/RateCounterService.cs @@ -0,0 +1,100 @@ +using Foundatio.Caching; +using Microsoft.Extensions.Logging; + +namespace Exceptionless.Core.Services; + +/// +/// Cache-backed 1-minute bucket counter service for rate notification evaluation. +/// Keys: +/// rate:v1:count:{epochMinute}:{counterKey} — TTL 3h +/// rate:v1:active:{epochMinute} — TTL 3h (list of counter keys) +/// rate:v1:cooldown:{ruleId}:{subjectKey} — TTL = cooldown + 10min +/// +public class RateCounterService +{ + private readonly ICacheClient _cache; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + private static readonly TimeSpan BucketTtl = TimeSpan.FromHours(3); + + public RateCounterService(ICacheClient cache, TimeProvider timeProvider, ILoggerFactory loggerFactory) + { + _cache = cache; + _timeProvider = timeProvider; + _logger = loggerFactory.CreateLogger(); + } + + /// Increments the 1-minute bucket counter for the given counter key at the current UTC minute. + public async Task IncrementAsync(string counterKey, CancellationToken ct = default) + { + var now = _timeProvider.GetUtcNow().UtcDateTime; + long epochMinute = GetEpochMinute(now); + + string countKey = GetCountKey(epochMinute, counterKey); + await _cache.IncrementAsync(countKey, 1, BucketTtl); + + string activeKey = GetActiveKey(epochMinute); + await _cache.ListAddAsync(activeKey, counterKey, BucketTtl); + } + + /// Sums all 1-minute bucket counts for the given counter key in the range [fromUtc, toUtc]. + public async Task SumBucketsAsync(string counterKey, DateTime fromUtc, DateTime toUtc, CancellationToken ct = default) + { + long fromMinute = GetEpochMinute(fromUtc); + long toMinute = GetEpochMinute(toUtc); + + long total = 0; + for (long minute = fromMinute; minute <= toMinute; minute++) + { + string key = GetCountKey(minute, counterKey); + var value = await _cache.GetAsync(key); + if (value.HasValue) + total += value.Value; + } + + return total; + } + + /// Returns all counter keys that were active during the given minute. + public async Task> GetActiveCounterKeysAsync(DateTime minute, CancellationToken ct = default) + { + long epochMinute = GetEpochMinute(minute); + string activeKey = GetActiveKey(epochMinute); + + var result = await _cache.GetListAsync(activeKey); + if (!result.HasValue || result.Value is null) + return Array.Empty(); + + return result.Value.Where(k => k is not null).Distinct().ToList()!; + } + + /// Returns true if the rule/subject is currently on cooldown. + public Task IsOnCooldownAsync(string ruleId, string subjectKey, CancellationToken ct = default) + { + string key = GetCooldownKey(ruleId, subjectKey); + return _cache.ExistsAsync(key); + } + + /// Sets a cooldown for the given rule/subject combination. + public Task SetCooldownAsync(string ruleId, string subjectKey, TimeSpan duration, CancellationToken ct = default) + { + string key = GetCooldownKey(ruleId, subjectKey); + // TTL = duration + 10 minutes buffer + return _cache.SetAsync(key, true, duration.Add(TimeSpan.FromMinutes(10))); + } + + // ---- Key helpers ---- + + private static long GetEpochMinute(DateTime utc) + => (long)(utc - DateTime.UnixEpoch).TotalMinutes; + + private static string GetCountKey(long epochMinute, string counterKey) + => $"rate:v1:count:{epochMinute}:{counterKey}"; + + private static string GetActiveKey(long epochMinute) + => $"rate:v1:active:{epochMinute}"; + + private static string GetCooldownKey(string ruleId, string subjectKey) + => $"rate:v1:cooldown:{ruleId}:{subjectKey}"; +} diff --git a/src/Exceptionless.Core/Services/RateNotificationRuleCache.cs b/src/Exceptionless.Core/Services/RateNotificationRuleCache.cs new file mode 100644 index 0000000000..777b754ac7 --- /dev/null +++ b/src/Exceptionless.Core/Services/RateNotificationRuleCache.cs @@ -0,0 +1,53 @@ +using Exceptionless.Core.Models; +using Exceptionless.Core.Repositories; +using Foundatio.Caching; +using Foundatio.Repositories; +using Foundatio.Repositories.Models; +using Microsoft.Extensions.Logging; + +namespace Exceptionless.Core.Services; + +/// +/// Cache layer over IRateNotificationRuleRepository. +/// Cache key: rate:v1:rules:project:{projectId} TTL: 5 minutes +/// +public class RateNotificationRuleCache +{ + private readonly IRateNotificationRuleRepository _repository; + private readonly ICacheClient _cache; + private readonly ILogger _logger; + + private static readonly TimeSpan CacheTtl = TimeSpan.FromMinutes(5); + + public RateNotificationRuleCache(IRateNotificationRuleRepository repository, ICacheClient cache, ILoggerFactory loggerFactory) + { + _repository = repository; + _cache = cache; + _logger = loggerFactory.CreateLogger(); + } + + /// Returns all enabled, non-deleted rules for the given project (cached). + public async Task> GetEnabledRulesAsync(string projectId, CancellationToken ct = default) + { + string cacheKey = GetCacheKey(projectId); + + var cached = await _cache.GetAsync>(cacheKey); + if (cached.HasValue && cached.Value is not null) + return cached.Value; + + var results = await _repository.GetEnabledByProjectIdAsync(projectId, o => o.PageLimit(1000)); + var rules = results.Documents.ToList(); + + await _cache.SetAsync(cacheKey, rules, CacheTtl); + return rules; + } + + /// Invalidates the cache for the given project. + public Task InvalidateAsync(string projectId) + { + string cacheKey = GetCacheKey(projectId); + return _cache.RemoveAsync(cacheKey); + } + + private static string GetCacheKey(string projectId) => $"rate:v1:rules:project:{projectId}"; +} diff --git a/src/Exceptionless.Insulation/Bootstrapper.cs b/src/Exceptionless.Insulation/Bootstrapper.cs index 6174ca2cc3..05ed57b454 100644 --- a/src/Exceptionless.Insulation/Bootstrapper.cs +++ b/src/Exceptionless.Insulation/Bootstrapper.cs @@ -89,6 +89,7 @@ private static IHealthChecksBuilder RegisterHealthChecks(IServiceCollection serv .AddAutoNamedCheck>("EventUserDescriptions", "AllJobs") .AddAutoNamedCheck>("EventNotifications", "AllJobs") .AddAutoNamedCheck>("WebHooks", "AllJobs") + .AddAutoNamedCheck>("RateNotifications", "AllJobs") .AddAutoNamedCheck>("AllJobs") .AddAutoNamedCheck>("WorkItem", "AllJobs") @@ -159,6 +160,7 @@ private static void RegisterQueue(IServiceCollection container, QueueOptions opt container.ReplaceSingleton(s => CreateAzureStorageQueue(s, options)); container.ReplaceSingleton(s => CreateAzureStorageQueue(s, options)); container.ReplaceSingleton(s => CreateAzureStorageQueue(s, options)); + container.ReplaceSingleton(s => CreateAzureStorageQueue(s, options)); container.ReplaceSingleton(s => CreateAzureStorageQueue(s, options)); container.ReplaceSingleton(s => CreateAzureStorageQueue(s, options, workItemTimeout: TimeSpan.FromHours(1))); } @@ -168,6 +170,7 @@ private static void RegisterQueue(IServiceCollection container, QueueOptions opt container.ReplaceSingleton(s => CreateRedisQueue(s, options, runMaintenanceTasks)); container.ReplaceSingleton(s => CreateRedisQueue(s, options, runMaintenanceTasks)); container.ReplaceSingleton(s => CreateRedisQueue(s, options, runMaintenanceTasks)); + container.ReplaceSingleton(s => CreateRedisQueue(s, options, runMaintenanceTasks)); container.ReplaceSingleton(s => CreateRedisQueue(s, options, runMaintenanceTasks)); container.ReplaceSingleton(s => CreateRedisQueue(s, options, runMaintenanceTasks, workItemTimeout: TimeSpan.FromHours(1))); } @@ -177,6 +180,7 @@ private static void RegisterQueue(IServiceCollection container, QueueOptions opt container.ReplaceSingleton(s => CreateSQSQueue(s, options)); container.ReplaceSingleton(s => CreateSQSQueue(s, options)); container.ReplaceSingleton(s => CreateSQSQueue(s, options)); + container.ReplaceSingleton(s => CreateSQSQueue(s, options)); container.ReplaceSingleton(s => CreateSQSQueue(s, options)); container.ReplaceSingleton(s => CreateSQSQueue(s, options, workItemTimeout: TimeSpan.FromHours(1))); } diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/api.svelte.ts new file mode 100644 index 0000000000..f0224d51d2 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/api.svelte.ts @@ -0,0 +1,210 @@ +import type { + NewRateNotificationRule, + SnoozeRateNotificationRuleRequest, + UpdateRateNotificationRule, + ViewRateNotificationRule +} from '$features/rate-notifications/types'; + +import { accessToken } from '$features/auth/index.svelte'; +import { type FetchClientResponse, type ProblemDetails, useFetchClient } from '@exceptionless/fetchclient'; +import { createMutation, createQuery, QueryClient, useQueryClient } from '@tanstack/svelte-query'; + +export const queryKeys = { + create: (userId: string | undefined, projectId: string | undefined) => + [...queryKeys.list(userId, projectId), 'create'] as const, + delete: (userId: string | undefined, projectId: string | undefined, ruleId: string | undefined) => + [...queryKeys.id(userId, projectId, ruleId), 'delete'] as const, + id: (userId: string | undefined, projectId: string | undefined, ruleId: string | undefined) => + [...queryKeys.list(userId, projectId), ruleId] as const, + list: (userId: string | undefined, projectId: string | undefined) => + ['RateNotificationRule', userId, projectId] as const, + snooze: (userId: string | undefined, projectId: string | undefined, ruleId: string | undefined) => + [...queryKeys.id(userId, projectId, ruleId), 'snooze'] as const, + unsnooze: (userId: string | undefined, projectId: string | undefined, ruleId: string | undefined) => + [...queryKeys.id(userId, projectId, ruleId), 'unsnooze'] as const, + update: (userId: string | undefined, projectId: string | undefined, ruleId: string | undefined) => + [...queryKeys.id(userId, projectId, ruleId), 'update'] as const +}; + +export async function invalidateRateNotificationQueries( + queryClient: QueryClient, + userId: string | undefined, + projectId: string | undefined, + ruleId?: string | undefined +) { + if (ruleId) { + await queryClient.invalidateQueries({ queryKey: queryKeys.id(userId, projectId, ruleId) }); + } + await queryClient.invalidateQueries({ queryKey: queryKeys.list(userId, projectId) }); +} + +function ruleRoute(userId: string, projectId: string, ruleId?: string): string { + const base = `users/${userId}/projects/${projectId}/rate-notifications`; + return ruleId ? `${base}/${ruleId}` : base; +} + +// ---- List ---- + +export interface GetRuleListRequest { + params?: { limit?: number; page?: number }; + route: { projectId: string | undefined; userId: string | undefined }; +} + +export function getRateNotificationRulesQuery(request: GetRuleListRequest) { + const queryClient = useQueryClient(); + + return createQuery, ProblemDetails>(() => ({ + enabled: () => !!accessToken.current && !!request.route.userId && !!request.route.projectId, + onSuccess: (data: FetchClientResponse) => { + data.data?.forEach((rule) => { + queryClient.setQueryData(queryKeys.id(request.route.userId, request.route.projectId, rule.id), rule); + }); + }, + queryClient, + queryFn: async ({ signal }: { signal: AbortSignal }) => { + const client = useFetchClient(); + const response = await client.getJSON( + ruleRoute(request.route.userId!, request.route.projectId!), + { params: { limit: 50, ...request.params }, signal } + ); + return response; + }, + queryKey: [...queryKeys.list(request.route.userId, request.route.projectId), { params: request.params }] + })); +} + +// ---- Create ---- + +export interface CreateRuleRequest { + route: { projectId: string | undefined; userId: string | undefined }; +} + +export function createRateNotificationRule(request: CreateRuleRequest) { + const queryClient = useQueryClient(); + + return createMutation(() => ({ + enabled: () => !!accessToken.current && !!request.route.userId && !!request.route.projectId, + mutationFn: async (body: NewRateNotificationRule) => { + const client = useFetchClient(); + const response = await client.postJSON( + ruleRoute(request.route.userId!, request.route.projectId!), + body + ); + return response.data!; + }, + mutationKey: queryKeys.create(request.route.userId, request.route.projectId), + onSuccess: (rule: ViewRateNotificationRule) => { + queryClient.setQueryData(queryKeys.id(request.route.userId, request.route.projectId, rule.id), rule); + queryClient.invalidateQueries({ queryKey: queryKeys.list(request.route.userId, request.route.projectId) }); + } + })); +} + +// ---- Update ---- + +export interface UpdateRuleRequest { + route: { projectId: string | undefined; ruleId: string | undefined; userId: string | undefined }; +} + +export function updateRateNotificationRule(request: UpdateRuleRequest) { + const queryClient = useQueryClient(); + + return createMutation(() => ({ + enabled: () => + !!accessToken.current && !!request.route.userId && !!request.route.projectId && !!request.route.ruleId, + mutationFn: async (body: UpdateRateNotificationRule) => { + const client = useFetchClient(); + const response = await client.putJSON( + ruleRoute(request.route.userId!, request.route.projectId!, request.route.ruleId!), + body + ); + return response.data!; + }, + mutationKey: queryKeys.update(request.route.userId, request.route.projectId, request.route.ruleId), + onSuccess: (rule: ViewRateNotificationRule) => { + queryClient.setQueryData(queryKeys.id(request.route.userId, request.route.projectId, rule.id), rule); + queryClient.invalidateQueries({ queryKey: queryKeys.list(request.route.userId, request.route.projectId) }); + } + })); +} + +// ---- Delete ---- + +export interface DeleteRuleRequest { + route: { projectId: string | undefined; ruleId: string | undefined; userId: string | undefined }; +} + +export function deleteRateNotificationRule(request: DeleteRuleRequest) { + const queryClient = useQueryClient(); + + return createMutation(() => ({ + enabled: () => + !!accessToken.current && !!request.route.userId && !!request.route.projectId && !!request.route.ruleId, + mutationFn: async () => { + const client = useFetchClient(); + await client.delete(ruleRoute(request.route.userId!, request.route.projectId!, request.route.ruleId!), { + expectedStatusCodes: [204] + }); + }, + mutationKey: queryKeys.delete(request.route.userId, request.route.projectId, request.route.ruleId), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: queryKeys.list(request.route.userId, request.route.projectId) }); + } + })); +} + +// ---- Snooze ---- + +export interface SnoozeRuleRequest { + route: { projectId: string | undefined; ruleId: string | undefined; userId: string | undefined }; +} + +export function snoozeRateNotificationRule(request: SnoozeRuleRequest) { + const queryClient = useQueryClient(); + + return createMutation(() => ({ + enabled: () => + !!accessToken.current && !!request.route.userId && !!request.route.projectId && !!request.route.ruleId, + mutationFn: async (body: SnoozeRateNotificationRuleRequest) => { + const client = useFetchClient(); + await client.postJSON( + `${ruleRoute(request.route.userId!, request.route.projectId!, request.route.ruleId!)}/snooze`, + body, + { expectedStatusCodes: [200, 204] } + ); + }, + mutationKey: queryKeys.snooze(request.route.userId, request.route.projectId, request.route.ruleId), + onSuccess: () => { + queryClient.invalidateQueries({ + queryKey: queryKeys.id(request.route.userId, request.route.projectId, request.route.ruleId) + }); + queryClient.invalidateQueries({ queryKey: queryKeys.list(request.route.userId, request.route.projectId) }); + } + })); +} + +// ---- Unsnooze ---- + +export function unsnoozeRateNotificationRule(request: SnoozeRuleRequest) { + const queryClient = useQueryClient(); + + return createMutation(() => ({ + enabled: () => + !!accessToken.current && !!request.route.userId && !!request.route.projectId && !!request.route.ruleId, + mutationFn: async () => { + const client = useFetchClient(); + await client.postJSON( + `${ruleRoute(request.route.userId!, request.route.projectId!, request.route.ruleId!)}/unsnooze`, + {}, + { expectedStatusCodes: [200, 204] } + ); + }, + mutationKey: queryKeys.unsnooze(request.route.userId, request.route.projectId, request.route.ruleId), + onSuccess: () => { + queryClient.invalidateQueries({ + queryKey: queryKeys.id(request.route.userId, request.route.projectId, request.route.ruleId) + }); + queryClient.invalidateQueries({ queryKey: queryKeys.list(request.route.userId, request.route.projectId) }); + } + })); +} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/components/rate-notification-rule-form.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/components/rate-notification-rule-form.svelte new file mode 100644 index 0000000000..cca7f22d0c --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/components/rate-notification-rule-form.svelte @@ -0,0 +1,300 @@ + + +
{ e.preventDefault(); handleSubmit(); }}> + {#if !hasPremiumFeatures} + + + + Upgrade now to enable and create rate notification rules. + + + {/if} + + + + This rule may be noisy. Use a cooldown to avoid repeated emails. + + + +
+ + + {#if nameError} +

{nameError}

+ {/if} +
+ + +
+ + + + {SIGNAL_LABELS[signal]} + + + {#each Object.entries(SIGNAL_LABELS) as [value, label] (value)} + {label} + {/each} + + +
+ + +
+ + + + {subject} + + + Project + Stack + + +
+ + + {#if subject === 'Stack'} +
+ + + {#if stackIdError} +

{stackIdError}

+ {/if} +
+ {/if} + + +
+ + + {#if thresholdError} +

{thresholdError}

+ {/if} +
+ + +
+ + + + {WINDOW_OPTIONS.find((o) => o.value === window)?.label ?? window} + + + {#each WINDOW_OPTIONS as option (option.value)} + {option.label} + {/each} + + +
+ + +
+ + + + {WINDOW_OPTIONS.find((o) => o.value === cooldown)?.label ?? cooldown} + + + {#each WINDOW_OPTIONS as option (option.value)} + {option.label} + {/each} + + 2 hours + 4 hours + 8 hours + 24 hours + + + {#if cooldownError} +

{cooldownError}

+ {/if} + Further notifications for this rule are suppressed during the cooldown period. +
+ + +
+ + +
+ + {#if formError} +

{formError}

+ {/if} + +
+ {#if onCancel} + + {/if} + +
+
diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/components/rate-notification-rule-list.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/components/rate-notification-rule-list.svelte new file mode 100644 index 0000000000..afee9f76d5 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/components/rate-notification-rule-list.svelte @@ -0,0 +1,216 @@ + + +
+ + + {#if !hasPremiumFeatures} + + + + Upgrade now to enable personal rate notifications! + + + {/if} + + {#if listQuery.isLoading} +
+ {#each { length: 2 } as _} +
+ {/each} +
+ {:else if rules.length === 0} +
+ +

No rate notification rules yet.

+ {#if hasPremiumFeatures} + + {/if} +
+ {:else} +
+ {#each rules as rule (rule.id)} +
+
+
+ +
+ {SIGNAL_LABELS[rule.signal]} + + ≥{rule.threshold} in {formatWindow(rule.window)} + + {#if rule.is_snoozed} + + + Snoozed + + {/if} + {#if rule.subject === 'Stack' && rule.stack_id} + Stack-scoped + {/if} +
+
+
+ + toggleEnabled(rule, checked)} + aria-label={rule.is_enabled ? 'Disable rule' : 'Enable rule'} + /> + +
+
+
+ {/each} +
+ + {#if hasPremiumFeatures && rules.length < MAX_RULES_PER_PROJECT} + + {/if} + {/if} +
+ + + !open && (confirmDeleteRuleId = undefined)}> + + + Delete rule? + + This action cannot be undone. The rate notification rule will be permanently deleted. + + + + + + + + diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/index.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/index.ts new file mode 100644 index 0000000000..8e3c411379 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/index.ts @@ -0,0 +1,5 @@ +// Rate notifications feature barrel export +export { default as RateNotificationRuleForm } from './components/rate-notification-rule-form.svelte'; +export { default as RateNotificationRuleList } from './components/rate-notification-rule-list.svelte'; +export * from './api.svelte'; +export * from './types'; diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/types.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/types.ts new file mode 100644 index 0000000000..7f56f5bbbd --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/rate-notifications/types.ts @@ -0,0 +1,78 @@ +// Types matching the RateNotificationRule API DTOs + +export type RateNotificationSignal = 'AllEvents' | 'Errors' | 'CriticalErrors' | 'NewErrors' | 'Regressions'; +export type RateNotificationSubject = 'Project' | 'Stack'; + +export interface ViewRateNotificationRule { + id: string; + organization_id: string; + project_id: string; + user_id: string; + version: number; + name: string; + is_enabled: boolean; + signal: RateNotificationSignal; + subject: RateNotificationSubject; + stack_id?: string; + threshold: number; + /** ISO 8601 duration string (e.g. "00:05:00") */ + window: string; + /** ISO 8601 duration string */ + cooldown: string; + snoozed_until_utc?: string; + last_fired_utc?: string; + created_utc: string; + updated_utc: string; + /** Computed: snoozed_until_utc is in the future */ + is_snoozed: boolean; +} + +export interface NewRateNotificationRule { + name: string; + signal: RateNotificationSignal; + subject: RateNotificationSubject; + stack_id?: string; + threshold: number; + /** ISO 8601 duration string (e.g. "00:05:00") */ + window: string; + /** ISO 8601 duration string */ + cooldown: string; + is_enabled: boolean; +} + +export interface UpdateRateNotificationRule { + name?: string; + signal?: RateNotificationSignal; + subject?: RateNotificationSubject; + stack_id?: string; + threshold?: number; + window?: string; + cooldown?: string; + is_enabled?: boolean; +} + +export interface SnoozeRateNotificationRuleRequest { + duration_seconds?: number; + until_utc?: string; +} + +/** Friendly labels for signal enum values */ +export const SIGNAL_LABELS: Record = { + AllEvents: 'All Events', + CriticalErrors: 'Critical Errors', + Errors: 'Errors', + NewErrors: 'New Errors', + Regressions: 'Regressions' +}; + +/** Allowed window durations (as ISO 8601) mapped to friendly labels */ +export const WINDOW_OPTIONS: { label: string; value: string }[] = [ + { label: '1 minute', value: '00:01:00' }, + { label: '5 minutes', value: '00:05:00' }, + { label: '10 minutes', value: '00:10:00' }, + { label: '15 minutes', value: '00:15:00' }, + { label: '30 minutes', value: '00:30:00' }, + { label: '1 hour', value: '01:00:00' } +]; + +export const MAX_RULES_PER_PROJECT = 20; diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/account/notifications/+page.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/account/notifications/+page.svelte index 5c660266e7..c23b8fe355 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/account/notifications/+page.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/account/notifications/+page.svelte @@ -1,14 +1,18 @@
@@ -201,5 +230,41 @@ {emailNotificationsEnabled} hasPremiumFeatures={selectedProject.has_premium_features} /> + +
+

Rate Notifications

+ Get notified when event rates for this project exceed your custom thresholds. +
+ + {/if}
+ + !open && closeRateRuleDialog()}> + + + {editingRateRule ? 'Edit Rate Notification Rule' : 'Create Rate Notification Rule'} + + {editingRateRule ? 'Update the rule settings below.' : 'Configure when you want to receive an email notification based on event rates.'} + + + {#if selectedProject} + + {/if} + + diff --git a/src/Exceptionless.Web/Controllers/RateNotificationRuleController.cs b/src/Exceptionless.Web/Controllers/RateNotificationRuleController.cs new file mode 100644 index 0000000000..c4bf00aa0d --- /dev/null +++ b/src/Exceptionless.Web/Controllers/RateNotificationRuleController.cs @@ -0,0 +1,387 @@ +using Exceptionless.Core.Authorization; +using Exceptionless.Core.Models; +using Exceptionless.Core.Queries.Validation; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Services; +using Exceptionless.Web.Controllers; +using Exceptionless.Web.Extensions; +using Exceptionless.Web.Models; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Foundatio.Repositories; + +namespace Exceptionless.App.Controllers.API; + +/// +/// Personal rate notification rule management. +/// +[Route(API_PREFIX + "/users/{userId:objectid}/projects/{projectId:objectid}/rate-notifications")] +[Authorize(Policy = AuthorizationRoles.UserPolicy)] +public class RateNotificationRuleController : ExceptionlessApiController +{ + private const int MaxRulesPerUserPerProject = 20; + + private static readonly TimeSpan[] ValidWindows = + [ + TimeSpan.FromMinutes(1), + TimeSpan.FromMinutes(5), + TimeSpan.FromMinutes(10), + TimeSpan.FromMinutes(15), + TimeSpan.FromMinutes(30), + TimeSpan.FromHours(1) + ]; + + private readonly IRateNotificationRuleRepository _ruleRepository; + private readonly IProjectRepository _projectRepository; + private readonly IOrganizationRepository _organizationRepository; + private readonly IStackRepository _stackRepository; + private readonly RateNotificationRuleCache _ruleCache; + + public RateNotificationRuleController( + IRateNotificationRuleRepository ruleRepository, + IProjectRepository projectRepository, + IOrganizationRepository organizationRepository, + IStackRepository stackRepository, + RateNotificationRuleCache ruleCache, + TimeProvider timeProvider, + ILoggerFactory loggerFactory) : base(timeProvider) + { + _ruleRepository = ruleRepository; + _projectRepository = projectRepository; + _organizationRepository = organizationRepository; + _stackRepository = stackRepository; + _ruleCache = ruleCache; + } + + /// Get all rate notification rules for a user/project. + [HttpGet] + public async Task>> GetAsync( + string userId, + string projectId, + int page = 1, + int limit = 25) + { + if (!CanManage(userId)) + return NotFound(); + + var project = await GetProjectAndCheckAccessAsync(projectId); + if (project is null) + return NotFound(); + + page = GetPage(page); + limit = GetLimit(limit); + + var results = await _ruleRepository.GetByProjectIdAndUserIdAsync(projectId, userId, o => o.PageNumber(page).PageLimit(limit)); + var viewModels = results.Documents.Select(MapToView).ToList(); + return OkWithResourceLinks(viewModels, results.HasMore && !NextPageExceedsSkipLimit(page, limit), page, results.Total); + } + + /// Create a rate notification rule. + [HttpPost] + [Consumes("application/json")] + [ProducesResponseType(StatusCodes.Status201Created)] + public async Task> PostAsync( + string userId, + string projectId, + [FromBody] NewRateNotificationRule model) + { + if (!CanManage(userId)) + return NotFound(); + + var project = await GetProjectAndCheckAccessAsync(projectId); + if (project is null) + return NotFound(); + + // Validate window + if (!ValidWindows.Contains(model.Window)) + return ValidationProblem(detail: $"Window must be one of: {String.Join(", ", ValidWindows.Select(w => w.ToString()))}"); + + // Validate cooldown >= window + if (model.Cooldown < model.Window) + return ValidationProblem(detail: "Cooldown must be greater than or equal to Window."); + + // Validate subject / stackId + if (model.Subject == RateNotificationSubject.Stack) + { + if (String.IsNullOrEmpty(model.StackId)) + return ValidationProblem(detail: "StackId is required when Subject is Stack."); + + var stack = await _stackRepository.GetByIdAsync(model.StackId, o => o.Cache()); + if (stack is null || !String.Equals(stack.ProjectId, projectId, StringComparison.Ordinal)) + return ValidationProblem(detail: "The specified StackId does not belong to this project."); + } + else if (!String.IsNullOrEmpty(model.StackId)) + { + return ValidationProblem(detail: "StackId must be empty when Subject is Project."); + } + + // Enforce max rules limit + long count = await _ruleRepository.CountByProjectIdAndUserIdAsync(projectId, userId); + if (count >= MaxRulesPerUserPerProject) + return ValidationProblem(detail: $"Maximum of {MaxRulesPerUserPerProject} rate notification rules per user per project."); + + // Premium gate — non-premium users can create rules but they start disabled + var org = await _organizationRepository.GetByIdAsync(project.OrganizationId, o => o.Cache()); + bool isEnabled = model.IsEnabled; + if (!org?.HasPremiumFeatures ?? false) + isEnabled = false; + + var now = _timeProvider.GetUtcNow().UtcDateTime; + var rule = new RateNotificationRule + { + OrganizationId = project.OrganizationId, + ProjectId = projectId, + UserId = userId, + Name = model.Name, + IsEnabled = isEnabled, + Signal = model.Signal, + Subject = model.Subject, + StackId = model.StackId, + Threshold = model.Threshold, + Window = model.Window, + Cooldown = model.Cooldown, + Version = 1, + CreatedUtc = now, + UpdatedUtc = now + }; + + rule = await _ruleRepository.AddAsync(rule, o => o.Cache().ImmediateConsistency()); + await _ruleCache.InvalidateAsync(projectId); + + return Created(new Uri(Url.Link("GetRateNotificationRuleById", new { userId, projectId, ruleId = rule.Id })!, UriKind.RelativeOrAbsolute), MapToView(rule)); + } + + /// Get a specific rate notification rule. + [HttpGet("{ruleId:objectid}", Name = "GetRateNotificationRuleById")] + public async Task> GetByIdAsync(string userId, string projectId, string ruleId) + { + if (!CanManage(userId)) + return NotFound(); + + var rule = await GetRuleAndCheckAccessAsync(ruleId, userId, projectId); + if (rule is null) + return NotFound(); + + return Ok(MapToView(rule)); + } + + /// Update a rate notification rule. + [HttpPut("{ruleId:objectid}")] + [Consumes("application/json")] + public async Task> PutAsync( + string userId, + string projectId, + string ruleId, + [FromBody] UpdateRateNotificationRule model) + { + if (!CanManage(userId)) + return NotFound(); + + var project = await GetProjectAndCheckAccessAsync(projectId); + if (project is null) + return NotFound(); + + var rule = await GetRuleAndCheckAccessAsync(ruleId, userId, projectId); + if (rule is null) + return NotFound(); + + // Apply updates + if (model.Name is not null) + rule.Name = model.Name; + + if (model.Signal.HasValue) + rule.Signal = model.Signal.Value; + + if (model.Subject.HasValue) + rule.Subject = model.Subject.Value; + + if (model.Threshold.HasValue) + rule.Threshold = model.Threshold.Value; + + // StackId update + var newStackId = model.StackId; + var newSubject = rule.Subject; + + if (newSubject == RateNotificationSubject.Stack) + { + if (String.IsNullOrEmpty(newStackId)) + return ValidationProblem(detail: "StackId is required when Subject is Stack."); + + var stack = await _stackRepository.GetByIdAsync(newStackId, o => o.Cache()); + if (stack is null || !String.Equals(stack.ProjectId, projectId, StringComparison.Ordinal)) + return ValidationProblem(detail: "The specified StackId does not belong to this project."); + + rule.StackId = newStackId; + } + else + { + rule.StackId = null; + } + + if (model.Window.HasValue) + { + if (!ValidWindows.Contains(model.Window.Value)) + return ValidationProblem(detail: $"Window must be one of: {String.Join(", ", ValidWindows.Select(w => w.ToString()))}"); + rule.Window = model.Window.Value; + } + + if (model.Cooldown.HasValue) + rule.Cooldown = model.Cooldown.Value; + + if (rule.Cooldown < rule.Window) + return ValidationProblem(detail: "Cooldown must be greater than or equal to Window."); + + if (model.IsEnabled.HasValue) + { + var org = await _organizationRepository.GetByIdAsync(project.OrganizationId, o => o.Cache()); + rule.IsEnabled = model.IsEnabled.Value && (org?.HasPremiumFeatures ?? false); + } + + rule.Version++; + rule.UpdatedUtc = _timeProvider.GetUtcNow().UtcDateTime; + + await _ruleRepository.SaveAsync(rule, o => o.Cache()); + await _ruleCache.InvalidateAsync(projectId); + + return Ok(MapToView(rule)); + } + + /// Delete a rate notification rule. + [HttpDelete("{ruleId:objectid}")] + public async Task DeleteAsync(string userId, string projectId, string ruleId) + { + if (!CanManage(userId)) + return NotFound(); + + var rule = await GetRuleAndCheckAccessAsync(ruleId, userId, projectId); + if (rule is null) + return NotFound(); + + await _ruleRepository.RemoveAsync(rule); + await _ruleCache.InvalidateAsync(projectId); + + return NoContent(); + } + + /// Snooze a rate notification rule. + [HttpPost("{ruleId:objectid}/snooze")] + [Consumes("application/json")] + public async Task> SnoozeAsync( + string userId, + string projectId, + string ruleId, + [FromBody] SnoozeRateNotificationRuleRequest request) + { + if (!CanManage(userId)) + return NotFound(); + + var rule = await GetRuleAndCheckAccessAsync(ruleId, userId, projectId); + if (rule is null) + return NotFound(); + + var now = _timeProvider.GetUtcNow().UtcDateTime; + + if (request.UntilUtc.HasValue) + { + rule.SnoozedUntilUtc = request.UntilUtc.Value; + } + else if (request.DurationSeconds.HasValue) + { + rule.SnoozedUntilUtc = now.AddSeconds(request.DurationSeconds.Value); + } + else + { + return ValidationProblem(detail: "Either DurationSeconds or UntilUtc must be provided."); + } + + rule.Version++; + rule.UpdatedUtc = now; + await _ruleRepository.SaveAsync(rule, o => o.Cache()); + await _ruleCache.InvalidateAsync(projectId); + + return Ok(MapToView(rule)); + } + + /// Unsnooze a rate notification rule. Sets SnoozedUntilUtc = now to establish a fresh baseline. + [HttpPost("{ruleId:objectid}/unsnooze")] + public async Task> UnsnoozeAsync(string userId, string projectId, string ruleId) + { + if (!CanManage(userId)) + return NotFound(); + + var rule = await GetRuleAndCheckAccessAsync(ruleId, userId, projectId); + if (rule is null) + return NotFound(); + + // Set to now (NOT null) so the evaluator uses now as the effective window start — no back-alert + rule.SnoozedUntilUtc = _timeProvider.GetUtcNow().UtcDateTime; + rule.Version++; + rule.UpdatedUtc = _timeProvider.GetUtcNow().UtcDateTime; + await _ruleRepository.SaveAsync(rule, o => o.Cache()); + await _ruleCache.InvalidateAsync(projectId); + + return Ok(MapToView(rule)); + } + + // ---- Helpers ---- + + private bool CanManage(string userId) + { + // User can manage their own rules; global admins can manage any user's rules + return String.Equals(CurrentUser.Id, userId, StringComparison.Ordinal) || Request.IsGlobalAdmin(); + } + + private async Task GetProjectAndCheckAccessAsync(string projectId) + { + var project = await _projectRepository.GetByIdAsync(projectId, o => o.Cache()); + if (project is null || !CanAccessOrganization(project.OrganizationId)) + return null; + return project; + } + + private async Task GetRuleAndCheckAccessAsync(string ruleId, string userId, string projectId) + { + var rule = await _ruleRepository.GetByIdAsync(ruleId); + if (rule is null) + return null; + + // Rule must belong to the specified user+project + if (!String.Equals(rule.UserId, userId, StringComparison.Ordinal)) + return null; + + if (!String.Equals(rule.ProjectId, projectId, StringComparison.Ordinal)) + return null; + + // Current user must be able to manage this rule + if (!CanManage(userId)) + return null; + + return rule; + } + + private ViewRateNotificationRule MapToView(RateNotificationRule rule) + { + var now = _timeProvider.GetUtcNow().UtcDateTime; + return new ViewRateNotificationRule + { + Id = rule.Id, + OrganizationId = rule.OrganizationId, + ProjectId = rule.ProjectId, + UserId = rule.UserId, + Version = rule.Version, + Name = rule.Name, + IsEnabled = rule.IsEnabled, + Signal = rule.Signal, + Subject = rule.Subject, + StackId = rule.StackId, + Threshold = rule.Threshold, + Window = rule.Window, + Cooldown = rule.Cooldown, + SnoozedUntilUtc = rule.SnoozedUntilUtc, + IsSnoozed = rule.SnoozedUntilUtc.HasValue && rule.SnoozedUntilUtc.Value > now, + LastFiredUtc = rule.LastFiredUtc, + CreatedUtc = rule.CreatedUtc, + UpdatedUtc = rule.UpdatedUtc + }; + } +} diff --git a/src/Exceptionless.Web/Models/RateNotification/RateNotificationRuleModels.cs b/src/Exceptionless.Web/Models/RateNotification/RateNotificationRuleModels.cs new file mode 100644 index 0000000000..0ff72cc38d --- /dev/null +++ b/src/Exceptionless.Web/Models/RateNotification/RateNotificationRuleModels.cs @@ -0,0 +1,82 @@ +using System.ComponentModel.DataAnnotations; +using Exceptionless.Core.Attributes; +using Exceptionless.Core.Models; + +namespace Exceptionless.Web.Models; + +public record NewRateNotificationRule +{ + [Required] + [MaxLength(100)] + public string Name { get; init; } = null!; + + public RateNotificationSignal Signal { get; init; } + + public RateNotificationSubject Subject { get; init; } + + [ObjectId] + public string? StackId { get; init; } + + [Range(1, int.MaxValue)] + public int Threshold { get; init; } = 10; + + public TimeSpan Window { get; init; } = TimeSpan.FromHours(1); + + public TimeSpan Cooldown { get; init; } = TimeSpan.FromHours(1); + + public bool IsEnabled { get; init; } = true; +} + +public record UpdateRateNotificationRule +{ + [MaxLength(100)] + public string? Name { get; init; } + + public RateNotificationSignal? Signal { get; init; } + + public RateNotificationSubject? Subject { get; init; } + + [ObjectId] + public string? StackId { get; init; } + + [Range(1, int.MaxValue)] + public int? Threshold { get; init; } + + public TimeSpan? Window { get; init; } + + public TimeSpan? Cooldown { get; init; } + + public bool? IsEnabled { get; init; } +} + +public record ViewRateNotificationRule +{ + public string Id { get; init; } = null!; + public string OrganizationId { get; init; } = null!; + public string ProjectId { get; init; } = null!; + public string UserId { get; init; } = null!; + public int Version { get; init; } + public string Name { get; init; } = null!; + public bool IsEnabled { get; init; } + public RateNotificationSignal Signal { get; init; } + public RateNotificationSubject Subject { get; init; } + public string? StackId { get; init; } + public int Threshold { get; init; } + public TimeSpan Window { get; init; } + public TimeSpan Cooldown { get; init; } + public DateTime? SnoozedUntilUtc { get; init; } + public bool IsSnoozed { get; init; } + public DateTime? LastFiredUtc { get; init; } + public DateTime CreatedUtc { get; init; } + public DateTime UpdatedUtc { get; init; } +} + +public record SnoozeRateNotificationRuleRequest +{ + /// Snooze duration in seconds. Mutually exclusive with UntilUtc. + [Range(1, int.MaxValue)] + public int? DurationSeconds { get; init; } + + /// Snooze until this UTC timestamp. Mutually exclusive with DurationSeconds. + public DateTime? UntilUtc { get; init; } +} diff --git a/tests/Exceptionless.Tests/Controllers/RateNotificationRuleControllerTests.cs b/tests/Exceptionless.Tests/Controllers/RateNotificationRuleControllerTests.cs new file mode 100644 index 0000000000..bdcf4a5a57 --- /dev/null +++ b/tests/Exceptionless.Tests/Controllers/RateNotificationRuleControllerTests.cs @@ -0,0 +1,405 @@ +using Exceptionless.Core.Models; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Utility; +using Exceptionless.Tests.Extensions; +using Exceptionless.Web.Models; +using Xunit; + +namespace Exceptionless.Tests.Controllers; + +public sealed class RateNotificationRuleControllerTests : IntegrationTestsBase +{ + private readonly IRateNotificationRuleRepository _ruleRepository; + private readonly IUserRepository _userRepository; + + public RateNotificationRuleControllerTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) + { + _ruleRepository = GetService(); + _userRepository = GetService(); + } + + protected override async Task ResetDataAsync() + { + await base.ResetDataAsync(); + var service = GetService(); + await service.CreateDataAsync(); + } + + // ---- Helper: get current user via /me endpoint ---- + private async Task GetTestOrganizationUserAsync() + { + var user = await SendRequestAsAsync(r => r + .AsTestOrganizationUser() + .AppendPath("users/me") + .StatusCodeShouldBeOk() + ); + Assert.NotNull(user); + return user; + } + + private async Task GetGlobalAdminUserAsync() + { + var user = await SendRequestAsAsync(r => r + .AsGlobalAdminUser() + .AppendPath("users/me") + .StatusCodeShouldBeOk() + ); + Assert.NotNull(user); + return user; + } + + private string RuleUrl(string userId, string projectId) => + $"users/{userId}/projects/{projectId}/rate-notifications"; + + private string RuleUrl(string userId, string projectId, string ruleId) => + $"users/{userId}/projects/{projectId}/rate-notifications/{ruleId}"; + + // ---- CRUD tests ---- + + [Fact] + public async Task GetAsync_AsOwnUser_ReturnsList() + { + var user = await GetTestOrganizationUserAsync(); + + var results = await SendRequestAsAsync>(r => r + .AsTestOrganizationUser() + .AppendPath(RuleUrl(user.Id, SampleDataService.TEST_PROJECT_ID)) + .StatusCodeShouldBeOk() + ); + + Assert.NotNull(results); + } + + [Fact] + public async Task GetAsync_AsDifferentUser_ReturnsNotFound() + { + // Using org user ID to try to access another user's rules + var adminUser = await GetGlobalAdminUserAsync(); + + await SendRequestAsync(r => r + .AsTestOrganizationUser() // Org user + .AppendPath(RuleUrl(adminUser.Id, SampleDataService.TEST_PROJECT_ID)) // accessing admin's rules + .StatusCodeShouldBeNotFound() + ); + } + + [Fact] + public async Task GetAsync_AsGlobalAdmin_CanAccessAnyUsersRules() + { + var orgUser = await GetTestOrganizationUserAsync(); + + var results = await SendRequestAsAsync>(r => r + .AsGlobalAdminUser() // admin accessing org user's rules + .AppendPath(RuleUrl(orgUser.Id, SampleDataService.TEST_PROJECT_ID)) + .StatusCodeShouldBeOk() + ); + + Assert.NotNull(results); + } + + [Fact] + public async Task PostAsync_ValidRule_CreatesRule() + { + var user = await GetTestOrganizationUserAsync(); + + var newRule = new NewRateNotificationRule + { + Name = "Error spike", + Signal = RateNotificationSignal.Errors, + Subject = RateNotificationSubject.Project, + Threshold = 10, + Window = TimeSpan.FromMinutes(5), + Cooldown = TimeSpan.FromMinutes(30), + IsEnabled = true + }; + + var created = await SendRequestAsAsync(r => r + .Post() + .AsTestOrganizationUser() + .AppendPath(RuleUrl(user.Id, SampleDataService.TEST_PROJECT_ID)) + .Content(newRule) + .StatusCodeShouldBeCreated() + ); + + Assert.NotNull(created); + Assert.Equal("Error spike", created.Name); + Assert.Equal(RateNotificationSignal.Errors, created.Signal); + Assert.Equal(10, created.Threshold); + } + + [Fact] + public async Task PostAsync_InvalidWindow_Returns422() + { + var user = await GetTestOrganizationUserAsync(); + + var newRule = new NewRateNotificationRule + { + Name = "Bad window", + Signal = RateNotificationSignal.Errors, + Subject = RateNotificationSubject.Project, + Threshold = 10, + Window = TimeSpan.FromMinutes(7), // invalid — not in allowed list + Cooldown = TimeSpan.FromMinutes(30), + IsEnabled = true + }; + + await SendRequestAsync(r => r + .Post() + .AsTestOrganizationUser() + .AppendPath(RuleUrl(user.Id, SampleDataService.TEST_PROJECT_ID)) + .Content(newRule) + .StatusCodeShouldBeUnprocessableEntity() + ); + } + + [Fact] + public async Task PostAsync_CooldownLessThanWindow_Returns422() + { + var user = await GetTestOrganizationUserAsync(); + + var newRule = new NewRateNotificationRule + { + Name = "Bad cooldown", + Signal = RateNotificationSignal.Errors, + Subject = RateNotificationSubject.Project, + Threshold = 10, + Window = TimeSpan.FromMinutes(30), + Cooldown = TimeSpan.FromMinutes(5), // less than window + IsEnabled = true + }; + + await SendRequestAsync(r => r + .Post() + .AsTestOrganizationUser() + .AppendPath(RuleUrl(user.Id, SampleDataService.TEST_PROJECT_ID)) + .Content(newRule) + .StatusCodeShouldBeUnprocessableEntity() + ); + } + + [Fact] + public async Task PostAsync_StackSubjectWithoutStackId_Returns422() + { + var user = await GetTestOrganizationUserAsync(); + + var newRule = new NewRateNotificationRule + { + Name = "Stack rule no id", + Signal = RateNotificationSignal.Errors, + Subject = RateNotificationSubject.Stack, // stack subject + StackId = null, // missing StackId + Threshold = 10, + Window = TimeSpan.FromMinutes(5), + Cooldown = TimeSpan.FromMinutes(30), + IsEnabled = true + }; + + await SendRequestAsync(r => r + .Post() + .AsTestOrganizationUser() + .AppendPath(RuleUrl(user.Id, SampleDataService.TEST_PROJECT_ID)) + .Content(newRule) + .StatusCodeShouldBeUnprocessableEntity() + ); + } + + [Fact] + public async Task GetByIdAsync_ExistingRule_ReturnsRule() + { + var user = await GetTestOrganizationUserAsync(); + + // Create first + var created = await SendRequestAsAsync(r => r + .Post() + .AsTestOrganizationUser() + .AppendPath(RuleUrl(user.Id, SampleDataService.TEST_PROJECT_ID)) + .Content(new NewRateNotificationRule + { + Name = "Get-by-id test", + Signal = RateNotificationSignal.Errors, + Subject = RateNotificationSubject.Project, + Threshold = 5, + Window = TimeSpan.FromMinutes(5), + Cooldown = TimeSpan.FromMinutes(30) + }) + .StatusCodeShouldBeCreated() + ); + + Assert.NotNull(created); + + // Fetch by ID + var fetched = await SendRequestAsAsync(r => r + .AsTestOrganizationUser() + .AppendPath(RuleUrl(user.Id, SampleDataService.TEST_PROJECT_ID, created.Id)) + .StatusCodeShouldBeOk() + ); + + Assert.NotNull(fetched); + Assert.Equal(created.Id, fetched.Id); + Assert.Equal("Get-by-id test", fetched.Name); + } + + [Fact] + public async Task DeleteAsync_ExistingRule_RemovesRule() + { + var user = await GetTestOrganizationUserAsync(); + + // Create + var created = await SendRequestAsAsync(r => r + .Post() + .AsTestOrganizationUser() + .AppendPath(RuleUrl(user.Id, SampleDataService.TEST_PROJECT_ID)) + .Content(new NewRateNotificationRule + { + Name = "Delete me", + Signal = RateNotificationSignal.Errors, + Subject = RateNotificationSubject.Project, + Threshold = 5, + Window = TimeSpan.FromMinutes(5), + Cooldown = TimeSpan.FromMinutes(30) + }) + .StatusCodeShouldBeCreated() + ); + + Assert.NotNull(created); + + // Delete + await SendRequestAsync(r => r + .Delete() + .AsTestOrganizationUser() + .AppendPath(RuleUrl(user.Id, SampleDataService.TEST_PROJECT_ID, created.Id)) + .StatusCodeShouldBeNoContent() + ); + + // Confirm deleted + var rule = await _ruleRepository.GetByIdAsync(created.Id); + Assert.Null(rule); + } + + [Fact] + public async Task SnoozeAsync_ValidDuration_SetsSnooze() + { + var user = await GetTestOrganizationUserAsync(); + + // Create a rule + var created = await SendRequestAsAsync(r => r + .Post() + .AsTestOrganizationUser() + .AppendPath(RuleUrl(user.Id, SampleDataService.TEST_PROJECT_ID)) + .Content(new NewRateNotificationRule + { + Name = "Snooze test", + Signal = RateNotificationSignal.Errors, + Subject = RateNotificationSubject.Project, + Threshold = 5, + Window = TimeSpan.FromMinutes(5), + Cooldown = TimeSpan.FromMinutes(30) + }) + .StatusCodeShouldBeCreated() + ); + + Assert.NotNull(created); + + // Snooze for 1 hour + var snoozed = await SendRequestAsAsync(r => r + .Post() + .AsTestOrganizationUser() + .AppendPath($"{RuleUrl(user.Id, SampleDataService.TEST_PROJECT_ID, created.Id)}/snooze") + .Content(new SnoozeRateNotificationRuleRequest { DurationSeconds = 3600 }) + .StatusCodeShouldBeOk() + ); + + Assert.NotNull(snoozed); + Assert.True(snoozed.IsSnoozed, "Rule should be snoozed after snooze request."); + Assert.NotNull(snoozed.SnoozedUntilUtc); + } + + [Fact] + public async Task UnsnoozeAsync_SnoozedRule_SetsSnoozedUntilToNow() + { + var user = await GetTestOrganizationUserAsync(); + + // Create + snooze + var created = await SendRequestAsAsync(r => r + .Post() + .AsTestOrganizationUser() + .AppendPath(RuleUrl(user.Id, SampleDataService.TEST_PROJECT_ID)) + .Content(new NewRateNotificationRule + { + Name = "Unsnooze test", + Signal = RateNotificationSignal.Errors, + Subject = RateNotificationSubject.Project, + Threshold = 5, + Window = TimeSpan.FromMinutes(5), + Cooldown = TimeSpan.FromMinutes(30) + }) + .StatusCodeShouldBeCreated() + ); + + Assert.NotNull(created); + + await SendRequestAsync(r => r + .Post() + .AsTestOrganizationUser() + .AppendPath($"{RuleUrl(user.Id, SampleDataService.TEST_PROJECT_ID, created.Id)}/snooze") + .Content(new SnoozeRateNotificationRuleRequest { DurationSeconds = 3600 }) + .StatusCodeShouldBeOk() + ); + + // Unsnooze — sets SnoozedUntilUtc = now, so IsSnoozed = false + var unsnoozed = await SendRequestAsAsync(r => r + .Post() + .AsTestOrganizationUser() + .AppendPath($"{RuleUrl(user.Id, SampleDataService.TEST_PROJECT_ID, created.Id)}/unsnooze") + .StatusCodeShouldBeOk() + ); + + Assert.NotNull(unsnoozed); + Assert.False(unsnoozed.IsSnoozed, "Rule should NOT be actively snoozed after unsnooze (SnoozedUntilUtc = now)."); + // SnoozedUntilUtc is still set (to now) — not null. This is the fresh baseline mechanism. + Assert.NotNull(unsnoozed.SnoozedUntilUtc); + } + + [Fact] + public async Task PostAsync_ExceedsMaxRulesPerUser_Returns422() + { + var user = await GetTestOrganizationUserAsync(); + + // Create 20 rules (the maximum) + for (int i = 0; i < 20; i++) + { + await SendRequestAsync(r => r + .Post() + .AsTestOrganizationUser() + .AppendPath(RuleUrl(user.Id, SampleDataService.TEST_PROJECT_ID)) + .Content(new NewRateNotificationRule + { + Name = $"Rule {i + 1}", + Signal = RateNotificationSignal.Errors, + Subject = RateNotificationSubject.Project, + Threshold = 5, + Window = TimeSpan.FromMinutes(5), + Cooldown = TimeSpan.FromMinutes(30) + }) + .StatusCodeShouldBeCreated() + ); + } + + // 21st rule should fail + await SendRequestAsync(r => r + .Post() + .AsTestOrganizationUser() + .AppendPath(RuleUrl(user.Id, SampleDataService.TEST_PROJECT_ID)) + .Content(new NewRateNotificationRule + { + Name = "Rule 21", + Signal = RateNotificationSignal.Errors, + Subject = RateNotificationSubject.Project, + Threshold = 5, + Window = TimeSpan.FromMinutes(5), + Cooldown = TimeSpan.FromMinutes(30) + }) + .StatusCodeShouldBeUnprocessableEntity() + ); + } +} diff --git a/tests/Exceptionless.Tests/Jobs/RateNotificationEvaluatorJobTests.cs b/tests/Exceptionless.Tests/Jobs/RateNotificationEvaluatorJobTests.cs new file mode 100644 index 0000000000..6a55b4653c --- /dev/null +++ b/tests/Exceptionless.Tests/Jobs/RateNotificationEvaluatorJobTests.cs @@ -0,0 +1,222 @@ +using Exceptionless.Core.Jobs; +using Exceptionless.Core.Models; +using Exceptionless.Core.Queues.Models; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Services; +using Exceptionless.Core.Utility; +using Foundatio.Queues; +using Foundatio.Repositories; +using Xunit; + +namespace Exceptionless.Tests.Jobs; + +public class RateNotificationEvaluatorJobTests : IntegrationTestsBase +{ + private readonly RateNotificationEvaluatorJob _job; + private readonly RateCounterService _counterService; + private readonly IRateNotificationRuleRepository _ruleRepository; + private readonly IOrganizationRepository _orgRepository; + private readonly IProjectRepository _projectRepository; + private readonly IQueue _notificationQueue; + + public RateNotificationEvaluatorJobTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) + { + _job = GetService(); + _counterService = GetService(); + _ruleRepository = GetService(); + _orgRepository = GetService(); + _projectRepository = GetService(); + _notificationQueue = GetService>(); + } + + protected override async Task ResetDataAsync() + { + await base.ResetDataAsync(); + var service = GetService(); + await service.CreateDataAsync(); + } + + private RateNotificationRule BuildRule(string? projectId = null, RateNotificationSignal signal = RateNotificationSignal.Errors, int threshold = 10, string? window = null, string? cooldown = null) + { + var now = TimeProvider.GetUtcNow().UtcDateTime; + return new RateNotificationRule + { + OrganizationId = SampleDataService.TEST_ORG_ID, + ProjectId = projectId ?? SampleDataService.TEST_PROJECT_ID, + UserId = "507f1f77bcf86cd799439011", + Name = "Test Rule", + IsEnabled = true, + Signal = signal, + Subject = RateNotificationSubject.Project, + Threshold = threshold, + Window = window is not null ? TimeSpan.Parse(window) : TimeSpan.FromMinutes(5), + Cooldown = cooldown is not null ? TimeSpan.Parse(cooldown) : TimeSpan.FromMinutes(10), + Version = 1, + CreatedUtc = now, + UpdatedUtc = now + }; + } + + private string BuildCounterKey(RateNotificationRule rule) => + $"project:{rule.ProjectId}:signal:{rule.Signal}"; + + [Fact] + public async Task RunAsync_WhenThresholdCrossed_EnqueuesNotification() + { + var ct = TestContext.Current.CancellationToken; + // Arrange: start at event time (2 min before "now"), then advance forward + var now = new DateTime(2024, 6, 1, 12, 0, 0, DateTimeKind.Utc); + TimeProvider.SetUtcNow(now.AddMinutes(-2)); + + var rule = await _ruleRepository.AddAsync(BuildRule(threshold: 5), o => o.ImmediateConsistency()); + string counterKey = BuildCounterKey(rule); + + // Simulate 10 events within the 5-minute window + for (int i = 0; i < 10; i++) + await _counterService.IncrementAsync(counterKey, ct); + + TimeProvider.Advance(TimeSpan.FromMinutes(2)); + + // Act + await _job.RunAsync(ct); + + // Assert — notification enqueued + var stats = await _notificationQueue.GetQueueStatsAsync(); + Assert.True(stats.Enqueued > 0, "Expected a RateNotification to be enqueued when threshold is crossed."); + } + + [Fact] + public async Task RunAsync_WhenBelowThreshold_DoesNotEnqueue() + { + var ct = TestContext.Current.CancellationToken; + // Arrange: start at event time, advance to "now" + var now = new DateTime(2024, 6, 1, 12, 0, 0, DateTimeKind.Utc); + TimeProvider.SetUtcNow(now.AddMinutes(-2)); + + var rule = await _ruleRepository.AddAsync(BuildRule(threshold: 50), o => o.ImmediateConsistency()); + string counterKey = BuildCounterKey(rule); + + // Only 5 events — well below threshold of 50 + for (int i = 0; i < 5; i++) + await _counterService.IncrementAsync(counterKey, ct); + + TimeProvider.Advance(TimeSpan.FromMinutes(2)); + long queueBefore = (await _notificationQueue.GetQueueStatsAsync()).Enqueued; + + // Act + await _job.RunAsync(ct); + + // Assert — no new notification + long queueAfter = (await _notificationQueue.GetQueueStatsAsync()).Enqueued; + Assert.Equal(queueBefore, queueAfter); + } + + [Fact] + public async Task RunAsync_WhenOnCooldown_DoesNotEnqueueAgain() + { + var ct = TestContext.Current.CancellationToken; + // Arrange: start at event time, advance to "now", then set cooldown + var now = new DateTime(2024, 6, 1, 12, 0, 0, DateTimeKind.Utc); + TimeProvider.SetUtcNow(now.AddMinutes(-2)); + + var rule = await _ruleRepository.AddAsync(BuildRule(threshold: 5), o => o.ImmediateConsistency()); + string counterKey = BuildCounterKey(rule); + string subjectKey = $"project:{rule.ProjectId}"; + + // Add enough events to cross threshold + for (int i = 0; i < 10; i++) + await _counterService.IncrementAsync(counterKey, ct); + + TimeProvider.Advance(TimeSpan.FromMinutes(2)); + + // Put rule on cooldown (at "now") + await _counterService.SetCooldownAsync(rule.Id, subjectKey, TimeSpan.FromHours(1), ct); + + long queueBefore = (await _notificationQueue.GetQueueStatsAsync()).Enqueued; + + // Act + await _job.RunAsync(ct); + + // Assert — still on cooldown, nothing extra enqueued + long queueAfter = (await _notificationQueue.GetQueueStatsAsync()).Enqueued; + Assert.Equal(queueBefore, queueAfter); + } + + [Fact] + public async Task RunAsync_WhenActivelySnoozed_SkipsEvaluation() + { + var ct = TestContext.Current.CancellationToken; + // Arrange: start at event time, advance to "now" + var now = new DateTime(2024, 6, 1, 12, 0, 0, DateTimeKind.Utc); + TimeProvider.SetUtcNow(now.AddMinutes(-2)); + + var ruleData = BuildRule(threshold: 5); + ruleData.SnoozedUntilUtc = now.AddHours(2); // snoozed for 2 more hours + var rule = await _ruleRepository.AddAsync(ruleData, o => o.ImmediateConsistency()); + string counterKey = BuildCounterKey(rule); + + // Add events above threshold + for (int i = 0; i < 20; i++) + await _counterService.IncrementAsync(counterKey, ct); + + TimeProvider.Advance(TimeSpan.FromMinutes(2)); + long queueBefore = (await _notificationQueue.GetQueueStatsAsync()).Enqueued; + + // Act + await _job.RunAsync(ct); + + // Assert — snoozed rule does not fire + long queueAfter = (await _notificationQueue.GetQueueStatsAsync()).Enqueued; + Assert.Equal(queueBefore, queueAfter); + } + + /// + /// CRITICAL REGRESSION: Snooze back-alert prevention at job level. + /// + /// Setup: Rule A was snoozed until T-2min (snooze recently expired). + /// 15 events were counted at T-3min (during the snooze period). + /// Rule B (identical but not snoozed) gets the same traffic. + /// + /// Expected: + /// Rule B fires: full 5-min window sees 15 events at or above threshold. + /// Rule A does NOT fire: effective window is [T-2min, now], so 0 events (prevented). + /// + [Fact] + public async Task RunAsync_SnoozeBackAlert_SnoozedRuleIgnoresTrafficDuringSnoozeWindow() + { + var ct = TestContext.Current.CancellationToken; + // Arrange: start at T-3min (events arrive during snooze), then advance to "now" + var now = new DateTime(2024, 6, 1, 12, 0, 0, DateTimeKind.Utc); + TimeProvider.SetUtcNow(now.AddMinutes(-3)); + + // Rule A: snoozed until T-2min (snooze has just expired at "now") + var ruleAData = BuildRule(threshold: 10); + ruleAData.SnoozedUntilUtc = now.AddMinutes(-2); + var ruleA = await _ruleRepository.AddAsync(ruleAData, o => o.ImmediateConsistency()); + + // Rule B: not snoozed — should fire normally + var ruleB = await _ruleRepository.AddAsync(BuildRule(threshold: 10), o => o.ImmediateConsistency()); + + // Both rules watch the same signal/counter key + string counterKey = BuildCounterKey(ruleA); + Assert.Equal(counterKey, BuildCounterKey(ruleB)); + + // Simulate 15 events at T-3min (during Rule A's snooze window) + for (int i = 0; i < 15; i++) + await _counterService.IncrementAsync(counterKey, ct); + + TimeProvider.Advance(TimeSpan.FromMinutes(3)); + long queueBefore = (await _notificationQueue.GetQueueStatsAsync()).Enqueued; + + // Act + await _job.RunAsync(ct); + + // Assert + long queueAfter = (await _notificationQueue.GetQueueStatsAsync()).Enqueued; + long newNotifications = queueAfter - queueBefore; + + // Rule B should have fired (1 notification), Rule A should NOT have fired + // So exactly 1 notification should be enqueued. + Assert.Equal(1, newNotifications); + } +} diff --git a/tests/Exceptionless.Tests/Mail/CountingMailer.cs b/tests/Exceptionless.Tests/Mail/CountingMailer.cs index 96d5671c3f..1a6f5c1097 100644 --- a/tests/Exceptionless.Tests/Mail/CountingMailer.cs +++ b/tests/Exceptionless.Tests/Mail/CountingMailer.cs @@ -65,6 +65,11 @@ public Task SendUserPasswordResetAsync(User user) return Task.CompletedTask; } + public Task SendRateNotificationAsync(User user, Project project, RateNotificationRule rule, long observedCount, DateTime windowStart, DateTime windowEnd, Stack? stack = null) + { + return Task.CompletedTask; + } + public void Reset() { Interlocked.Exchange(ref _organizationNoticeCount, 0); diff --git a/tests/Exceptionless.Tests/Mail/NullMailer.cs b/tests/Exceptionless.Tests/Mail/NullMailer.cs index 826be1324d..bef6901698 100644 --- a/tests/Exceptionless.Tests/Mail/NullMailer.cs +++ b/tests/Exceptionless.Tests/Mail/NullMailer.cs @@ -44,4 +44,9 @@ public Task SendUserPasswordResetAsync(User user) { return Task.CompletedTask; } + + public Task SendRateNotificationAsync(User user, Project project, RateNotificationRule rule, long observedCount, DateTime windowStart, DateTime windowEnd, Stack? stack = null) + { + return Task.CompletedTask; + } } diff --git a/tests/Exceptionless.Tests/Services/RateCounterServiceTests.cs b/tests/Exceptionless.Tests/Services/RateCounterServiceTests.cs new file mode 100644 index 0000000000..8d8a53bbbf --- /dev/null +++ b/tests/Exceptionless.Tests/Services/RateCounterServiceTests.cs @@ -0,0 +1,274 @@ +using Exceptionless.Core.Services; +using Exceptionless.Tests.Utility; +using Foundatio.Caching; +using Microsoft.Extensions.Logging.Abstractions; +using Xunit; + +namespace Exceptionless.Tests.Services; + +/// +/// Unit tests for RateCounterService, including the critical snooze back-alert regression test. +/// Uses an in-memory cache client and ProxyTimeProvider — no Elasticsearch required. +/// +public class RateCounterServiceTests +{ + private const string CounterKey = "project:P1:signal:Errors"; + private const string RuleId = "rule-001"; + private const string SubjectKey = "project:P1"; + + private static (RateCounterService service, ProxyTimeProvider timeProvider, InMemoryCacheClient cache) Create() + { + var timeProvider = new ProxyTimeProvider(); + var cache = new InMemoryCacheClient(new InMemoryCacheClientOptions + { + LoggerFactory = NullLoggerFactory.Instance + }); + var service = new RateCounterService(cache, timeProvider, NullLoggerFactory.Instance); + return (service, timeProvider, cache); + } + + // ------------------------------------------------------------------------- + // CRITICAL REGRESSION TEST: Snooze back-alert prevention + // ------------------------------------------------------------------------- + + /// + /// Verifies that when a rule was snoozed and the snooze recently expired, the + /// evaluator's SumBucketsAsync call uses max(windowStart, snoozedUntil) as the + /// effective window start — so traffic counted during the snooze window does NOT + /// trigger the rule. + /// + /// Without the snooze fix: Rule A would see the 15 events counted 3 minutes ago + /// (inside the 5-minute window) and fire incorrectly. + /// + /// With the snooze fix: Rule A uses snoozedUntilUtc as the effective lower boundary, + /// so it only counts events after the snooze expired (0 events) and does NOT fire. + /// + [Fact] + public async Task SumBucketsAsync_WithSnoozeFix_IgnoresTrafficDuringSnooze() + { + var ct = TestContext.Current.CancellationToken; + var (service, timeProvider, _) = Create(); + + // Start at T-3min (the time the snoozed events occurred) + var baseTime = new DateTime(2024, 1, 15, 12, 0, 0, DateTimeKind.Utc); + var eventTime = baseTime.AddMinutes(-3); + timeProvider.SetUtcNow(eventTime); + + for (int i = 0; i < 15; i++) + await service.IncrementAsync(CounterKey, ct); + + // Advance to "now" (T+0) + timeProvider.Advance(TimeSpan.FromMinutes(3)); + + var now = baseTime; + var windowDuration = TimeSpan.FromMinutes(5); + var windowStartUtc = now.Subtract(windowDuration); // T-5min + + // Rule A: snoozed until T-2min (snooze recently expired) + var snoozedUntilUtc = now.AddMinutes(-2); // T-2min + + // Without snooze fix: sum from T-5min to now = 15 events (fires incorrectly) + long withoutFix = await service.SumBucketsAsync(CounterKey, windowStartUtc, now, ct); + + // With snooze fix: effective window start = max(T-5min, T-2min) = T-2min + // Events were counted at T-3min which is BEFORE T-2min, so they're excluded + var effectiveWindowStart = snoozedUntilUtc > windowStartUtc ? snoozedUntilUtc : windowStartUtc; + long withFix = await service.SumBucketsAsync(CounterKey, effectiveWindowStart, now, ct); + + // Without fix: 15 events (would fire incorrectly) + Assert.Equal(15, withoutFix); + + // With fix: 0 events (correctly prevents back-alert) + Assert.Equal(0, withFix); + } + + /// + /// Verifies that Rule B (not snoozed) DOES fire for the same traffic. + /// Companion test to the snooze regression — proves the snooze fix is selective. + /// + [Fact] + public async Task SumBucketsAsync_NonSnoozedRule_CountsAllTrafficInWindow() + { + var ct = TestContext.Current.CancellationToken; + var (service, timeProvider, _) = Create(); + + // Start at T-3min, then advance forward + var baseTime = new DateTime(2024, 1, 15, 12, 0, 0, DateTimeKind.Utc); + var eventTime = baseTime.AddMinutes(-3); + timeProvider.SetUtcNow(eventTime); + + for (int i = 0; i < 15; i++) + await service.IncrementAsync(CounterKey, ct); + + timeProvider.Advance(TimeSpan.FromMinutes(3)); + + var now = baseTime; + var windowStartUtc = now.AddMinutes(-5); // full 5min window + + // Non-snoozed rule: uses full window + long count = await service.SumBucketsAsync(CounterKey, windowStartUtc, now, ct); + + // All 15 events are in the 5-minute window => threshold=10 => fires + Assert.Equal(15, count); + } + + // ------------------------------------------------------------------------- + // Bucket increment and sum tests + // ------------------------------------------------------------------------- + + [Fact] + public async Task IncrementAsync_SingleIncrement_CreatesCountBucket() + { + var ct = TestContext.Current.CancellationToken; + var (service, timeProvider, _) = Create(); + var now = new DateTime(2024, 1, 15, 10, 0, 0, DateTimeKind.Utc); + timeProvider.SetUtcNow(now); + + await service.IncrementAsync(CounterKey, ct); + + long count = await service.SumBucketsAsync(CounterKey, now.AddMinutes(-1), now, ct); + Assert.Equal(1, count); + } + + [Fact] + public async Task IncrementAsync_MultipleIncrements_AccumulatesCount() + { + var ct = TestContext.Current.CancellationToken; + var (service, timeProvider, _) = Create(); + var now = new DateTime(2024, 1, 15, 10, 0, 0, DateTimeKind.Utc); + timeProvider.SetUtcNow(now); + + for (int i = 0; i < 7; i++) + await service.IncrementAsync(CounterKey, ct); + + long count = await service.SumBucketsAsync(CounterKey, now.AddMinutes(-1), now, ct); + Assert.Equal(7, count); + } + + [Fact] + public async Task SumBucketsAsync_AcrossMultipleMinutes_SumsAllBuckets() + { + var ct = TestContext.Current.CancellationToken; + var (service, timeProvider, _) = Create(); + var now = new DateTime(2024, 1, 15, 10, 5, 0, DateTimeKind.Utc); + + // Add 3 events at T-4min + timeProvider.SetUtcNow(now.AddMinutes(-4)); + for (int i = 0; i < 3; i++) + await service.IncrementAsync(CounterKey, ct); + + // Add 5 events at T-2min + timeProvider.SetUtcNow(now.AddMinutes(-2)); + for (int i = 0; i < 5; i++) + await service.IncrementAsync(CounterKey, ct); + + // Add 2 events at T-0 + timeProvider.SetUtcNow(now); + for (int i = 0; i < 2; i++) + await service.IncrementAsync(CounterKey, ct); + + long count = await service.SumBucketsAsync(CounterKey, now.AddMinutes(-5), now, ct); + Assert.Equal(10, count); + } + + [Fact] + public async Task SumBucketsAsync_EmptyWindow_ReturnsZero() + { + var ct = TestContext.Current.CancellationToken; + var (service, timeProvider, _) = Create(); + var now = new DateTime(2024, 1, 15, 10, 0, 0, DateTimeKind.Utc); + timeProvider.SetUtcNow(now); + + long count = await service.SumBucketsAsync(CounterKey, now.AddMinutes(-5), now, ct); + Assert.Equal(0, count); + } + + [Fact] + public async Task SumBucketsAsync_EventsOutsideWindow_NotCounted() + { + var ct = TestContext.Current.CancellationToken; + var (service, timeProvider, _) = Create(); + var now = new DateTime(2024, 1, 15, 10, 10, 0, DateTimeKind.Utc); + + // Add events 10 minutes ago — outside a 5-minute window + timeProvider.SetUtcNow(now.AddMinutes(-10)); + for (int i = 0; i < 20; i++) + await service.IncrementAsync(CounterKey, ct); + + timeProvider.SetUtcNow(now); + long count = await service.SumBucketsAsync(CounterKey, now.AddMinutes(-5), now, ct); + Assert.Equal(0, count); + } + + // ------------------------------------------------------------------------- + // Active counter key tracking + // ------------------------------------------------------------------------- + + [Fact] + public async Task GetActiveCounterKeysAsync_AfterIncrement_ReturnsKey() + { + var ct = TestContext.Current.CancellationToken; + var (service, timeProvider, _) = Create(); + var now = new DateTime(2024, 1, 15, 10, 0, 0, DateTimeKind.Utc); + timeProvider.SetUtcNow(now); + + await service.IncrementAsync(CounterKey, ct); + + var keys = await service.GetActiveCounterKeysAsync(now, ct); + Assert.Contains(CounterKey, keys); + } + + [Fact] + public async Task GetActiveCounterKeysAsync_DifferentMinute_ReturnsEmpty() + { + var ct = TestContext.Current.CancellationToken; + var (service, timeProvider, _) = Create(); + var now = new DateTime(2024, 1, 15, 10, 5, 0, DateTimeKind.Utc); + timeProvider.SetUtcNow(now); + + await service.IncrementAsync(CounterKey, ct); + + // Ask for a different minute + var keys = await service.GetActiveCounterKeysAsync(now.AddMinutes(-3), ct); + Assert.DoesNotContain(CounterKey, keys); + } + + // ------------------------------------------------------------------------- + // Cooldown tests + // ------------------------------------------------------------------------- + + [Fact] + public async Task IsOnCooldownAsync_WhenNoCooldownSet_ReturnsFalse() + { + var ct = TestContext.Current.CancellationToken; + var (service, _, _) = Create(); + bool onCooldown = await service.IsOnCooldownAsync(RuleId, SubjectKey, ct); + Assert.False(onCooldown); + } + + [Fact] + public async Task IsOnCooldownAsync_AfterSetCooldown_ReturnsTrue() + { + var ct = TestContext.Current.CancellationToken; + var (service, _, _) = Create(); + await service.SetCooldownAsync(RuleId, SubjectKey, TimeSpan.FromHours(1), ct); + bool onCooldown = await service.IsOnCooldownAsync(RuleId, SubjectKey, ct); + Assert.True(onCooldown); + } + + [Fact] + public async Task SetCooldownAsync_DifferentRules_IndependentCooldowns() + { + var ct = TestContext.Current.CancellationToken; + var (service, _, _) = Create(); + const string ruleId2 = "rule-002"; + + await service.SetCooldownAsync(RuleId, SubjectKey, TimeSpan.FromHours(1), ct); + + bool rule1OnCooldown = await service.IsOnCooldownAsync(RuleId, SubjectKey, ct); + bool rule2OnCooldown = await service.IsOnCooldownAsync(ruleId2, SubjectKey, ct); + + Assert.True(rule1OnCooldown); + Assert.False(rule2OnCooldown); + } +}