diff --git a/.env.example b/.env.example index 020c2e72..aa173089 100644 --- a/.env.example +++ b/.env.example @@ -236,3 +236,11 @@ OPENCHARGEMAP_API_KEY= # figure to correct it. Left unset, a plug-in car keeps the gateway's number and a # hybrid shows state of charge as a percentage only. # HV_BATTERY_CAPACITY_KWH=1.83 + +# ── Telemetry detail ─────────────────────────────────────────────────────────── +# Days of telemetry kept exactly as the car reported it. Older telemetry is folded +# into one row per quarter of an hour, so the database grows far more slowly, while +# every reading's latest value, each day's totals and your saved trips are kept. +# Nothing in the app shows finer detail than that past 90 days, so a lower value is +# raised to 90. Set to 0 to keep every row. +# TELEMETRY_FULL_DETAIL_DAYS=365 diff --git a/README.md b/README.md index cd4922f9..d1cfcf35 100644 --- a/README.md +++ b/README.md @@ -298,6 +298,10 @@ docker start garagestack To restore, stop the container, replace `garagestack-data` with the backup, and start it again. +### Database size + +Telemetry older than a year is folded into one row per quarter of an hour, so the database keeps growing, but far more slowly. Saved trips, each day's totals and the latest value of every reading are kept; only the order of events within a quarter of an hour is lost. Compaction cannot be undone, so take a backup first if you want to keep the raw rows. Set `TELEMETRY_FULL_DETAIL_DAYS` to change the period, or to `0` to keep every row. See [Data kept over time](documentation/ARCHITECTURE.md#data-kept-over-time). + --- ## Push notifications diff --git a/docker-compose.yml b/docker-compose.yml index d8b119dd..caea2003 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -117,6 +117,8 @@ services: <<: *dotnet-environment # Language of push notification texts (en or nl); the Worker has no browser to ask. NOTIFICATION_LANGUAGE: "${NOTIFICATION_LANGUAGE:-}" + # Days of telemetry kept at full detail before the Worker compacts it. See .env.example. + TELEMETRY_FULL_DETAIL_DAYS: "${TELEMETRY_FULL_DETAIL_DAYS:-}" volumes: - worker_logs:/app/logs diff --git a/documentation/ARCHITECTURE.md b/documentation/ARCHITECTURE.md index 2d773980..71e864ac 100644 --- a/documentation/ARCHITECTURE.md +++ b/documentation/ARCHITECTURE.md @@ -83,13 +83,24 @@ The trip log (`/trip-log`, `TripEndpoints`) shows only saved trips, because only The `AddTripLog` migration clears the trips saved before it and resets the recording line, and the Worker saves them again with their new columns. Nothing is lost: until that migration, nothing a driver could enter was kept on a trip. +## Data kept over time + +Telemetry is the one table that grows with every poll. The Worker's `HousekeepingService` runs every six hours and keeps it in check. + +- **Compaction.** Telemetry older than `TELEMETRY_FULL_DETAIL_DAYS` (a year by default) is folded into one row per quarter of an hour by `TelemetryCompaction` in Core. It is the same last-value-wins merge that folds a poll's messages into one row, over a longer window. Each window keeps its last row, filled in with the newest earlier value of every field that row lacks. "The latest reading at or before a moment" (the odometer, the last position, a schedule the car reported once) therefore answers the same at the end of every window, and never with a reading taken later. A window is cut in two where a counter that starts again from zero drops (the day's distance and energy, the counters since the last charge, the journey distance), so the total of each day and each charge survives. A quarter of an hour divides every UTC offset, so a window never spans the car's local midnight. +- **The line.** `TelemetryCompactor` works through a whole UTC day at a time from `Vehicle.TelemetryCompactedUntil`, and saves each day together with the line in one transaction, so a run can stop anywhere and the next one carries on. It never passes `TripsRecordedUntil`: the trip recorder still needs every fix it has not cut into trips. A vehicle that has never recorded a trip is not compacted at all. Saved trips keep their own fixes (`Trip.PointsJson`), so compaction changes nothing about them. It does mean old trips can no longer be cut again from telemetry, one more reason never to clear saved trips in a migration. +- **What it costs.** Nothing in the app reads finer detail than this past 90 days: the statistics page and the map ask for 90 days at most, and the history query thins 90 days to a few points a day anyway. That is why a lower setting is raised to 90. What is lost is the order of events within a quarter of an hour, such as a short charge that started and stopped inside one. +- **Caches and deleted notifications.** The same job removes place names and snapped trips a day after they expire: nothing reads an expired one, and a lookup may be refreshing it at that moment. It removes notifications the user deleted once they are 30 days old; the notification cooldowns read that history, deleted rows included, at most a week back. Map POI tiles are left alone, because the map shows an expired tile's stations while it fetches the tile again. + +PostgreSQL reuses the space of removed rows for new ones rather than handing it back to the disk. To shrink the database files after the first compaction of a long history, run `VACUUM FULL "TelemetrySnapshots";` once. It locks the table for as long as it runs, so pick a quiet moment. + ## Projects under `src/` | Project | Contains | Depends on | | --- | --- | --- | | `GarageStack.Core` | Domain models (`Models/`), repository/service interfaces (`Interfaces/`), and pure helpers with no I/O (`Helpers/`) - the shared vocabulary every other project builds on. | nothing (leaf project) | | `GarageStack.Data` | EF Core: `AppDbContext`, migrations, concrete repository implementations, and `Demo/` (in-memory fakes used when `DEMO_MODE=true`). | `Core` | -| `GarageStack.Worker` | The MQTT-ingestion process: `Mqtt/MqttConsumerService` and the message handlers beside it, plus the periodic jobs (maintenance reminders, POI pre-caching, push-notification checks, saving finished trips), each a `PeriodicBackgroundService`. | `Core`, `Data` | +| `GarageStack.Worker` | The MQTT-ingestion process: `Mqtt/MqttConsumerService` and the message handlers beside it, plus the periodic jobs (maintenance reminders, POI pre-caching, push-notification checks, saving finished trips, housekeeping), each a `PeriodicBackgroundService`. | `Core`, `Data` | | `GarageStack.Api` | ASP.NET Core minimal APIs (`Endpoints/`), the SignalR hub (`Hubs/`), and Api-only services (outbound MQTT publishing, POI/charging-station lookups, the Postgres-LISTEN-to-SignalR bridge). | `Core`, `Data` | | `GarageStack.Tests` | xUnit tests across all of the above. | all four | @@ -192,9 +203,12 @@ A new value arriving from the car passes through several layers, in this order: 3. The frontend's `TelemetrySnapshot` interface in `services/vehicleApi.ts`, mirrored by hand. 4. Wherever it should show up: a card in `frontend/src/cards/registry.ts`, and the demo data in `src/GarageStack.Data/Demo/` so the field is visible without a car. +5. Only for a counter that starts again from zero (per day, per charge, per journey): its entry in + `TelemetryCompaction`'s resetting counters, so compaction keeps the total it reached before + each reset. -What it does *not* need is a mention in the repository's merge loops or its "does this row carry -anything" filters, or in `WidgetStatusDto`: the first two are derived from the model itself, and +What it does *not* need is a mention in the merge loops (`TelemetryFields`) or the repository's +"does this row carry anything" filters, or in `WidgetStatusDto`: the first two are derived from the model itself, and the widget deliberately serves a curated subset. Charts are the same story in reverse: add the field to `TelemetryHistoryPoint` and the history query starts returning rows that carry it. diff --git a/documentation/CONFIGURATION.md b/documentation/CONFIGURATION.md index dc27aa63..8da7cbb1 100644 --- a/documentation/CONFIGURATION.md +++ b/documentation/CONFIGURATION.md @@ -98,6 +98,12 @@ The full guide, with provider examples, is [`AUTHENTICATION.md`](AUTHENTICATION. | `TYRE_PRESSURE_LOW_BAR` / `TYRE_PRESSURE_GOOD_BAR` / `TYRE_PRESSURE_HIGH_BAR` | `2.2` / `2.6` / `3.2` | Tyre pressure bands in bar, for the diagram's colours and the tyre notifications. Set them from your car's placard. | | `HV_BATTERY_CAPACITY_KWH` | none | Usable size of the traction battery in kWh. The gateway assumes an EV-sized pack, so its kWh figures are far too large on a plain hybrid (an MG HS Hybrid+ carries 1.83 and is reported as 72.5). Empty keeps the gateway's figure on a plug-in car and shows a hybrid's charge as a percentage only. | +## Stored data + +| Variable | Default | What it does | +| --- | --- | --- | +| `TELEMETRY_FULL_DETAIL_DAYS` | `365` | Days of telemetry kept exactly as the car reported it. Older telemetry is folded into one row per quarter of an hour, keeping the latest value of every reading, each day's totals and the saved trips. Below `90` is raised to `90`, the furthest back the statistics page and the map read. `0` keeps every row. See [Data kept over time](ARCHITECTURE.md#data-kept-over-time). | + ## API, logging and ports | Variable | Default | What it does | diff --git a/src/GarageStack.Core/Configuration/EnvironmentAliases.cs b/src/GarageStack.Core/Configuration/EnvironmentAliases.cs index 63851ee1..e72fc9ae 100644 --- a/src/GarageStack.Core/Configuration/EnvironmentAliases.cs +++ b/src/GarageStack.Core/Configuration/EnvironmentAliases.cs @@ -34,6 +34,9 @@ public static class EnvironmentAliases ["TYRE_PRESSURE_HIGH_BAR"] = "TyrePressure:HighBar", ["HV_BATTERY_CAPACITY_KWH"] = "HvBattery:CapacityKwh", + // How much detail old telemetry keeps. + ["TELEMETRY_FULL_DETAIL_DAYS"] = "TelemetryRetention:FullDetailDays", + // Who may call the API, and how often. ["CORS_ORIGIN"] = "Cors:Origins:0", ["RATE_LIMIT_GLOBAL_PER_MINUTE"] = "RateLimits:GlobalPerMinute", diff --git a/src/GarageStack.Core/Configuration/TelemetryRetentionOptions.cs b/src/GarageStack.Core/Configuration/TelemetryRetentionOptions.cs new file mode 100644 index 00000000..3dd2ac96 --- /dev/null +++ b/src/GarageStack.Core/Configuration/TelemetryRetentionOptions.cs @@ -0,0 +1,26 @@ +using Microsoft.Extensions.Configuration; + +namespace GarageStack.Core.Configuration; + +/// +/// How long telemetry keeps every row before the Worker compacts it into quarter-hour rows +/// (TELEMETRY_FULL_DETAIL_DAYS). 0 or less switches compaction off. A setting below +/// is raised to it, since the statistics page and the map read that far +/// back at full detail. +/// +public sealed class TelemetryRetentionOptions +{ + public const int DefaultDays = 365; + + /// The longest period the statistics page and the map ask the API for. + public const int MinimumDays = 90; + + /// Days of telemetry kept at full detail, or null when compaction is off. + public int? FullDetailDays { get; init; } = DefaultDays; + + public static TelemetryRetentionOptions From(IConfiguration configuration) + { + var days = configuration.IntegerOrDefault("TelemetryRetention:FullDetailDays", DefaultDays); + return new() { FullDetailDays = days <= 0 ? null : Math.Max(days, MinimumDays) }; + } +} diff --git a/src/GarageStack.Core/Helpers/TelemetryCompaction.cs b/src/GarageStack.Core/Helpers/TelemetryCompaction.cs new file mode 100644 index 00000000..5abf1dc6 --- /dev/null +++ b/src/GarageStack.Core/Helpers/TelemetryCompaction.cs @@ -0,0 +1,93 @@ +using GarageStack.Core.Models; + +namespace GarageStack.Core.Helpers; + +/// +/// Folds old telemetry into fewer rows. The Worker already folds the messages of one poll into a +/// single row over a 15-second window; compaction does the same over a quarter of an hour, once +/// nothing reads the finer detail. Each window keeps its last row, filled in with the newest earlier +/// value of every field that row lacks. "The latest reading at or before a moment" (the odometer, +/// the last position, the current state) therefore gives the same answer at the end of every window +/// as before, and never a reading from later on. +/// +public static class TelemetryCompaction +{ + /// + /// How much time one compacted row covers. A quarter of an hour divides every UTC offset in + /// use, so a window never spans the car's local midnight, wherever it is driven. + /// + public static readonly TimeSpan Window = TimeSpan.FromMinutes(15); + + // Counters that start again from zero: at midnight, after a charge, at a new journey. Folding a + // reset into the reading before it would lose that reading's total (a day's distance or fuel), + // so a window is cut in two where one of them drops. + private static readonly Func[] ResettingCounters = + [ + s => s.MileageOfTheDay, + s => s.PowerUsageOfDay, + s => s.MileageSinceLastCharge, + s => s.PowerUsageSinceLastCharge, + s => s.CurrentJourneyDistance, + ]; + + /// The start of the window falls in. + public static DateTime WindowStart(DateTime at) => new(at.Ticks - at.Ticks % Window.Ticks, at.Kind); + + /// + /// Compacts one vehicle's . The row each window keeps is filled in place, + /// so a caller holding tracked entities saves the result as it stands. Rows that are already + /// compacted come out unchanged. + /// + /// The rows folded into a kept row, for the caller to remove. + public static IReadOnlyList Fold(IEnumerable rows) + { + var folded = new List(); + foreach (var group in Groups(rows.OrderBy(r => r.RecordedAt).ThenBy(r => r.Id))) + { + var kept = group[^1]; + for (var i = group.Count - 2; i >= 0; i--) + { + TelemetryFields.ApplyFirstNonNullFields(kept, group[i]); + folded.Add(group[i]); + } + } + + return folded; + } + + // Consecutive rows that share a window and see no counter reset between them, oldest first. + private static IEnumerable> Groups(IEnumerable ordered) + { + List? group = null; + var windowStart = DateTime.MinValue; + var counters = new double?[ResettingCounters.Length]; + + foreach (var row in ordered) + { + var start = WindowStart(row.RecordedAt); + if (group is null || start != windowStart || Resets(counters, row)) + { + if (group is not null) yield return group; + group = []; + windowStart = start; + Array.Clear(counters); + } + + group.Add(row); + for (var i = 0; i < ResettingCounters.Length; i++) + counters[i] = ResettingCounters[i](row) ?? counters[i]; + } + + if (group is not null) yield return group; + } + + private static bool Resets(double?[] latest, TelemetrySnapshot row) + { + for (var i = 0; i < ResettingCounters.Length; i++) + { + if (ResettingCounters[i](row) < latest[i]) return true; + } + + return false; + } +} diff --git a/src/GarageStack.Core/Helpers/TelemetryFields.cs b/src/GarageStack.Core/Helpers/TelemetryFields.cs new file mode 100644 index 00000000..e2e1173e --- /dev/null +++ b/src/GarageStack.Core/Helpers/TelemetryFields.cs @@ -0,0 +1,76 @@ +using System.Linq.Expressions; +using System.Reflection; +using GarageStack.Core.Models; + +namespace GarageStack.Core.Helpers; + +/// +/// The fields of a that carry telemetry, and the two ways rows are +/// merged field by field. Every merge goes through here (the Worker folding a poll into one row, +/// the dashboard's view of the current state, and compaction of old rows), so a new telemetry field +/// only needs to be added to the model, not hand-copied into each merge loop. +/// +public static class TelemetryFields +{ + // Identity and bookkeeping: never merged. + private static readonly HashSet NonMergeable = + [ + nameof(TelemetrySnapshot.Id), nameof(TelemetrySnapshot.VehicleId), + nameof(TelemetrySnapshot.Vehicle), nameof(TelemetrySnapshot.RecordedAt), + nameof(TelemetrySnapshot.RawTopic), + ]; + + /// Every property that carries telemetry, in declaration order. + public static readonly IReadOnlyList Mergeable = typeof(TelemetrySnapshot) + .GetProperties(BindingFlags.Public | BindingFlags.Instance) + .Where(p => p.CanRead && p.CanWrite && !NonMergeable.Contains(p.Name)) + .ToArray(); + + // Compiled expression-tree accessors instead of live reflection (PropertyInfo.GetValue/ + // SetValue): merging runs on every MQTT message, multiple times per second per vehicle, and + // reflection's per-call overhead is avoidable since the property set is fixed at startup. + // Built once here, then invoked like a regular delegate call from then on. + private sealed record PropertyAccessor(string Name, Func Get, Action Set); + + private static PropertyAccessor BuildAccessor(PropertyInfo prop) + { + var instance = Expression.Parameter(typeof(TelemetrySnapshot), "instance"); + var value = Expression.Parameter(typeof(object), "value"); + + var getter = Expression.Lambda>( + Expression.Convert(Expression.Property(instance, prop), typeof(object)), + instance).Compile(); + + var setter = Expression.Lambda>( + Expression.Assign( + Expression.Property(instance, prop), + Expression.Convert(value, prop.PropertyType)), + instance, value).Compile(); + + return new PropertyAccessor(prop.Name, getter, setter); + } + + private static readonly PropertyAccessor[] Accessors = [.. Mergeable.Select(BuildAccessor)]; + + /// Overwrites every field on with the non-null value from , if any. + public static void ApplyNonNullFields(TelemetrySnapshot target, TelemetrySnapshot source) + { + foreach (var prop in Accessors) + { + var value = prop.Get(source); + if (value is not null) prop.Set(target, value); + } + } + + /// Fills any still-empty field on from , leaving already-set fields untouched. + public static void ApplyFirstNonNullFields(TelemetrySnapshot target, TelemetrySnapshot source, ISet? skip = null) + { + foreach (var prop in Accessors) + { + if (skip is not null && skip.Contains(prop.Name)) continue; + if (prop.Get(target) is not null) continue; + var value = prop.Get(source); + if (value is not null) prop.Set(target, value); + } + } +} diff --git a/src/GarageStack.Core/Interfaces/IHousekeepingRepository.cs b/src/GarageStack.Core/Interfaces/IHousekeepingRepository.cs new file mode 100644 index 00000000..7d029e8b --- /dev/null +++ b/src/GarageStack.Core/Interfaces/IHousekeepingRepository.cs @@ -0,0 +1,29 @@ +using GarageStack.Core.Models; + +namespace GarageStack.Core.Interfaces; + +/// +/// Keeps the database from growing without end: folds old telemetry into quarter-hour rows, and +/// removes what nothing reads any more. +/// +public interface IHousekeepingRepository +{ + /// When the vehicle's oldest telemetry row was recorded, or null when it has none. + Task GetFirstTelemetryAtAsync(int vehicleId, CancellationToken ct = default); + + /// + /// Compacts the vehicle's telemetry from up to (not including) + /// and moves to + /// , in one transaction. + /// + /// How many rows were folded into others and removed. + Task CompactTelemetryAsync(int vehicleId, DateTime from, DateTime until, CancellationToken ct = default); + + /// Removes the place names and snapped trips that expired before . + /// How many cache entries were removed. + Task RemoveExpiredCacheEntriesAsync(DateTime before, CancellationToken ct = default); + + /// Removes the notifications the user deleted that were created before . + /// How many notifications were removed. + Task RemoveDeletedNotificationsAsync(DateTime createdBefore, CancellationToken ct = default); +} diff --git a/src/GarageStack.Core/Models/Vehicle.cs b/src/GarageStack.Core/Models/Vehicle.cs index 4271f3b6..45377b73 100644 --- a/src/GarageStack.Core/Models/Vehicle.cs +++ b/src/GarageStack.Core/Models/Vehicle.cs @@ -28,6 +28,12 @@ public class Vehicle /// public DateTime? TripsRecordedUntil { get; set; } + /// + /// How far the Worker has compacted this vehicle's telemetry into quarter-hour rows: every row + /// before this moment is compacted. Null until the first compaction. + /// + public DateTime? TelemetryCompactedUntil { get; set; } + public ICollection TelemetrySnapshots { get; set; } = []; } diff --git a/src/GarageStack.Data/Extensions/ServiceCollectionExtensions.cs b/src/GarageStack.Data/Extensions/ServiceCollectionExtensions.cs index 3f93a9d9..1eb008fd 100644 --- a/src/GarageStack.Data/Extensions/ServiceCollectionExtensions.cs +++ b/src/GarageStack.Data/Extensions/ServiceCollectionExtensions.cs @@ -25,6 +25,7 @@ public static IServiceCollection AddGarageStackData(this IServiceCollection serv services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); services.AddGarageStackPoiClients(); return services; diff --git a/src/GarageStack.Data/Migrations/20261005091632_AddTelemetryCompactedUntil.Designer.cs b/src/GarageStack.Data/Migrations/20261005091632_AddTelemetryCompactedUntil.Designer.cs new file mode 100644 index 00000000..146f1652 --- /dev/null +++ b/src/GarageStack.Data/Migrations/20261005091632_AddTelemetryCompactedUntil.Designer.cs @@ -0,0 +1,843 @@ +// +using System; +using GarageStack.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace GarageStack.Data.Migrations +{ + [DbContext(typeof(AppDbContext))] + [Migration("20261005091632_AddTelemetryCompactedUntil")] + partial class AddTelemetryCompactedUntil + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.12") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("GarageStack.Core.Models.AppNotification", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Body") + .IsRequired() + .HasColumnType("text"); + + b.Property("Category") + .HasColumnType("text"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("IsArchived") + .HasColumnType("boolean"); + + b.Property("IsDeleted") + .HasColumnType("boolean"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text"); + + b.Property("VehicleId") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("CreatedAt"); + + b.HasIndex("IsDeleted", "CreatedAt"); + + b.ToTable("AppNotifications"); + }); + + modelBuilder.Entity("GarageStack.Core.Models.GeocodeCacheEntry", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CachedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CellLat") + .HasColumnType("integer"); + + b.Property("CellLng") + .HasColumnType("integer"); + + b.Property("City") + .HasMaxLength(120) + .HasColumnType("character varying(120)"); + + b.Property("CountryCode") + .HasMaxLength(8) + .HasColumnType("character varying(8)"); + + b.Property("DisplayName") + .HasMaxLength(300) + .HasColumnType("character varying(300)"); + + b.Property("ExpiresAt") + .HasColumnType("timestamp with time zone"); + + b.Property("HouseNumber") + .HasMaxLength(120) + .HasColumnType("character varying(120)"); + + b.Property("Language") + .IsRequired() + .HasMaxLength(8) + .HasColumnType("character varying(8)"); + + b.Property("Postcode") + .HasMaxLength(16) + .HasColumnType("character varying(16)"); + + b.Property("Precision") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)"); + + b.Property("Road") + .HasMaxLength(120) + .HasColumnType("character varying(120)"); + + b.HasKey("Id"); + + b.HasIndex("ExpiresAt") + .HasDatabaseName("IX_GeocodeCacheEntries_ExpiresAt"); + + b.HasIndex("Precision", "Language", "CellLat", "CellLng") + .IsUnique() + .HasDatabaseName("IX_GeocodeCacheEntries_Precision_Language_CellLatLng"); + + b.ToTable("GeocodeCacheEntries"); + }); + + modelBuilder.Entity("GarageStack.Core.Models.MaintenanceItem", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("IntervalKm") + .HasColumnType("double precision"); + + b.Property("IntervalMonths") + .HasColumnType("integer"); + + b.Property("LastServiceDate") + .HasColumnType("timestamp with time zone"); + + b.Property("LastServiceOdometerKm") + .HasColumnType("double precision"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Notes") + .HasMaxLength(1000) + .HasColumnType("character varying(1000)"); + + b.Property("VehicleId") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("VehicleId"); + + b.ToTable("MaintenanceItems"); + }); + + modelBuilder.Entity("GarageStack.Core.Models.MaintenanceLogEntry", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("MaintenanceItemId") + .HasColumnType("integer"); + + b.Property("Notes") + .HasMaxLength(1000) + .HasColumnType("character varying(1000)"); + + b.Property("OdometerKm") + .HasColumnType("double precision"); + + b.Property("PerformedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("MaintenanceItemId", "PerformedAt"); + + b.ToTable("MaintenanceLogEntries"); + }); + + modelBuilder.Entity("GarageStack.Core.Models.MapMatchCacheEntry", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CachedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("ExpiresAt") + .HasColumnType("timestamp with time zone"); + + b.Property("MatchedKm") + .HasColumnType("double precision"); + + b.Property("PointIndexesJson") + .HasColumnType("text"); + + b.Property("Provider") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("Shape") + .HasColumnType("text"); + + b.Property("SpeedLimitRunsJson") + .HasColumnType("text"); + + b.Property("TraceHash") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.HasKey("Id"); + + b.HasIndex("ExpiresAt") + .HasDatabaseName("IX_MapMatchCacheEntries_ExpiresAt"); + + b.HasIndex("Provider", "TraceHash") + .IsUnique() + .HasDatabaseName("IX_MapMatchCacheEntries_Provider_TraceHash"); + + b.ToTable("MapMatchCacheEntries"); + }); + + modelBuilder.Entity("GarageStack.Core.Models.PoiCacheTile", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CachedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CellLat") + .HasColumnType("integer"); + + b.Property("CellLng") + .HasColumnType("integer"); + + b.Property("ExpiresAt") + .HasColumnType("timestamp with time zone"); + + b.Property("PoiType") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("Source") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.HasKey("Id"); + + b.HasIndex("ExpiresAt") + .HasDatabaseName("IX_PoiCacheTiles_ExpiresAt"); + + b.HasIndex("Source", "PoiType", "CellLat", "CellLng") + .IsUnique() + .HasDatabaseName("IX_PoiCacheTiles_Source_PoiType_CellLatLng"); + + b.ToTable("PoiCacheTiles"); + }); + + modelBuilder.Entity("GarageStack.Core.Models.PoiItem", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Brand") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("CellLat") + .HasColumnType("integer"); + + b.Property("CellLng") + .HasColumnType("integer"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("ExternalId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("Latitude") + .HasColumnType("double precision"); + + b.Property("Longitude") + .HasColumnType("double precision"); + + b.Property("MetaJson") + .HasColumnType("text"); + + b.Property("Name") + .HasColumnType("text"); + + b.Property("PoiType") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("Source") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("Source", "ExternalId") + .IsUnique() + .HasDatabaseName("IX_PoiItems_Source_ExternalId"); + + b.HasIndex("Source", "PoiType", "Brand") + .HasDatabaseName("IX_PoiItems_Source_PoiType_Brand"); + + b.HasIndex("Source", "PoiType", "CellLat", "CellLng") + .HasDatabaseName("IX_PoiItems_Source_PoiType_CellLatLng"); + + b.HasIndex("Source", "PoiType", "Latitude", "Longitude") + .HasDatabaseName("IX_PoiItems_Source_PoiType_LatLon"); + + b.ToTable("PoiItems"); + }); + + modelBuilder.Entity("GarageStack.Core.Models.PushSubscription", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AuthKey") + .IsRequired() + .HasColumnType("text"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Endpoint") + .IsRequired() + .HasColumnType("text"); + + b.Property("P256DhKey") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("Endpoint") + .IsUnique(); + + b.ToTable("PushSubscriptions"); + }); + + modelBuilder.Entity("GarageStack.Core.Models.RevokedToken", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ExpiresAtUtc") + .HasColumnType("timestamp with time zone"); + + b.Property("Jti") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.HasKey("Id"); + + b.HasIndex("Jti") + .IsUnique(); + + b.ToTable("RevokedTokens"); + }); + + modelBuilder.Entity("GarageStack.Core.Models.TelemetrySnapshot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("BatteryHeating") + .HasColumnType("boolean"); + + b.Property("BatteryHeatingScheduleMode") + .HasColumnType("text"); + + b.Property("BatteryHeatingScheduleStartTime") + .HasColumnType("text"); + + b.Property("BatteryVoltage") + .HasColumnType("double precision"); + + b.Property("BmsChargeStatus") + .HasColumnType("text"); + + b.Property("BonnetOpen") + .HasColumnType("boolean"); + + b.Property("ChargerConnected") + .HasColumnType("boolean"); + + b.Property("ChargingCableLock") + .HasColumnType("boolean"); + + b.Property("ChargingLastEndAt") + .HasColumnType("timestamp with time zone"); + + b.Property("ChargingScheduleEndTime") + .HasColumnType("text"); + + b.Property("ChargingScheduleMode") + .HasColumnType("text"); + + b.Property("ChargingScheduleStartTime") + .HasColumnType("text"); + + b.Property("ChargingType") + .HasColumnType("text"); + + b.Property("ClimateOn") + .HasColumnType("boolean"); + + b.Property("CurrentJourneyDistance") + .HasColumnType("double precision"); + + b.Property("DriverDoorOpen") + .HasColumnType("boolean"); + + b.Property("DriverWindowOpen") + .HasColumnType("boolean"); + + b.Property("ElectricRangeKm") + .HasColumnType("double precision"); + + b.Property("Elevation") + .HasColumnType("double precision"); + + b.Property("EngineRunning") + .HasColumnType("boolean"); + + b.Property("EvSocPercent") + .HasColumnType("double precision"); + + b.Property("ExteriorTemperature") + .HasColumnType("double precision"); + + b.Property("FuelLevelPercent") + .HasColumnType("double precision"); + + b.Property("FuelRangeKm") + .HasColumnType("double precision"); + + b.Property("Heading") + .HasColumnType("double precision"); + + b.Property("HeatedSeatFrontLeft") + .HasColumnType("integer"); + + b.Property("HeatedSeatFrontRight") + .HasColumnType("integer"); + + b.Property("HvBatteryActive") + .HasColumnType("boolean"); + + b.Property("HvCurrent") + .HasColumnType("double precision"); + + b.Property("HvPower") + .HasColumnType("double precision"); + + b.Property("HvSocKwh") + .HasColumnType("double precision"); + + b.Property("HvTotalCapacityKwh") + .HasColumnType("double precision"); + + b.Property("HvVoltage") + .HasColumnType("double precision"); + + b.Property("InteriorTemperature") + .HasColumnType("double precision"); + + b.Property("IsAvailable") + .HasColumnType("boolean"); + + b.Property("IsCharging") + .HasColumnType("boolean"); + + b.Property("IsLocked") + .HasColumnType("boolean"); + + b.Property("LastChargeEndingPower") + .HasColumnType("double precision"); + + b.Property("LastChargeStateAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastVehicleStateAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Latitude") + .HasColumnType("double precision"); + + b.Property("LightsDippedBeam") + .HasColumnType("boolean"); + + b.Property("LightsMainBeam") + .HasColumnType("boolean"); + + b.Property("LightsSide") + .HasColumnType("boolean"); + + b.Property("Longitude") + .HasColumnType("double precision"); + + b.Property("MileageOfTheDay") + .HasColumnType("double precision"); + + b.Property("MileageSinceLastCharge") + .HasColumnType("double precision"); + + b.Property("ObcCurrent") + .HasColumnType("double precision"); + + b.Property("ObcPowerSinglePhase") + .HasColumnType("double precision"); + + b.Property("ObcPowerThreePhase") + .HasColumnType("double precision"); + + b.Property("ObcVoltage") + .HasColumnType("double precision"); + + b.Property("OdometerKm") + .HasColumnType("double precision"); + + b.Property("OffboardChargerPlugStatus") + .HasColumnType("integer"); + + b.Property("OnboardChargerPlugStatus") + .HasColumnType("integer"); + + b.Property("PassengerDoorOpen") + .HasColumnType("boolean"); + + b.Property("PassengerWindowOpen") + .HasColumnType("boolean"); + + b.Property("PowerUsageOfDay") + .HasColumnType("double precision"); + + b.Property("PowerUsageSinceLastCharge") + .HasColumnType("double precision"); + + b.Property("RawTopic") + .HasColumnType("text"); + + b.Property("RearLeftDoorOpen") + .HasColumnType("boolean"); + + b.Property("RearLeftWindowOpen") + .HasColumnType("boolean"); + + b.Property("RearRightDoorOpen") + .HasColumnType("boolean"); + + b.Property("RearRightWindowOpen") + .HasColumnType("boolean"); + + b.Property("RearWindowDefroster") + .HasColumnType("boolean"); + + b.Property("RecordedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("RemainingChargingTime") + .HasColumnType("integer"); + + b.Property("RemoteTemperature") + .HasColumnType("double precision"); + + b.Property("Speed") + .HasColumnType("double precision"); + + b.Property("SunRoofOpen") + .HasColumnType("boolean"); + + b.Property("TrunkOpen") + .HasColumnType("boolean"); + + b.Property("TyrePressureFrontLeft") + .HasColumnType("double precision"); + + b.Property("TyrePressureFrontRight") + .HasColumnType("double precision"); + + b.Property("TyrePressureRearLeft") + .HasColumnType("double precision"); + + b.Property("TyrePressureRearRight") + .HasColumnType("double precision"); + + b.Property("VehicleId") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("VehicleId", "RecordedAt"); + + b.HasIndex("VehicleId", "Latitude", "Longitude", "RecordedAt") + .HasDatabaseName("IX_TelemetrySnapshots_VehicleId_LatLon_RecordedAt") + .HasFilter("\"Latitude\" IS NOT NULL AND \"Longitude\" IS NOT NULL"); + + b.HasIndex(new[] { "VehicleId", "RecordedAt" }, "IX_TelemetrySnapshots_VehicleId_RecordedAt_Schedule") + .HasFilter("\"ChargingScheduleMode\" IS NOT NULL OR \"ChargingScheduleStartTime\" IS NOT NULL OR \"ChargingScheduleEndTime\" IS NOT NULL OR \"BatteryHeatingScheduleMode\" IS NOT NULL OR \"BatteryHeatingScheduleStartTime\" IS NOT NULL"); + + b.ToTable("TelemetrySnapshots"); + }); + + modelBuilder.Entity("GarageStack.Core.Models.Trip", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("DistanceKm") + .HasColumnType("double precision"); + + b.Property("EndLatitude") + .HasColumnType("double precision"); + + b.Property("EndLongitude") + .HasColumnType("double precision"); + + b.Property("EndPlaceJson") + .HasColumnType("text"); + + b.Property("EndedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Notes") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("OdometerEndKm") + .HasColumnType("double precision"); + + b.Property("OdometerStartKm") + .HasColumnType("double precision"); + + b.Property("PointCount") + .HasColumnType("integer"); + + b.Property("PointsJson") + .IsRequired() + .HasColumnType("text"); + + b.Property("Purpose") + .HasMaxLength(16) + .HasColumnType("character varying(16)"); + + b.Property("StartLatitude") + .HasColumnType("double precision"); + + b.Property("StartLongitude") + .HasColumnType("double precision"); + + b.Property("StartPlaceJson") + .HasColumnType("text"); + + b.Property("StartedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("VehicleId") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("VehicleId", "StartedAt") + .IsUnique(); + + b.ToTable("Trips"); + }); + + modelBuilder.Entity("GarageStack.Core.Models.Vehicle", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ConfigJson") + .HasColumnType("text"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastMessageId") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("LastParkedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Model") + .HasColumnType("text"); + + b.Property("SaicUser") + .HasColumnType("text"); + + b.Property("Series") + .HasColumnType("text"); + + b.Property("TelemetryCompactedUntil") + .HasColumnType("timestamp with time zone"); + + b.Property("TripsRecordedUntil") + .HasColumnType("timestamp with time zone"); + + b.Property("Vin") + .IsRequired() + .HasMaxLength(17) + .HasColumnType("character varying(17)"); + + b.HasKey("Id"); + + b.HasIndex("Vin") + .IsUnique(); + + b.ToTable("Vehicles"); + }); + + modelBuilder.Entity("GarageStack.Core.Models.MaintenanceItem", b => + { + b.HasOne("GarageStack.Core.Models.Vehicle", "Vehicle") + .WithMany() + .HasForeignKey("VehicleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Vehicle"); + }); + + modelBuilder.Entity("GarageStack.Core.Models.MaintenanceLogEntry", b => + { + b.HasOne("GarageStack.Core.Models.MaintenanceItem", "MaintenanceItem") + .WithMany("LogEntries") + .HasForeignKey("MaintenanceItemId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("MaintenanceItem"); + }); + + modelBuilder.Entity("GarageStack.Core.Models.TelemetrySnapshot", b => + { + b.HasOne("GarageStack.Core.Models.Vehicle", "Vehicle") + .WithMany("TelemetrySnapshots") + .HasForeignKey("VehicleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Vehicle"); + }); + + modelBuilder.Entity("GarageStack.Core.Models.Trip", b => + { + b.HasOne("GarageStack.Core.Models.Vehicle", "Vehicle") + .WithMany() + .HasForeignKey("VehicleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Vehicle"); + }); + + modelBuilder.Entity("GarageStack.Core.Models.MaintenanceItem", b => + { + b.Navigation("LogEntries"); + }); + + modelBuilder.Entity("GarageStack.Core.Models.Vehicle", b => + { + b.Navigation("TelemetrySnapshots"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/GarageStack.Data/Migrations/20261005091632_AddTelemetryCompactedUntil.cs b/src/GarageStack.Data/Migrations/20261005091632_AddTelemetryCompactedUntil.cs new file mode 100644 index 00000000..77956393 --- /dev/null +++ b/src/GarageStack.Data/Migrations/20261005091632_AddTelemetryCompactedUntil.cs @@ -0,0 +1,29 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace GarageStack.Data.Migrations +{ + /// + public partial class AddTelemetryCompactedUntil : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "TelemetryCompactedUntil", + table: "Vehicles", + type: "timestamp with time zone", + nullable: true); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "TelemetryCompactedUntil", + table: "Vehicles"); + } + } +} diff --git a/src/GarageStack.Data/Migrations/AppDbContextModelSnapshot.cs b/src/GarageStack.Data/Migrations/AppDbContextModelSnapshot.cs index f3ffd936..ce532715 100644 --- a/src/GarageStack.Data/Migrations/AppDbContextModelSnapshot.cs +++ b/src/GarageStack.Data/Migrations/AppDbContextModelSnapshot.cs @@ -762,6 +762,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Series") .HasColumnType("text"); + b.Property("TelemetryCompactedUntil") + .HasColumnType("timestamp with time zone"); + b.Property("TripsRecordedUntil") .HasColumnType("timestamp with time zone"); diff --git a/src/GarageStack.Data/Repositories/HousekeepingRepository.cs b/src/GarageStack.Data/Repositories/HousekeepingRepository.cs new file mode 100644 index 00000000..c0d66ddf --- /dev/null +++ b/src/GarageStack.Data/Repositories/HousekeepingRepository.cs @@ -0,0 +1,57 @@ +using GarageStack.Core.Helpers; +using GarageStack.Core.Interfaces; +using Microsoft.EntityFrameworkCore; + +namespace GarageStack.Data.Repositories; + +public class HousekeepingRepository(AppDbContext db) : IHousekeepingRepository +{ + public Task GetFirstTelemetryAtAsync(int vehicleId, CancellationToken ct = default) => + db.TelemetrySnapshots + .AsNoTracking() + .Where(s => s.VehicleId == vehicleId) + .OrderBy(s => s.RecordedAt) + .Select(s => (DateTime?)s.RecordedAt) + .FirstOrDefaultAsync(ct); + + public async Task CompactTelemetryAsync(int vehicleId, DateTime from, DateTime until, CancellationToken ct = default) + { + var rows = await db.TelemetrySnapshots + .Where(s => s.VehicleId == vehicleId && s.RecordedAt >= from && s.RecordedAt < until) + .ToListAsync(ct); + + // The kept rows are filled in place and saved as tracked changes, together with the line. + var folded = TelemetryCompaction.Fold(rows); + db.TelemetrySnapshots.RemoveRange(folded); + + var vehicle = await db.Vehicles.FirstAsync(v => v.Id == vehicleId, ct); + vehicle.TelemetryCompactedUntil = until; + + await db.SaveChangesAsync(ct); + + // A first run can work through years of history a day at a time; letting go of each day's + // rows keeps the change tracker from holding all of them. + db.ChangeTracker.Clear(); + return folded.Count; + } + + public async Task RemoveExpiredCacheEntriesAsync(DateTime before, CancellationToken ct = default) => + await DeleteAsync(db.GeocodeCacheEntries.Where(e => e.ExpiresAt < before), ct) + + await DeleteAsync(db.MapMatchCacheEntries.Where(e => e.ExpiresAt < before), ct); + + public Task RemoveDeletedNotificationsAsync(DateTime createdBefore, CancellationToken ct = default) => + DeleteAsync(db.AppNotifications.Where(n => n.IsDeleted && n.CreatedAt < createdBefore), ct); + + // A single DELETE on PostgreSQL; the in-memory provider the tests use cannot run bulk + // operations, so it falls back to loading the rows into the change tracker. + private async Task DeleteAsync(IQueryable rows, CancellationToken ct) where T : class + { + if (db.Database.IsRelational()) + return await rows.ExecuteDeleteAsync(ct); + + var loaded = await rows.ToListAsync(ct); + db.RemoveRange(loaded); + await db.SaveChangesAsync(ct); + return loaded.Count; + } +} diff --git a/src/GarageStack.Data/Repositories/TelemetryRepository.cs b/src/GarageStack.Data/Repositories/TelemetryRepository.cs index 9d3d0af7..16daaf41 100644 --- a/src/GarageStack.Data/Repositories/TelemetryRepository.cs +++ b/src/GarageStack.Data/Repositories/TelemetryRepository.cs @@ -1,5 +1,6 @@ using System.Linq.Expressions; using System.Reflection; +using GarageStack.Core.Helpers; using GarageStack.Core.Interfaces; using GarageStack.Core.Models; using GarageStack.Data.Extensions; @@ -32,49 +33,6 @@ public class TelemetryRepository( // volume - it exists purely to bound worst-case memory rather than to affect normal queries. private const int MaxRawRowsPerQuery = 200_000; - // All TelemetrySnapshot properties except identity/bookkeeping fields (Id, VehicleId, Vehicle, - // RecordedAt, RawTopic) participate in field-by-field merging. Computed once and reused by both - // MergeIntoAsync (last-write-wins) and GetMergedLatestAsync (first-non-null-wins) so a new - // telemetry field only needs to be added to the model - not hand-copied into two merge loops. - private static readonly HashSet NonMergeableProperties = - [ - nameof(TelemetrySnapshot.Id), nameof(TelemetrySnapshot.VehicleId), - nameof(TelemetrySnapshot.Vehicle), nameof(TelemetrySnapshot.RecordedAt), - nameof(TelemetrySnapshot.RawTopic), - ]; - - // Compiled expression-tree accessors instead of live reflection (PropertyInfo.GetValue/ - // SetValue): this runs on every MQTT message merge, multiple times per second per vehicle, - // and reflection's per-call overhead is avoidable since the property set is fixed at - // startup. Built once here, then invoked like a regular delegate call from then on. - private sealed record PropertyAccessor(string Name, Func Get, Action Set); - - private static PropertyAccessor BuildAccessor(PropertyInfo prop) - { - var instance = Expression.Parameter(typeof(TelemetrySnapshot), "instance"); - var value = Expression.Parameter(typeof(object), "value"); - - var getter = Expression.Lambda>( - Expression.Convert(Expression.Property(instance, prop), typeof(object)), - instance).Compile(); - - var setter = Expression.Lambda>( - Expression.Assign( - Expression.Property(instance, prop), - Expression.Convert(value, prop.PropertyType)), - instance, value).Compile(); - - return new PropertyAccessor(prop.Name, getter, setter); - } - - private static readonly PropertyInfo[] MergeablePropertyInfos = typeof(TelemetrySnapshot) - .GetProperties(BindingFlags.Public | BindingFlags.Instance) - .Where(p => p.CanRead && p.CanWrite && !NonMergeableProperties.Contains(p.Name)) - .ToArray(); - - private static readonly PropertyAccessor[] MergeableProperties = - [.. MergeablePropertyInfos.Select(BuildAccessor)]; - /// /// Builds "any of these fields is set" as an expression tree EF can translate to SQL. /// Non-nullable properties are left out: they always have a value, so they say nothing about @@ -100,33 +58,11 @@ private static Expression> AnyFieldSet(IEnumerable nameof(TelemetrySnapshot.MileageOfTheDay), nameof(TelemetrySnapshot.PowerUsageOfDay), ]; - /// Overwrites every field on with the non-null value from , if any. - private static void ApplyNonNullFields(TelemetrySnapshot target, TelemetrySnapshot source) - { - foreach (var prop in MergeableProperties) - { - var value = prop.Get(source); - if (value is not null) prop.Set(target, value); - } - } - - /// Fills any still-empty field on from , leaving already-set fields untouched. - private static void ApplyFirstNonNullFields(TelemetrySnapshot target, TelemetrySnapshot source, ISet? skip = null) - { - foreach (var prop in MergeableProperties) - { - if (skip is not null && skip.Contains(prop.Name)) continue; - if (prop.Get(target) is not null) continue; - var value = prop.Get(source); - if (value is not null) prop.Set(target, value); - } - } - // A row is worth reading when any mergeable field carries a value. Derived from the model for // the same reason the merge loops are: hand-listing seventy fields here means a new telemetry // field is silently treated as empty until someone remembers to add it. private static readonly Expression> HasData = - AnyFieldSet(MergeablePropertyInfos); + AnyFieldSet(TelemetryFields.Mergeable); // Chart history excludes GPS-only rows: latitude/longitude arrive every minute during driving // and inflate the row count, causing the stride downsampler to skip the sparser fuel/EV/kWh @@ -160,7 +96,7 @@ public async Task MergeIntoAsync(long rowId, TelemetrySnapshot patch, Cancellati } // Last-write-wins per field: overwrite with any non-null value from the patch. - ApplyNonNullFields(existing, patch); + TelemetryFields.ApplyNonNullFields(existing, patch); await db.SaveChangesAsync(ct); await NotifyUpdatedAsync(existing.VehicleId, ct); @@ -213,7 +149,7 @@ private async Task NotifyUpdatedAsync(int vehicleId, CancellationToken ct) var merged = new TelemetrySnapshot { VehicleId = vehicleId, RecordedAt = rows[0].RecordedAt }; foreach (var row in rows) { - ApplyFirstNonNullFields(merged, row, skip: DailyCounterFields); + TelemetryFields.ApplyFirstNonNullFields(merged, row, skip: DailyCounterFields); // Daily counters are only meaningful from today - don't carry yesterday's values forward if (row.RecordedAt >= todayStart) diff --git a/src/GarageStack.Tests/DeploymentOptionsTests.cs b/src/GarageStack.Tests/DeploymentOptionsTests.cs index 10504e2c..3b13eef5 100644 --- a/src/GarageStack.Tests/DeploymentOptionsTests.cs +++ b/src/GarageStack.Tests/DeploymentOptionsTests.cs @@ -90,4 +90,16 @@ public void Vapid_NamesAContactEvenWhenTheDeploymentDoesNot(string? subject, str { Assert.Equal(expected, VapidOptions.From(Config(("Vapid:Subject", subject))).Subject); } + + [Theory] + [InlineData(null, 365)] + [InlineData("", 365)] + [InlineData("400", 400)] + [InlineData("30", 90)] + [InlineData("0", null)] + [InlineData("-1", null)] + public void TelemetryRetention_KeepsAYear_GoesOffAtZero_AndNeverDropsBelowNinetyDays(string? configured, int? expected) + { + Assert.Equal(expected, TelemetryRetentionOptions.From(Config(("TelemetryRetention:FullDetailDays", configured))).FullDetailDays); + } } diff --git a/src/GarageStack.Tests/HousekeepingRepositoryTests.cs b/src/GarageStack.Tests/HousekeepingRepositoryTests.cs new file mode 100644 index 00000000..0b2b292b --- /dev/null +++ b/src/GarageStack.Tests/HousekeepingRepositoryTests.cs @@ -0,0 +1,54 @@ +using GarageStack.Core.Models; +using GarageStack.Data; +using GarageStack.Data.Repositories; +using Microsoft.EntityFrameworkCore; + +namespace GarageStack.Tests; + +public class HousekeepingRepositoryTests +{ + private static readonly DateTime Now = new(2026, 6, 1, 12, 0, 0, DateTimeKind.Utc); + + private static AppDbContext CreateDb() => + new(new DbContextOptionsBuilder() + .UseInMemoryDatabase(Guid.NewGuid().ToString()) + .Options); + + [Fact] + public async Task ExpiredPlaceNamesAndSnappedTrips_AreRemoved_AndValidOnesKept() + { + var ct = TestContext.Current.CancellationToken; + await using var db = CreateDb(); + db.GeocodeCacheEntries.AddRange( + new GeocodeCacheEntry { Precision = "city", Language = "en", CellLat = 1, City = "Expired", ExpiresAt = Now.AddDays(-2) }, + new GeocodeCacheEntry { Precision = "city", Language = "en", CellLat = 2, City = "Valid", ExpiresAt = Now.AddDays(1) }); + db.MapMatchCacheEntries.AddRange( + new MapMatchCacheEntry { Provider = "valhalla", TraceHash = "expired", ExpiresAt = Now.AddDays(-2) }, + new MapMatchCacheEntry { Provider = "valhalla", TraceHash = "valid", ExpiresAt = Now.AddDays(1) }); + await db.SaveChangesAsync(ct); + + Assert.Equal(2, await new HousekeepingRepository(db).RemoveExpiredCacheEntriesAsync(Now.AddDays(-1), ct)); + + Assert.Equal(["Valid"], await db.GeocodeCacheEntries.Select(e => e.City).ToListAsync(ct)); + Assert.Equal(["valid"], await db.MapMatchCacheEntries.Select(e => e.TraceHash).ToListAsync(ct)); + } + + [Fact] + public async Task DeletedNotifications_AreRemovedOnceOldEnough_AndArchivedOnesKept() + { + var ct = TestContext.Current.CancellationToken; + await using var db = CreateDb(); + db.AppNotifications.AddRange( + new AppNotification { Title = "deleted long ago", IsDeleted = true, CreatedAt = Now.AddDays(-40) }, + new AppNotification { Title = "deleted recently", IsDeleted = true, CreatedAt = Now.AddDays(-5) }, + new AppNotification { Title = "archived long ago", IsArchived = true, CreatedAt = Now.AddDays(-40) }, + new AppNotification { Title = "unread long ago", CreatedAt = Now.AddDays(-40) }); + await db.SaveChangesAsync(ct); + + Assert.Equal(1, await new HousekeepingRepository(db).RemoveDeletedNotificationsAsync(Now.AddDays(-30), ct)); + + Assert.Equal( + ["archived long ago", "deleted recently", "unread long ago"], + await db.AppNotifications.OrderBy(n => n.Title).Select(n => n.Title).ToListAsync(ct)); + } +} diff --git a/src/GarageStack.Tests/TelemetryCompactionTests.cs b/src/GarageStack.Tests/TelemetryCompactionTests.cs new file mode 100644 index 00000000..09f2da70 --- /dev/null +++ b/src/GarageStack.Tests/TelemetryCompactionTests.cs @@ -0,0 +1,81 @@ +using GarageStack.Core.Helpers; +using GarageStack.Core.Models; + +namespace GarageStack.Tests; + +public class TelemetryCompactionTests +{ + // On a quarter-hour, so T0.AddMinutes(15) starts the next window. + private static readonly DateTime T0 = new(2025, 3, 1, 10, 0, 0, DateTimeKind.Utc); + + [Fact] + public void AWindow_KeepsItsLastRow_FilledWithTheNewestEarlierValueOfEachField() + { + var first = new TelemetrySnapshot { Id = 1, RecordedAt = T0, FuelLevelPercent = 50, OdometerKm = 1000, EngineRunning = true }; + var second = new TelemetrySnapshot { Id = 2, RecordedAt = T0.AddMinutes(5), FuelLevelPercent = 49, BatteryVoltage = 12.6 }; + var last = new TelemetrySnapshot { Id = 3, RecordedAt = T0.AddMinutes(14), Latitude = 52.0, EngineRunning = false }; + + var folded = TelemetryCompaction.Fold([last, first, second]); + + Assert.Equal([1L, 2L], folded.Select(r => r.Id).Order()); + Assert.Equal(T0.AddMinutes(14), last.RecordedAt); + Assert.Equal(49.0, last.FuelLevelPercent); + Assert.Equal(1000.0, last.OdometerKm); + Assert.Equal(12.6, last.BatteryVoltage); + Assert.Equal(52.0, last.Latitude); + Assert.False(last.EngineRunning); + } + + [Fact] + public void RowsInDifferentWindows_StaySeparate() + { + var rows = new[] + { + new TelemetrySnapshot { Id = 1, RecordedAt = T0.AddMinutes(14).AddSeconds(59), FuelLevelPercent = 50 }, + new TelemetrySnapshot { Id = 2, RecordedAt = T0.AddMinutes(15), FuelLevelPercent = 49 }, + }; + + Assert.Empty(TelemetryCompaction.Fold(rows)); + Assert.Equal(50.0, rows[0].FuelLevelPercent); + } + + [Fact] + public void ACounterReset_CutsTheWindow_SoTheTotalBeforeItSurvives() + { + // The day's counters reach 125 km and 6.5 L, then start again from zero within the window. + var early = new TelemetrySnapshot { Id = 1, RecordedAt = T0.AddMinutes(1), MileageOfTheDay = 120, PowerUsageOfDay = 640 }; + var dayTotal = new TelemetrySnapshot { Id = 2, RecordedAt = T0.AddMinutes(4), MileageOfTheDay = 125, PowerUsageOfDay = 650 }; + var reset = new TelemetrySnapshot { Id = 3, RecordedAt = T0.AddMinutes(6), MileageOfTheDay = 0 }; + var after = new TelemetrySnapshot { Id = 4, RecordedAt = T0.AddMinutes(10), FuelLevelPercent = 40 }; + + var folded = TelemetryCompaction.Fold([early, dayTotal, reset, after]); + + Assert.Equal([1L, 3L], folded.Select(r => r.Id).Order()); + Assert.Equal(125.0, dayTotal.MileageOfTheDay); + Assert.Equal(650.0, dayTotal.PowerUsageOfDay); + Assert.Equal(0.0, after.MileageOfTheDay); + Assert.Null(after.PowerUsageOfDay); + } + + [Fact] + public void CompactedRows_ComeOutUnchanged() + { + var rows = new List + { + new() { Id = 1, RecordedAt = T0, MileageOfTheDay = 120 }, + new() { Id = 2, RecordedAt = T0.AddMinutes(4), MileageOfTheDay = 125 }, + new() { Id = 3, RecordedAt = T0.AddMinutes(6), MileageOfTheDay = 0 }, + new() { Id = 4, RecordedAt = T0.AddMinutes(20), OdometerKm = 1000 }, + }; + var kept = rows.Except(TelemetryCompaction.Fold(rows)).ToList(); + + Assert.Empty(TelemetryCompaction.Fold(kept)); + } + + [Fact] + public void WindowStart_IsTheQuarterHourATimeFallsIn() + { + Assert.Equal(T0.AddMinutes(15), TelemetryCompaction.WindowStart(T0.AddMinutes(29).AddSeconds(59))); + Assert.Equal(DateTimeKind.Utc, TelemetryCompaction.WindowStart(T0.AddMinutes(1)).Kind); + } +} diff --git a/src/GarageStack.Tests/TelemetryCompactorTests.cs b/src/GarageStack.Tests/TelemetryCompactorTests.cs new file mode 100644 index 00000000..707126b6 --- /dev/null +++ b/src/GarageStack.Tests/TelemetryCompactorTests.cs @@ -0,0 +1,171 @@ +using GarageStack.Core.Configuration; +using GarageStack.Core.Models; +using GarageStack.Data; +using GarageStack.Data.Repositories; +using GarageStack.Worker.Services; +using Microsoft.EntityFrameworkCore; + +namespace GarageStack.Tests; + +public class TelemetryCompactorTests +{ + private static readonly DateTime Now = new(2026, 6, 1, 12, 0, 0, DateTimeKind.Utc); + + // 90 days of full detail from Now: rows before 3 March 2026 may be compacted. + private static readonly TelemetryRetentionOptions NinetyDays = new() { FullDetailDays = 90 }; + private static readonly DateTime FirstKeptDay = new(2026, 3, 3, 0, 0, 0, DateTimeKind.Utc); + + private sealed class Setup : IAsyncDisposable + { + public AppDbContext Db { get; } = new(new DbContextOptionsBuilder() + .UseInMemoryDatabase(Guid.NewGuid().ToString()) + .Options); + + public Vehicle Vehicle { get; } = new() { Vin = "FAKEVN00000000001", TripsRecordedUntil = Now }; + + public Setup() + { + Db.Vehicles.Add(Vehicle); + Db.SaveChanges(); + } + + public async Task AddAsync(CancellationToken ct, params TelemetrySnapshot[] rows) + { + foreach (var row in rows) row.VehicleId = Vehicle.Id; + Db.TelemetrySnapshots.AddRange(rows); + await Db.SaveChangesAsync(ct); + Db.ChangeTracker.Clear(); + } + + // Read back the way the Worker does: fresh from the database, untracked. + public async Task CompactAsync(TelemetryRetentionOptions options, CancellationToken ct) + { + var vehicle = await Db.Vehicles.AsNoTracking().SingleAsync(ct); + return await new TelemetryCompactor(new HousekeepingRepository(Db), options).CompactAsync(vehicle, Now, ct); + } + + public Task> RowTimesAsync(CancellationToken ct) => + Db.TelemetrySnapshots.AsNoTracking().OrderBy(s => s.RecordedAt).Select(s => s.RecordedAt).ToListAsync(ct); + + public Task CompactedUntilAsync(CancellationToken ct) => + Db.Vehicles.AsNoTracking().Select(v => v.TelemetryCompactedUntil).SingleAsync(ct); + + public ValueTask DisposeAsync() => Db.DisposeAsync(); + } + + // Three polls in one quarter of an hour, starting at `start`. + private static TelemetrySnapshot[] Polls(DateTime start) => + [ + new() { RecordedAt = start, FuelLevelPercent = 50 }, + new() { RecordedAt = start.AddMinutes(5), BatteryVoltage = 12.6 }, + new() { RecordedAt = start.AddMinutes(10), OdometerKm = 1000 }, + ]; + + [Fact] + public async Task OnlyWholeDaysOlderThanTheFullDetailPeriod_AreCompacted() + { + var ct = TestContext.Current.CancellationToken; + await using var s = new Setup(); + var oldDay = FirstKeptDay.AddDays(-2).AddHours(10); + var keptDay = FirstKeptDay.AddHours(10); + await s.AddAsync(ct, [.. Polls(oldDay), .. Polls(keptDay)]); + + Assert.Equal(2, await s.CompactAsync(NinetyDays, ct)); + + Assert.Equal([oldDay.AddMinutes(10), .. Polls(keptDay).Select(p => p.RecordedAt)], await s.RowTimesAsync(ct)); + Assert.Equal(FirstKeptDay, await s.CompactedUntilAsync(ct)); + } + + [Fact] + public async Task Compaction_NeverPassesTheTripRecordingLine() + { + var ct = TestContext.Current.CancellationToken; + await using var s = new Setup(); + var recordedUntil = new DateTime(2026, 2, 15, 10, 30, 0, DateTimeKind.Utc); + s.Vehicle.TripsRecordedUntil = recordedUntil; + await s.Db.SaveChangesAsync(ct); + await s.AddAsync(ct, [.. Polls(recordedUntil.AddDays(-1)), .. Polls(recordedUntil.AddHours(-1))]); + + Assert.Equal(2, await s.CompactAsync(NinetyDays, ct)); + + Assert.Equal(recordedUntil.Date, await s.CompactedUntilAsync(ct)); + Assert.Equal(4, (await s.RowTimesAsync(ct)).Count); + } + + [Fact] + public async Task AVehicleWhoseTripsWereNeverRecorded_IsLeftAlone() + { + var ct = TestContext.Current.CancellationToken; + await using var s = new Setup(); + s.Vehicle.TripsRecordedUntil = null; + await s.Db.SaveChangesAsync(ct); + await s.AddAsync(ct, Polls(FirstKeptDay.AddDays(-30))); + + Assert.Equal(0, await s.CompactAsync(NinetyDays, ct)); + + Assert.Equal(3, (await s.RowTimesAsync(ct)).Count); + Assert.Null(await s.CompactedUntilAsync(ct)); + } + + [Fact] + public async Task SwitchedOff_KeepsEveryRow() + { + var ct = TestContext.Current.CancellationToken; + await using var s = new Setup(); + await s.AddAsync(ct, Polls(FirstKeptDay.AddDays(-300))); + + Assert.Equal(0, await s.CompactAsync(new TelemetryRetentionOptions { FullDetailDays = null }, ct)); + + Assert.Equal(3, (await s.RowTimesAsync(ct)).Count); + } + + [Fact] + public async Task RunningAgain_FoldsNothingMoreAndKeepsTheLine() + { + var ct = TestContext.Current.CancellationToken; + await using var s = new Setup(); + await s.AddAsync(ct, Polls(FirstKeptDay.AddDays(-10))); + + Assert.Equal(2, await s.CompactAsync(NinetyDays, ct)); + Assert.Equal(0, await s.CompactAsync(NinetyDays, ct)); + + Assert.Single(await s.RowTimesAsync(ct)); + Assert.Equal(FirstKeptDay, await s.CompactedUntilAsync(ct)); + } + + [Fact] + public async Task TheLatestReadings_AtTheEndOfEachWindow_AreTheSameAfterCompaction() + { + var ct = TestContext.Current.CancellationToken; + await using var s = new Setup(); + var start = FirstKeptDay.AddDays(-20).AddHours(10); + await s.AddAsync(ct, + new TelemetrySnapshot { RecordedAt = start, OdometerKm = 1000, Latitude = 52.0, Longitude = 5.0 }, + new TelemetrySnapshot { RecordedAt = start.AddMinutes(5), Latitude = 52.1, Longitude = 5.0, Speed = 40 }, + new TelemetrySnapshot { RecordedAt = start.AddMinutes(10), OdometerKm = 1003, FuelLevelPercent = 50 }, + new TelemetrySnapshot { RecordedAt = start.AddMinutes(20), Latitude = 52.2, Longitude = 5.0, Speed = 0 }, + new TelemetrySnapshot { RecordedAt = start.AddMinutes(29), OdometerKm = 1005 }); + + // The odometer and the car's last position as of each window's end. A kept row carries its + // window's last time, so only the readings are compared, not when a fix was taken. + var telemetry = new TelemetryRepository(s.Db); + DateTime[] windowEnds = [start.AddMinutes(15).AddTicks(-1), start.AddMinutes(30).AddTicks(-1), start.AddDays(1)]; + async Task> ReadAsync() + { + var readings = new List<(double?, double?, double?)>(); + foreach (var at in windowEnds) + { + var lastFix = (await telemetry.GetGpsFixesAsync(s.Vehicle.Id, start, at, ct)).LastOrDefault(); + readings.Add((await telemetry.GetOdometerAtAsync(s.Vehicle.Id, at, ct), lastFix?.Latitude, lastFix?.Longitude)); + } + + return readings; + } + + var before = await ReadAsync(); + Assert.Equal(3, await s.CompactAsync(NinetyDays, ct)); + + Assert.Equal(before, await ReadAsync()); + Assert.Equal((1005.0, 52.2, 5.0), (before[^1].Odometer!.Value, before[^1].Latitude!.Value, before[^1].Longitude!.Value)); + } +} diff --git a/src/GarageStack.Worker/Program.cs b/src/GarageStack.Worker/Program.cs index 3d0a5fe6..1f6df31b 100644 --- a/src/GarageStack.Worker/Program.cs +++ b/src/GarageStack.Worker/Program.cs @@ -36,6 +36,14 @@ builder.Services.AddHostedService(); builder.Services.AddHostedService(); + var retention = TelemetryRetentionOptions.From(builder.Configuration); + if (retention.FullDetailDays is { } fullDetailDays) + Log.Information("Telemetry keeps full detail for {Days} days, then is compacted into quarter-hour rows", fullDetailDays); + else + Log.Information("Telemetry compaction is off; every row is kept"); + builder.Services.AddSingleton(retention); + builder.Services.AddHostedService(); + var host = builder.Build(); host.Run(); diff --git a/src/GarageStack.Worker/Services/HousekeepingService.cs b/src/GarageStack.Worker/Services/HousekeepingService.cs new file mode 100644 index 00000000..03b33211 --- /dev/null +++ b/src/GarageStack.Worker/Services/HousekeepingService.cs @@ -0,0 +1,53 @@ +using GarageStack.Core.Configuration; +using GarageStack.Core.Helpers; +using GarageStack.Core.Interfaces; + +namespace GarageStack.Worker.Services; + +/// +/// Keeps the database from growing without end. Compacts each vehicle's old telemetry with +/// , and removes cached place names and snapped trips that have +/// expired, and notifications the user deleted a while ago. Cached map POIs stay: the map shows an +/// expired tile's stations while it fetches the tile again. +/// +public class HousekeepingService( + ILogger logger, + IServiceScopeFactory scopeFactory, + TelemetryRetentionOptions retention) : PeriodicBackgroundService(logger) +{ + // Nothing reads an expired answer, but a lookup may be refreshing it at this very moment. + // A day's grace keeps out of its way. + internal static readonly TimeSpan ExpiredCacheGrace = TimeSpan.FromDays(1); + + // The notification cooldowns read the history, deleted rows included, at most a week back. + internal static readonly TimeSpan DeletedNotificationsKept = TimeSpan.FromDays(30); + + protected override string Name => "Housekeeping"; + + // After the trip recorder's first run, whose line compaction waits for. + protected override TimeSpan InitialDelay => TimeSpan.FromMinutes(10); + + protected override TimeSpan Interval => TimeSpan.FromHours(6); + + protected override async Task RunOnceAsync(CancellationToken ct) + { + using var scope = scopeFactory.CreateScope(); + var housekeeping = scope.ServiceProvider.GetRequiredService(); + var vehicles = await scope.ServiceProvider.GetRequiredService().GetAllAsync(ct); + var compactor = new TelemetryCompactor(housekeeping, retention); + var now = DateTime.UtcNow; + + foreach (var vehicle in vehicles) + { + var folded = await compactor.CompactAsync(vehicle, now, ct); + if (folded > 0) + logger.LogInformation("Compacted old telemetry for VIN={Vin}: {Count} row(s) folded into quarter-hour rows", + LogRedaction.Vin(vehicle.Vin), folded); + } + + var expired = await housekeeping.RemoveExpiredCacheEntriesAsync(now - ExpiredCacheGrace, ct); + var deleted = await housekeeping.RemoveDeletedNotificationsAsync(now - DeletedNotificationsKept, ct); + if (expired > 0 || deleted > 0) + logger.LogInformation("Housekeeping removed {Expired} expired cache entries and {Deleted} deleted notifications", expired, deleted); + } +} diff --git a/src/GarageStack.Worker/Services/TelemetryCompactor.cs b/src/GarageStack.Worker/Services/TelemetryCompactor.cs new file mode 100644 index 00000000..08220289 --- /dev/null +++ b/src/GarageStack.Worker/Services/TelemetryCompactor.cs @@ -0,0 +1,33 @@ +using GarageStack.Core.Configuration; +using GarageStack.Core.Interfaces; +using GarageStack.Core.Models; + +namespace GarageStack.Worker.Services; + +/// +/// Compacts a vehicle's telemetry once it is older than the full-detail period, a whole UTC day at +/// a time, from its compaction line onwards. It never passes the trip recording line: the trip +/// recorder cuts trips from every fix after that line, and a quarter-hour row has lost the fixes in +/// between. A vehicle whose trips have never been recorded is left alone for the same reason. +/// +public class TelemetryCompactor(IHousekeepingRepository housekeeping, TelemetryRetentionOptions options) +{ + /// How many rows were folded away. + public async Task CompactAsync(Vehicle vehicle, DateTime now, CancellationToken ct) + { + if (options.FullDetailDays is not { } days || vehicle.TripsRecordedUntil is not { } recordedUntil) + return 0; + + var fullDetailFrom = now.AddDays(-days); + var until = (recordedUntil < fullDetailFrom ? recordedUntil : fullDetailFrom).Date; + var from = vehicle.TelemetryCompactedUntil + ?? (await housekeeping.GetFirstTelemetryAtAsync(vehicle.Id, ct))?.Date; + if (from is null) return 0; + + var folded = 0; + for (var day = from.Value; day < until; day = day.AddDays(1)) + folded += await housekeeping.CompactTelemetryAsync(vehicle.Id, day, day.AddDays(1), ct); + + return folded; + } +} diff --git a/unraid/garagestack.xml b/unraid/garagestack.xml index 92535d4e..f4d3f5e6 100644 --- a/unraid/garagestack.xml +++ b/unraid/garagestack.xml @@ -532,6 +532,17 @@ Required="false" Mask="false"> + 365 +