diff --git a/Services/Caching/CacheService.cs b/Services/Caching/CacheService.cs index 85fe8f5..f966f3d 100644 --- a/Services/Caching/CacheService.cs +++ b/Services/Caching/CacheService.cs @@ -182,6 +182,30 @@ public string BuildCacheKey(string endpoint, IQueryCollection query, string? use ? el.GetString() : null; + /// Honours an explicit ?{InvalidateParam}=true on the request. + public void ApplyRequestedInvalidation(HttpContext context) + { + if (!context.Request.Query.ContainsKey(InvalidateParam)) return; + if (!string.Equals(context.Request.Query[InvalidateParam], "true", StringComparison.OrdinalIgnoreCase)) + return; + + InvalidateScopedOrAll(context); + } + + /// + /// Invalidates just the caller's scope when one is present, otherwise everything — without a scope + /// there is no way to know which entries a write affected. + /// + public void InvalidateScopedOrAll(HttpContext context) + { + var scopeValue = !string.IsNullOrEmpty(ScopeParam) + ? context.Request.Query[ScopeParam].ToString() + : ""; + + if (!string.IsNullOrEmpty(scopeValue)) InvalidateByScope(scopeValue); + else InvalidateAll(); + } + /// /// Get the TTL for a given endpoint. /// diff --git a/Services/Configuration/EgressLimitSettings.cs b/Services/Configuration/EgressLimitSettings.cs index edbcd08..e5d5a38 100644 --- a/Services/Configuration/EgressLimitSettings.cs +++ b/Services/Configuration/EgressLimitSettings.cs @@ -69,8 +69,8 @@ public class EgressLimitSettings /// /// Days of bucketed rows to retain in the table before a periodic purge deletes them. Default 7. - /// Floored at 1. Env override: CRAFT_API_EGRESS_RETENTION_DAYS. Does not affect the local file, - /// which only ever holds the current UTC day. + /// Floored at 1. Env override: CRAFT_API_EGRESS_RETENTION_DAYS. Also the number of days (today + /// included) the local ledger file keeps. /// public int RetentionDays { get; set; } = 7; diff --git a/Services/Endpoints/CraftEndpointAttribute.cs b/Services/Endpoints/CraftEndpointAttribute.cs index b512a3c..1830a4d 100644 --- a/Services/Endpoints/CraftEndpointAttribute.cs +++ b/Services/Endpoints/CraftEndpointAttribute.cs @@ -23,6 +23,25 @@ public enum EndpointDispatch Direct, } +/// +/// Whether a native endpoint's GET responses go through the response cache (App:Cache). +/// +public enum EndpointCaching +{ + /// + /// The PowerShell path's rule: GETs to a route named List… are cached, the name being the + /// endpoint's promise that it is a side-effect-free read. The default, so native and PowerShell + /// endpoints of the same name cache alike. + /// + Convention, + + /// Every GET is cached, whatever the route is called — a read that isn't named List…. + Cached, + + /// Never cached: a read whose answer must be live (a status view, a log tail). + Uncached, +} + /// /// Marks a class as a native endpoint and declares its route and metadata. The C# counterpart of the /// PowerShell convention where Invoke-GetIPInfo becomes /API/GetIPInfo and the @@ -87,4 +106,11 @@ public CraftEndpointAttribute(string route) /// unaffected until they ship one. /// public EndpointDispatch Dispatch { get; init; } = EndpointDispatch.Central; + + /// + /// Whether this endpoint's GET responses are cached. Defaults to + /// (GETs to List… routes), as the PowerShell path does; App:Cache:ExcludedEndpoints and the rest of + /// the cache policy still apply on top of it. + /// + public EndpointCaching Caching { get; init; } = EndpointCaching.Convention; } diff --git a/Services/Endpoints/CraftResult.cs b/Services/Endpoints/CraftResult.cs index 65ce38e..2a78c73 100644 --- a/Services/Endpoints/CraftResult.cs +++ b/Services/Endpoints/CraftResult.cs @@ -2,6 +2,7 @@ using System.Text; using System.Text.Json; using Craft.Hosting; +using Craft.Services; namespace Craft.Endpoints; @@ -105,6 +106,25 @@ public static CraftResult Problem(int status, string message) => /// public static CraftResult Handled { get; } = new(Kind.Handled, 0, null, null); + /// + /// This result as the response cache stores it, or null when it can't be: only a JSON body with a 2xx or 3xx + /// status, as the PowerShell path caches. Bytes, streams and endpoint-written responses are never cached. + /// + internal ScriptResult? ToCacheable() => + _kind == Kind.Json && StatusCode is >= 200 and < 400 + ? new ScriptResult + { + StatusCode = StatusCode, + Body = _text ?? "null", + ContentType = ContentType, + Headers = Headers?.ToDictionary(h => h.Key, h => h.Value), + } + : null; + + /// A cached response, written back exactly as it was stored. + internal static CraftResult FromCached(ScriptResult cached) => + new(Kind.Json, cached.StatusCode, cached.ContentType ?? "application/json", cached.Headers, text: cached.Body); + /// Writes this result to the response. internal async Task WriteAsync(HttpContext context, CancellationToken ct) { diff --git a/Services/Hosting/ApiEgressLimiterMiddleware.cs b/Services/Hosting/ApiEgressLimiterMiddleware.cs index 1a2c693..bf84258 100644 --- a/Services/Hosting/ApiEgressLimiterMiddleware.cs +++ b/Services/Hosting/ApiEgressLimiterMiddleware.cs @@ -61,7 +61,7 @@ public async Task InvokeAsync(HttpContext context) // concurrency cap, the overshoot is bounded to (concurrency × largest response). if (_ledger.ShouldReject()) { - _ledger.RecordShed(appId, context.GetRouteValue("endpoint") as string); + _ledger.RecordShed(appId, context.GetRouteValue("endpoint") as string, RateLimitPartitionKey.ClientAddress(context)); await RejectAsync(context); return; } diff --git a/Services/Hosting/ApiEgressWireCounterMiddleware.cs b/Services/Hosting/ApiEgressWireCounterMiddleware.cs index fc68b6c..52aff0c 100644 --- a/Services/Hosting/ApiEgressWireCounterMiddleware.cs +++ b/Services/Hosting/ApiEgressWireCounterMiddleware.cs @@ -78,9 +78,9 @@ public async Task InvokeAsync(HttpContext context) { context.Response.Body = original; if (context.Items.TryGetValue(ChargeItemKey, out var charge) && charge is string appId && appId.Length > 0) - _ledger.Record(counting.BytesWritten, appId, ResolveEndpoint(context), IsCacheHit(context), context.Response.StatusCode); + _ledger.Record(counting.BytesWritten, appId, ResolveEndpoint(context), IsCacheHit(context), context.Response.StatusCode, RateLimitPartitionKey.ClientAddress(context), context.Response.Headers.ContentEncoding.ToString()); else if (context.Items.ContainsKey(InteractiveItemKey)) - _ledger.RecordInteractive(counting.BytesWritten, ResolveEndpoint(context), IsCacheHit(context), context.Response.StatusCode); + _ledger.RecordInteractive(counting.BytesWritten, ResolveEndpoint(context), IsCacheHit(context), context.Response.StatusCode, RateLimitPartitionKey.ClientAddress(context), context.Response.Headers.ContentEncoding.ToString()); } } diff --git a/Services/Hosting/EgressLedger.cs b/Services/Hosting/EgressLedger.cs index 6a66465..b9dca7b 100644 --- a/Services/Hosting/EgressLedger.cs +++ b/Services/Hosting/EgressLedger.cs @@ -53,6 +53,8 @@ public sealed class EgressLedger : BackgroundService private bool _tableDirty; // something changed since the last table sync private readonly Dictionary _clients = new(StringComparer.OrdinalIgnoreCase); + // Finished days kept in the file, newest first, so readers of the file alone see a week of history. + private List _history = []; // Interactive (user) traffic: reported in its own partition, never part of _bytes or the cap. private ClientTotals _interactive = new(); // Pending table buckets not yet finalised: bucketRowKey -> (partition -> accum). Partitions are the @@ -66,7 +68,8 @@ private sealed class ClientTotals { public long Bytes; public long Requests; public long Shed; public DateTime LastSeenUtc; public Dictionary Endpoints = new(StringComparer.OrdinalIgnoreCase); - public ClientTotals Clone() => new() { Bytes = Bytes, Requests = Requests, Shed = Shed, LastSeenUtc = LastSeenUtc, Endpoints = EndpointStats.CloneMap(Endpoints) }; + public Dictionary Ips = new(StringComparer.OrdinalIgnoreCase); + public ClientTotals Clone() => new() { Bytes = Bytes, Requests = Requests, Shed = Shed, LastSeenUtc = LastSeenUtc, Endpoints = EndpointStats.CloneMap(Endpoints), Ips = EndpointStats.CloneMap(Ips) }; } private sealed class BucketAccum @@ -134,7 +137,7 @@ public bool ShouldReject() /// Add the outbound bytes of one API-client response to today's running totals, attributed to /// the client and, in its daily and the instance bucket breakdowns, to . - public void Record(long bytes, string appId, string? endpoint = null, bool cacheHit = false, int statusCode = 200) + public void Record(long bytes, string appId, string? endpoint = null, bool cacheHit = false, int statusCode = 200, string? ip = null, string? encoding = null) { if (bytes <= 0) return; appId = Normalize(appId); @@ -147,14 +150,15 @@ public void Record(long bytes, string appId, string? endpoint = null, bool cache client.Bytes += bytes; client.Requests += 1; client.LastSeenUtc = _utcNow(); - EndpointStats.In(client.Endpoints, label).Add(bytes, cacheHit, statusCode); + EndpointStats.In(client.Endpoints, label).Add(bytes, cacheHit, statusCode, encoding); + if (!string.IsNullOrEmpty(ip)) EndpointStats.In(client.Ips, ip).Add(bytes, cacheHit, statusCode, encoding); var bucket = Bucket(appId); bucket.Bytes += bytes; bucket.Requests += 1; var system = Bucket(EgressTableSchema.SystemPartition); system.Bytes += bytes; system.Requests += 1; - EndpointStats.In(system.Endpoints, label).Add(bytes, cacheHit, statusCode); + EndpointStats.In(system.Endpoints, label).Add(bytes, cacheHit, statusCode, encoding); _dirty = true; _tableDirty = true; } @@ -162,7 +166,7 @@ public void Record(long bytes, string appId, string? endpoint = null, bool cache /// Add the outbound bytes of one interactive (user) response. Reported under /// ; never counts toward the cap. - public void RecordInteractive(long bytes, string? endpoint = null, bool cacheHit = false, int statusCode = 200) + public void RecordInteractive(long bytes, string? endpoint = null, bool cacheHit = false, int statusCode = 200, string? ip = null, string? encoding = null) { if (bytes <= 0) return; var label = EgressTableSchema.EndpointLabel(endpoint); @@ -172,11 +176,12 @@ public void RecordInteractive(long bytes, string? endpoint = null, bool cacheHit _interactive.Bytes += bytes; _interactive.Requests += 1; _interactive.LastSeenUtc = _utcNow(); - EndpointStats.In(_interactive.Endpoints, label).Add(bytes, cacheHit, statusCode); + EndpointStats.In(_interactive.Endpoints, label).Add(bytes, cacheHit, statusCode, encoding); + if (!string.IsNullOrEmpty(ip)) EndpointStats.In(_interactive.Ips, ip).Add(bytes, cacheHit, statusCode, encoding); var bucket = Bucket(EgressTableSchema.InteractivePartition); bucket.Bytes += bytes; bucket.Requests += 1; - EndpointStats.In(bucket.Endpoints, label).Add(bytes, cacheHit, statusCode); + EndpointStats.In(bucket.Endpoints, label).Add(bytes, cacheHit, statusCode, encoding); _dirty = true; _tableDirty = true; } @@ -184,7 +189,7 @@ public void RecordInteractive(long bytes, string? endpoint = null, bool cacheHit /// Record that a request from was shed (429) once over budget. The /// shed response body is deliberately not counted as bytes; this tracks the cap-hit count. - public void RecordShed(string appId, string? endpoint = null) + public void RecordShed(string appId, string? endpoint = null, string? ip = null) { appId = Normalize(appId); var label = EgressTableSchema.EndpointLabel(endpoint); @@ -196,6 +201,7 @@ public void RecordShed(string appId, string? endpoint = null) var client = Client(appId); client.Shed += 1; EndpointStats.In(client.Endpoints, label).Shed += 1; + if (!string.IsNullOrEmpty(ip)) EndpointStats.In(client.Ips, ip).Shed += 1; Bucket(appId).Shed += 1; var system = Bucket(EgressTableSchema.SystemPartition); system.Shed += 1; @@ -251,6 +257,8 @@ private void RolloverIfNeeded() { var today = DateOnly.FromDateTime(_utcNow()); if (today == _dateUtc) return; + _history = [Snapshot(), .. _history]; + PruneHistory(today); _dateUtc = today; _bytes = 0; _shedRequests = 0; @@ -317,8 +325,9 @@ private async Task InitTableAsync(CancellationToken ct) } } - /// Load the persisted daily totals. Restores only when the file is stamped with today's UTC - /// date; a stale or unreadable file leaves today starting from zero. Tolerates the v1 and v2 formats. + /// Load the persisted daily totals and the finished days. Today's totals are restored only when the + /// file is stamped with today's UTC date; a stale file's day moves into the history instead, and an unreadable + /// file leaves today starting from zero. Tolerates the v1 to v3 formats. internal void Load() { try @@ -333,12 +342,22 @@ internal void Load() DateTimeStyles.None, out var stored) && stored == today; if (!sameDay || state.Bytes < 0) { + lock (_lock) + { + _history = state.Days ?? []; + state.Days = null; + if (state.Bytes >= 0) _history.Insert(0, state); + PruneHistory(today); + _dirty = true; + } _logger.LogInformation("[Egress] Ledger file not for today ({Stored}) — starting the day fresh", state.DateUtc); return; } lock (_lock) { + _history = state.Days ?? []; + PruneHistory(today); _dateUtc = today; _bytes = state.Bytes; _shedRequests = Math.Max(0, state.ShedRequests); @@ -371,18 +390,8 @@ internal void Flush() { if (!_dirty) return; RolloverIfNeeded(); - snapshot = new LedgerState - { - Version = 3, - DateUtc = _dateUtc.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture), - Bytes = _bytes, - CapBytes = _capBytes, - CapReachedUtc = _capReachedUtc, - ShedRequests = _shedRequests, - UpdatedUtc = _utcNow(), - Clients = _clients.ToDictionary(kv => kv.Key, kv => ClientState.From(kv.Value)), - Interactive = _interactive.Requests > 0 ? ClientState.From(_interactive) : null, - }; + snapshot = Snapshot(); + snapshot.Days = _history; _dirty = false; } @@ -400,6 +409,30 @@ internal void Flush() } } + // Must be called under _lock. Today's totals in the file shape, without the history. + private LedgerState Snapshot() => new() + { + Version = 4, + DateUtc = _dateUtc.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture), + Bytes = _bytes, + CapBytes = _capBytes, + CapReachedUtc = _capReachedUtc, + ShedRequests = _shedRequests, + UpdatedUtc = _utcNow(), + Clients = _clients.ToDictionary(kv => kv.Key, kv => ClientState.From(kv.Value)), + Interactive = _interactive.Requests > 0 ? ClientState.From(_interactive) : null, + }; + + // Must be called under _lock. Keeps the finished days inside the retention window (today counts as one). + private void PruneHistory(DateOnly today) + { + var oldest = today.AddDays(1 - _retentionDays).ToString("yyyy-MM-dd", CultureInfo.InvariantCulture); + _history = _history + .Where(d => string.CompareOrdinal(d.DateUtc, oldest) >= 0 && string.CompareOrdinal(d.DateUtc, today.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)) < 0) + .OrderByDescending(d => d.DateUtc, StringComparer.Ordinal) + .ToList(); + } + // ── Table mirror ───────────────────────────────────────────────────────────────────────────────── /// Mirror the current bucket(s) and today's daily summary to the table. Only rows for clients @@ -416,6 +449,7 @@ internal async Task SyncToTableAsync(CancellationToken ct) Dictionary clientDaily; ClientTotals? interactiveDaily; var instanceEndpoints = new Dictionary(StringComparer.OrdinalIgnoreCase); + var instanceIps = new Dictionary(StringComparer.OrdinalIgnoreCase); lock (_lock) { if (!_tableDirty && !_buckets.Keys.Any(k => k != CurrentBucketKey())) return; @@ -426,11 +460,18 @@ internal async Task SyncToTableAsync(CancellationToken ct) clientDaily = active.Where(a => _clients.ContainsKey(a)).ToDictionary(a => a, a => _clients[a].Clone(), StringComparer.OrdinalIgnoreCase); interactiveDaily = active.Contains(EgressTableSchema.InteractivePartition) ? _interactive.Clone() : null; foreach (var client in _clients.Values) + { foreach (var (label, stats) in client.Endpoints) { if (!instanceEndpoints.TryGetValue(label, out var sum)) instanceEndpoints[label] = sum = new EndpointStats(); sum.Merge(stats); } + foreach (var (ip, stats) in client.Ips) + { + if (!instanceIps.TryGetValue(ip, out var sum)) instanceIps[ip] = sum = new EndpointStats(); + sum.Merge(stats); + } + } day = _dateUtc; dayBytes = _bytes; dayRequests = _clients.Values.Sum(c => c.Requests); dayShed = _shedRequests; capReached = _capReachedUtc; enforcing = _capBytes > 0; _tableDirty = false; @@ -453,11 +494,11 @@ internal async Task SyncToTableAsync(CancellationToken ct) // Daily audit rows — per active client, interactive, and the instance summary (cap, when hit, how many shed). foreach (var (appId, c) in clientDaily) - await _store!.UpsertAsync(_tableName, EgressTableSchema.ClientDailyRow(appId, day, c.Bytes, c.Requests, c.Shed, _capBytes, c.Endpoints), ct).ConfigureAwait(false); + await _store!.UpsertAsync(_tableName, EgressTableSchema.ClientDailyRow(appId, day, c.Bytes, c.Requests, c.Shed, _capBytes, c.Endpoints, c.Ips), ct).ConfigureAwait(false); if (interactiveDaily is not null) await _store!.UpsertAsync(_tableName, EgressTableSchema.ClientDailyRow(EgressTableSchema.InteractivePartition, day, - interactiveDaily.Bytes, interactiveDaily.Requests, 0, _capBytes, interactiveDaily.Endpoints), ct).ConfigureAwait(false); - await _store!.UpsertAsync(_tableName, EgressTableSchema.SystemDailyRow(day, dayBytes, dayRequests, dayShed, _capBytes, capReached, enforcing, instanceEndpoints), ct).ConfigureAwait(false); + interactiveDaily.Bytes, interactiveDaily.Requests, 0, _capBytes, interactiveDaily.Endpoints, interactiveDaily.Ips), ct).ConfigureAwait(false); + await _store!.UpsertAsync(_tableName, EgressTableSchema.SystemDailyRow(day, dayBytes, dayRequests, dayShed, _capBytes, capReached, enforcing, instanceEndpoints, instanceIps), ct).ConfigureAwait(false); // Drop finalised buckets we just wrote (strictly older than the current one). lock (_lock) @@ -573,6 +614,8 @@ private sealed class LedgerState public DateTime? UpdatedUtc { get; set; } public Dictionary? Clients { get; set; } public ClientState? Interactive { get; set; } + /// Finished days, newest first, within the retention window (v4+). + public List? Days { get; set; } } private sealed class ClientState @@ -581,8 +624,10 @@ private sealed class ClientState public long Requests { get; set; } public long Shed { get; set; } public DateTime LastSeenUtc { get; set; } - // Same [Bytes, Requests, MaxBytes, CacheHits, Errors, Shed] layout as the table column. + // Same layout as the table column (EgressTableSchema.PropEndpoints). public Dictionary? Endpoints { get; set; } + // Same layout, keyed by the caller's address (v4+). + public Dictionary? Ips { get; set; } public static ClientState From(ClientTotals t) => new() { @@ -591,6 +636,7 @@ private sealed class ClientState Shed = t.Shed, LastSeenUtc = t.LastSeenUtc, Endpoints = t.Endpoints.ToDictionary(kv => kv.Key, kv => kv.Value.ToArray()), + Ips = t.Ips.ToDictionary(kv => kv.Key, kv => kv.Value.ToArray()), }; public ClientTotals ToTotals() @@ -598,6 +644,8 @@ public ClientTotals ToTotals() var t = new ClientTotals { Bytes = Bytes, Requests = Requests, Shed = Shed, LastSeenUtc = LastSeenUtc }; foreach (var (label, values) in Endpoints ?? []) t.Endpoints[label] = EndpointStats.FromArray(values); + foreach (var (ip, values) in Ips ?? []) + t.Ips[ip] = EndpointStats.FromArray(values); return t; } } diff --git a/Services/Hosting/EgressTableSchema.cs b/Services/Hosting/EgressTableSchema.cs index a8b7fb3..2e82ddd 100644 --- a/Services/Hosting/EgressTableSchema.cs +++ b/Services/Hosting/EgressTableSchema.cs @@ -50,9 +50,12 @@ internal static class EgressTableSchema public const string PropCapReachedUtc = "CapReachedUtc"; public const string PropEnforcing = "Enforcing"; public const string PropAppId = "AppId"; - // JSON object of endpoint label -> [Bytes, Requests, MaxBytes, CacheHits, Errors, Shed]. Arrays rather + // JSON object of endpoint label -> [Bytes, Requests, MaxBytes, CacheHits, Errors, Shed, BrRequests, BrBytes, + // GzipRequests, GzipBytes]; uncompressed is the remainder. Arrays rather // than named fields keep a full map under the 32K-char property limit; CIPP can't reassemble Craft splits. public const string PropEndpoints = "Endpoints"; + // Same layout as Endpoints, keyed by caller IP. Daily rows only. + public const string PropIps = "Ips"; /// Endpoints kept per map; the rest fold into . public const int MaxEndpoints = 100; @@ -111,10 +114,11 @@ public static StoreRow SystemBucketRow(DateTime bucketStartUtc, } public static StoreRow ClientDailyRow(string appId, DateOnly dayUtc, - long bytes, long requests, long shed, long capBytes, IReadOnlyDictionary? endpoints = null) + long bytes, long requests, long shed, long capBytes, IReadOnlyDictionary? endpoints = null, + IReadOnlyDictionary? ips = null) { var row = new StoreRow(appId, DailyRowKey(dayUtc)); - FillCounts(row, bytes, requests, shed, capBytes, endpoints); + FillCounts(row, bytes, requests, shed, capBytes, endpoints, ips); row[PropDateUtc] = dayUtc.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture); row[PropAppId] = appId; return row; @@ -122,10 +126,10 @@ public static StoreRow ClientDailyRow(string appId, DateOnly dayUtc, public static StoreRow SystemDailyRow(DateOnly dayUtc, long bytes, long requests, long shed, long capBytes, DateTime? capReachedUtc, bool enforcing, - IReadOnlyDictionary? endpoints = null) + IReadOnlyDictionary? endpoints = null, IReadOnlyDictionary? ips = null) { var row = new StoreRow(SystemPartition, DailyRowKey(dayUtc)); - FillCounts(row, bytes, requests, shed, capBytes, endpoints); + FillCounts(row, bytes, requests, shed, capBytes, endpoints, ips); row[PropDateUtc] = dayUtc.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture); row[PropEnforcing] = enforcing; if (capReachedUtc.HasValue) row[PropCapReachedUtc] = capReachedUtc.Value; @@ -133,13 +137,14 @@ public static StoreRow SystemDailyRow(DateOnly dayUtc, } private static void FillCounts(StoreRow row, long bytes, long requests, long shed, long capBytes, - IReadOnlyDictionary? endpoints) + IReadOnlyDictionary? endpoints, IReadOnlyDictionary? ips = null) { row[PropBytes] = bytes; row[PropRequests] = requests; row[PropShed] = shed; row[PropCapBytes] = capBytes; if (endpoints is { Count: > 0 }) row[PropEndpoints] = SerializeEndpoints(endpoints); + if (ips is { Count: > 0 }) row[PropIps] = SerializeEndpoints(ips); } /// Clamps an endpoint label to a bounded, table-safe form; blank is . @@ -191,15 +196,17 @@ public static Dictionary ParseEndpoints(string? json) /// Per-endpoint counters within one client-day or one bucket. internal sealed class EndpointStats { - public long Bytes, Requests, MaxBytes, CacheHits, Errors, Shed; + public long Bytes, Requests, MaxBytes, CacheHits, Errors, Shed, BrRequests, BrBytes, GzipRequests, GzipBytes; - public void Add(long bytes, bool cacheHit, int statusCode) + public void Add(long bytes, bool cacheHit, int statusCode, string? encoding = null) { Bytes += bytes; Requests += 1; if (bytes > MaxBytes) MaxBytes = bytes; if (cacheHit) CacheHits += 1; if (statusCode >= 400) Errors += 1; + if (encoding == "br") { BrRequests += 1; BrBytes += bytes; } + else if (encoding == "gzip") { GzipRequests += 1; GzipBytes += bytes; } } public void Merge(EndpointStats other) @@ -210,6 +217,10 @@ public void Merge(EndpointStats other) CacheHits += other.CacheHits; Errors += other.Errors; Shed += other.Shed; + BrRequests += other.BrRequests; + BrBytes += other.BrBytes; + GzipRequests += other.GzipRequests; + GzipBytes += other.GzipBytes; } public EndpointStats Clone() => (EndpointStats)MemberwiseClone(); @@ -227,11 +238,23 @@ public static EndpointStats In(Dictionary map, string lab public static Dictionary CloneMap(Dictionary map) => map.ToDictionary(kv => kv.Key, kv => kv.Value.Clone(), StringComparer.OrdinalIgnoreCase); - public long[] ToArray() => [Bytes, Requests, MaxBytes, CacheHits, Errors, Shed]; + public long[] ToArray() => [Bytes, Requests, MaxBytes, CacheHits, Errors, Shed, BrRequests, BrBytes, GzipRequests, GzipBytes]; public static EndpointStats FromArray(long[]? v) { long At(int i) => v is not null && i < v.Length ? v[i] : 0; - return new EndpointStats { Bytes = At(0), Requests = At(1), MaxBytes = At(2), CacheHits = At(3), Errors = At(4), Shed = At(5) }; + return new EndpointStats + { + Bytes = At(0), + Requests = At(1), + MaxBytes = At(2), + CacheHits = At(3), + Errors = At(4), + Shed = At(5), + BrRequests = At(6), + BrBytes = At(7), + GzipRequests = At(8), + GzipBytes = At(9) + }; } } diff --git a/Services/Hosting/Endpoints/AuthEndpoints.cs b/Services/Hosting/Endpoints/AuthEndpoints.cs index 8c5ef41..ef2910d 100644 --- a/Services/Hosting/Endpoints/AuthEndpoints.cs +++ b/Services/Hosting/Endpoints/AuthEndpoints.cs @@ -15,14 +15,17 @@ namespace Craft.Hosting.Endpoints; public static class AuthEndpoints { /// Maps /.auth/me and /api/me. + /// + /// A native endpoint answers /api/me (see ), so it is not mapped here: an + /// application without PowerShell workers has nothing to run the PowerShell me function on. + /// public static WebApplication MapCraftAuthEndpoints( - this WebApplication app, CraftSettings settings, ILogger logger) + this WebApplication app, CraftSettings settings, ILogger logger, bool meIsNative = false) { ArgumentNullException.ThrowIfNull(app); ArgumentNullException.ThrowIfNull(settings); ArgumentNullException.ThrowIfNull(logger); - var psRunner = app.Services.GetRequiredService(); var isDevelopment = app.Environment.IsDevelopment(); // In production EasyAuth serves /.auth/me at the edge, so this handler is shadowed and only @@ -40,11 +43,16 @@ public static WebApplication MapCraftAuthEndpoints( : null, })); + if (meIsNative) + { + logger.LogInformation("[Auth] /api/me is answered by the native endpoint {Route}", MeRoute(settings)); + return app; + } + + var psRunner = app.Services.GetRequiredService(); app.MapGet("/api/me", async (HttpContext context) => { - var meFunction = string.IsNullOrEmpty(settings.Auth.MeEndpointFunction) - ? "me" - : settings.Auth.MeEndpointFunction; + var meFunction = MeRoute(settings); var request = await PowerShellRunnerService.SnapshotRequest(context); var parameters = (Hashtable)request["Params"]!; @@ -90,4 +98,11 @@ await context.Response.WriteAsync(JsonSerializer.Serialize(new return app; } + + /// + /// The endpoint behind /api/me: , or "me". A native + /// endpoint on this route answers /api/me itself; otherwise the PowerShell function of that name does. + /// + public static string MeRoute(CraftSettings settings) => + string.IsNullOrEmpty(settings.Auth.MeEndpointFunction) ? "me" : settings.Auth.MeEndpointFunction; } diff --git a/Services/Hosting/Endpoints/NativeDispatchEndpoint.cs b/Services/Hosting/Endpoints/NativeDispatchEndpoint.cs index efde597..fd52f6f 100644 --- a/Services/Hosting/Endpoints/NativeDispatchEndpoint.cs +++ b/Services/Hosting/Endpoints/NativeDispatchEndpoint.cs @@ -1,5 +1,6 @@ using System.Collections.Concurrent; using System.Globalization; +using Craft.Caching; using Craft.Configuration; using Craft.Endpoints; @@ -29,7 +30,8 @@ public static WebApplication MapCraftNativeEndpoints( IReadOnlyList endpoints, RequestCounter activeRequests, ILogger logger, - EndpointSettings? settings = null) + EndpointSettings? settings = null, + string? meRoute = null) { ArgumentNullException.ThrowIfNull(app); ArgumentNullException.ThrowIfNull(endpoints); @@ -42,6 +44,7 @@ public static WebApplication MapCraftNativeEndpoints( // startup-time fact — resolve it once here rather than per request. var handler = app.Services.GetService(); EnsureHandlerRequirement(settings, handler is not null, endpoints); + var cache = app.Services.GetService(); if (handler is not null) { @@ -90,7 +93,14 @@ public static WebApplication MapCraftNativeEndpoints( } app.MapMethods(path, descriptor.Metadata.Methods, - (HttpContext context) => Handle(context, descriptor, handler, activeRequests, logger)); + (HttpContext context) => Handle(context, descriptor, handler, cache, activeRequests, logger)); + + // The SPA asks /api/me who is signed in; an endpoint on the configured route answers it, + // through the same gate, filters and central handler as its own path (which for a route + // named "me" already is /api/me: routes ignore case, so mapping it twice would be ambiguous) + if (string.Equals(descriptor.Route, meRoute, StringComparison.OrdinalIgnoreCase) + && !string.Equals(path, "/api/me", StringComparison.OrdinalIgnoreCase)) + app.MapGet("/api/me", (HttpContext context) => Handle(context, descriptor, handler, cache, activeRequests, logger)); } if (endpoints.Count > 0) @@ -136,18 +146,107 @@ internal static ValueTask ExecuteAsync( CraftRequest request, ICraftEndpoint endpoint, ICraftEndpointHandler? handler, + CancellationToken ct) => + ExecuteAsync(request, () => endpoint.HandleAsync(request, ct), handler, ct); + + /// As above, with standing in for the endpoint (the response cache). + internal static ValueTask ExecuteAsync( + CraftRequest request, + CraftEndpointNext invokeEndpoint, + ICraftEndpointHandler? handler, CancellationToken ct) { if (handler is null || request.Endpoint.Dispatch != EndpointDispatch.Central) - return endpoint.HandleAsync(request, ct); + return invokeEndpoint(); - return handler.HandleAsync(request, () => endpoint.HandleAsync(request, ct), ct); + return handler.HandleAsync(request, invokeEndpoint, ct); + } + + /// Whether this request's response may come from, and go into, the response cache. + internal static bool IsCacheable(NativeEndpointDescriptor descriptor, string method) => + HttpMethods.IsGet(method) && descriptor.Metadata.Caching switch + { + EndpointCaching.Cached => true, + EndpointCaching.Uncached => false, + _ => descriptor.Route.StartsWith("List", StringComparison.OrdinalIgnoreCase), + }; + + /// + /// The endpoint's result through the cache: a hit is served as stored and refreshed in the background (the + /// PowerShell path's stale-while-revalidate); a miss runs the endpoint and stores a cacheable result. + /// + private static async ValueTask FromCacheAsync( + HttpContext context, CraftRequest request, ICraftEndpoint endpoint, NativeEndpointDescriptor descriptor, + ICraftEndpointHandler? handler, CacheService cache, ILogger logger, CancellationToken ct) + { + var key = cache.BuildCacheKey(descriptor.Route, context.Request.Query, CacheService.GetUserKey(context)); + if (await cache.Get(key, descriptor.Route) is { } hit) + { + cache.Touch(key); + context.Response.Headers["X-Cache"] = hit.IsStale ? "HIT-STALE" : "HIT"; + context.Response.Headers["X-Cache-Age"] = $"{hit.Age.TotalSeconds:F0}s"; + context.Response.Headers["X-Cache-TTL"] = $"{cache.GetTtl(descriptor.Route).TotalSeconds:F0}s"; + if (cache.TryStartRefresh(key)) StartBackgroundRefresh(context, descriptor, handler, cache, key, logger); + return CraftResult.FromCached(hit.Result); + } + + var result = await endpoint.HandleAsync(request, ct); + if (!ct.IsCancellationRequested && result.ToCacheable() is { } cacheable) await cache.Set(key, cacheable); + context.Response.Headers["X-Cache"] = "MISS"; + return result; + } + + /// + /// Re-runs the request off the request path and replaces the entry. Rebuilt from a snapshot in a scope of its own + /// (the caller's context is gone by then), and back through the central handler, so a caller who has since lost + /// access refreshes nothing. + /// + private static void StartBackgroundRefresh( + HttpContext context, NativeEndpointDescriptor descriptor, ICraftEndpointHandler? handler, + CacheService cache, string key, ILogger logger) + { + var scopes = context.RequestServices.GetRequiredService(); + var path = context.Request.Path; + var query = context.Request.QueryString; + var headers = context.Request.Headers.ToDictionary(h => h.Key, h => h.Value, StringComparer.OrdinalIgnoreCase); + var user = context.User; + var generation = cache.Generation; + + _ = Task.Run(async () => + { + try + { + await cache.WaitForBgRefreshSlot(descriptor.Route); + await using var scope = scopes.CreateAsyncScope(); + var replay = new DefaultHttpContext { RequestServices = scope.ServiceProvider, User = user }; + replay.Request.Method = HttpMethods.Get; + replay.Request.Path = path; + replay.Request.QueryString = query; + foreach (var (name, value) in headers) replay.Request.Headers[name] = value; + + var endpoint = (ICraftEndpoint)scope.ServiceProvider.GetRequiredService(descriptor.ImplementationType); + var request = new CraftRequest(replay, descriptor.Route, descriptor.Metadata); + var fresh = await ExecuteAsync(request, () => endpoint.HandleAsync(request, CancellationToken.None), handler, CancellationToken.None); + // An invalidation may have landed meanwhile, and writing back would undo it + if (fresh.ToCacheable() is { } cacheable) cache.SetIfSameGeneration(key, cacheable, generation); + } + catch (Exception ex) + { + logger.LogError(ex, "Background refresh failed: /API/{Endpoint}", descriptor.Route); + } + finally + { + cache.ReleaseBgRefreshSlot(descriptor.Route); + cache.FinishRefresh(key); + } + }); } private static async Task Handle( HttpContext context, NativeEndpointDescriptor descriptor, ICraftEndpointHandler? handler, + CacheService? cache, RequestCounter activeRequests, ILogger logger) { @@ -198,7 +297,24 @@ await context.Response.WriteAsync( } } - var result = await ExecuteAsync(request, endpoint, handler, ct); + // As on the PowerShell path: any write can change what any read returns + if (cache is not null) + { + cache.ApplyRequestedInvalidation(context); + if (!HttpMethods.IsGet(context.Request.Method)) cache.InvalidateScopedOrAll(context); + } + + string? bypass = null; + var useCache = cache is not null && IsCacheable(descriptor, context.Request.Method) + && cache.Policy.Allows(descriptor.Route, context.Request, out bypass); + if (bypass is not null) context.Response.Headers["X-Cache-Bypass"] = bypass; + + // The cache stands in for the endpoint only, so filters and the central handler still run on a hit + var result = await ExecuteAsync(request, + useCache + ? () => FromCacheAsync(context, request, endpoint, descriptor, handler, cache!, logger, ct) + : () => endpoint.HandleAsync(request, ct), + handler, ct); await result.WriteAsync(context, ct); } catch (OperationCanceledException) when (context.RequestAborted.IsCancellationRequested) diff --git a/Services/Hosting/Endpoints/PowerShellDispatchEndpoint.cs b/Services/Hosting/Endpoints/PowerShellDispatchEndpoint.cs index e221831..0bde5c2 100644 --- a/Services/Hosting/Endpoints/PowerShellDispatchEndpoint.cs +++ b/Services/Hosting/Endpoints/PowerShellDispatchEndpoint.cs @@ -61,7 +61,7 @@ public static WebApplication MapCraftPowerShellDispatch( logger.LogInformation("[HTTP] {Method} /API/{Endpoint} start concurrent={Concurrent}", context.Request.Method, endpoint, concurrent); - ApplyRequestedInvalidation(context, cache); + cache.ApplyRequestedInvalidation(context); // Two separate gates. IsCacheableRead is about the handler — does its name promise a // side-effect-free read. The policy is about this request — is caching it worthwhile @@ -102,7 +102,7 @@ await ServeCachedAsync(context, cache, psRunner, orchestrator, logger, // Writes can invalidate anything, so they clear the cache regardless of endpoint name. // Do this even on a client abort: a write may have partially applied before the // connection dropped, so dropping the possibly-stale cache is the safe choice. - if (context.Request.Method != "GET") InvalidateForWrite(context, cache); + if (context.Request.Method != "GET") cache.InvalidateScopedOrAll(context); // Client hung up mid-request (navigated away / hit Cancel). The pipeline was already // stopped and the worker reclaimed; there is no live connection to write to, and a @@ -154,34 +154,6 @@ await ServeCachedAsync(context, cache, psRunner, orchestrator, logger, return app; } - /// Honours an explicit ?{InvalidateParam}=true on the request. - private static void ApplyRequestedInvalidation(HttpContext context, CacheService cache) - { - if (!context.Request.Query.ContainsKey(cache.InvalidateParam)) return; - if (!string.Equals(context.Request.Query[cache.InvalidateParam], "true", StringComparison.OrdinalIgnoreCase)) - return; - - InvalidateScopedOrAll(context, cache); - } - - private static void InvalidateForWrite(HttpContext context, CacheService cache) => - InvalidateScopedOrAll(context, cache); - - /// - /// Invalidates just the caller's scope when one is present, otherwise everything — without a scope - /// there is no way to know which entries a write affected. - /// - private static void InvalidateScopedOrAll(HttpContext context, CacheService cache) - { - var scopeParam = cache.ScopeParam; - var scopeValue = !string.IsNullOrEmpty(scopeParam) - ? context.Request.Query[scopeParam].ToString() - : ""; - - if (!string.IsNullOrEmpty(scopeValue)) cache.InvalidateByScope(scopeValue); - else cache.InvalidateAll(); - } - private static async Task<(string? Key, CachedResponse? Response)> TryReadCacheAsync( HttpContext context, CacheService cache, string endpoint, ILogger logger) { diff --git a/Services/Hosting/GcHeapLimit.cs b/Services/Hosting/GcHeapLimit.cs index 89ca774..7827189 100644 --- a/Services/Hosting/GcHeapLimit.cs +++ b/Services/Hosting/GcHeapLimit.cs @@ -30,6 +30,12 @@ namespace Craft.Hosting; /// public static class GcHeapLimit { + /// + /// What the last successful did + /// ("set by SkuProfile", "disabled by CRAFT_GC_HEAP_LIMIT_MB"), or null when nothing changed the image default. + /// + public static string? Applied { get; private set; } + /// /// Applies a GC heap hard limit: the CRAFT_GC_HEAP_LIMIT_MB env override if set, otherwise /// from the matched profile. @@ -88,6 +94,7 @@ public static bool Apply(SkuProfile? profile, Action log, Action? ? $"GC heap hard limit disabled by {source} — deferring to container/physical memory" : $"GC heap hard limit set to {mb} MB by {source}"; log($"[System] {what} (TotalAvailableMemory {beforeMB} MB -> {afterMB} MB)"); + Applied = mb == 0 ? $"disabled by {source}" : $"set by {source}"; return true; } catch (Exception ex) diff --git a/Services/Hosting/RateLimitPartitionKey.cs b/Services/Hosting/RateLimitPartitionKey.cs index 95c4294..3fa609d 100644 --- a/Services/Hosting/RateLimitPartitionKey.cs +++ b/Services/Hosting/RateLimitPartitionKey.cs @@ -1,3 +1,5 @@ +using System.Net; + namespace Craft.Hosting; /// @@ -34,13 +36,20 @@ public static string Resolve(HttpContext context) var principal = context.Request.Headers["x-ms-client-principal-name"].ToString(); if (!string.IsNullOrEmpty(principal)) return principal; - var forwardedFor = context.Request.Headers["X-Forwarded-For"].ToString(); - if (!string.IsNullOrEmpty(forwardedFor)) - { - var firstHop = forwardedFor.Split(',')[0].Trim(); - if (!string.IsNullOrEmpty(firstHop)) return firstHop; - } + return ClientAddress(context) ?? "anonymous"; + } + + /// + /// The originating client address: the first X-Forwarded-For hop when it parses as an address + /// (any :port App Service appends dropped), else the socket peer; null when neither is known. + /// + public static string? ClientAddress(HttpContext context) + { + ArgumentNullException.ThrowIfNull(context); + + var firstHop = context.Request.Headers["X-Forwarded-For"].ToString().Split(',')[0].Trim(); + if (IPEndPoint.TryParse(firstHop, out var forwarded)) return forwarded.Address.ToString(); - return context.Connection.RemoteIpAddress?.ToString() ?? "anonymous"; + return context.Connection.RemoteIpAddress?.ToString(); } } diff --git a/Services/Hosting/StatsHistoryService.cs b/Services/Hosting/StatsHistoryService.cs index 738c511..2e7ae00 100644 --- a/Services/Hosting/StatsHistoryService.cs +++ b/Services/Hosting/StatsHistoryService.cs @@ -1,6 +1,7 @@ using System.Text.Json; using System.Text.Json.Serialization; using Craft.Configuration; +using Craft.Orchestration; using Craft.Services; namespace Craft.Hosting; @@ -8,7 +9,7 @@ namespace Craft.Hosting; /// /// Background service that periodically samples worker metrics and maintains /// a rolling history for trend visualization. Data is kept in memory and -/// flushed to disk for persistence across container restarts. +/// flushed to stats-history.jsonl in the log directory for persistence across container restarts. /// /// Default: 60-second sample interval, 7-day retention (~10,080 data points). /// @@ -17,6 +18,8 @@ public class StatsHistoryService : BackgroundService private readonly ILogger _logger; private readonly CraftSettings _settings; private readonly string _dataFilePath; + private readonly JobQueueStatusReader _queueReader; + private readonly string _queueTelemetryPath; // Circular buffer — newest at the end private readonly List _history = new(); @@ -44,13 +47,14 @@ public class StatsHistoryService : BackgroundService WriteIndented = false, }; - public StatsHistoryService(ILogger logger, CraftSettings settings) + public StatsHistoryService(ILogger logger, CraftSettings settings, JobQueueStatusReader queueReader) { _logger = logger; _settings = settings; - - var dataDir = Path.Combine(AppContext.BaseDirectory, "_data"); - _dataFilePath = Path.Combine(dataDir, "stats-history.jsonl"); + _queueReader = queueReader; + // Beside egress-ledger.json: survives restarts, and external monitoring reads files there without storage access. + _queueTelemetryPath = Path.Combine(settings.FileLogging.ResolvedDirectory, "queue-telemetry.json"); + _dataFilePath = Path.Combine(settings.FileLogging.ResolvedDirectory, "stats-history.jsonl"); } public int SampleIntervalSeconds => _settings.StatsHistory.SampleIntervalSeconds; @@ -75,8 +79,10 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) { try { - var point = CollectSample(); + var snapshot = WorkerMetricsBridge.GetSnapshot(); + var point = CollectSample(snapshot); AppendPointToDisk(point); + await WriteQueueTelemetryAsync(snapshot, stoppingToken); _ticksSinceCompact++; if (_ticksSinceCompact >= CompactEveryNTicks) @@ -102,9 +108,8 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) } /// Take a metrics snapshot and record a data point with delta computation. - private StatsDataPoint CollectSample() + private StatsDataPoint CollectSample(WorkerMetricsSnapshot snapshot) { - var snapshot = WorkerMetricsBridge.GetSnapshot(); var now = DateTime.UtcNow; var httpPool = snapshot.HttpPool; @@ -248,6 +253,36 @@ public int GetCount() } } + /// + /// Rewrite the read-only queue telemetry file: the current pool, limiter, job and memory metrics plus every + /// run's queued/running/completed/failed counts. Temp file + atomic move, so a reader never sees a torn file. + /// + internal async Task WriteQueueTelemetryAsync(WorkerMetricsSnapshot snapshot, CancellationToken ct) + { + try + { + var telemetry = new + { + Version = 1, + UpdatedUtc = snapshot.TimestampUtc, + Metrics = snapshot, + Runs = await _queueReader.GetRunSummariesAsync(ct), + }; + Directory.CreateDirectory(Path.GetDirectoryName(_queueTelemetryPath)!); + var tmp = _queueTelemetryPath + ".tmp"; + await File.WriteAllTextAsync(tmp, JsonSerializer.Serialize(telemetry, s_jsonOptions), ct); + File.Move(tmp, _queueTelemetryPath, overwrite: true); + } + catch (Exception ex) when (!ct.IsCancellationRequested) + { + _logger.LogWarning(ex, "[StatsHistory] Failed to write queue telemetry"); + } + catch (OperationCanceledException) + { + // Shutting down. + } + } + // ── Disk persistence (JSONL append-only + periodic compaction) ── /// Append a single sample as one JSON line. O(1) write, no full-file rewrite. diff --git a/Services/Program.cs b/Services/Program.cs index d1f73a2..90c5940 100644 --- a/Services/Program.cs +++ b/Services/Program.cs @@ -102,6 +102,10 @@ var repo = app.Services.GetRequiredService(); var pool = app.Services.GetRequiredService(); var logger = app.Services.GetRequiredService>(); +// The limit is applied before the file logger exists, so its own line only reaches the console: record what took effect +_ = app.Services.GetRequiredService(); +logger.LogInformation("[System] GC heap limit: {LimitMB} MB available to the GC ({Source})", + GC.GetGCMemoryInfo().TotalAvailableMemoryBytes / (1024 * 1024), GcHeapLimit.Applied ?? "image default"); try { using var keyTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(15)); @@ -415,6 +419,12 @@ static bool IsEventsPath(HttpContext c) => // the feature switch). See Services/Hosting/Endpoints/PrmEndpoint.cs. app.MapCraftPrmEndpoint(CraftSettings, logger); +// Native endpoints that will be mapped, collisions with PowerShell routes resolved: decided here because the +// auth routes below need to know whether one of them answers /api/me. +var mappableNative = nativeCatalog.Endpoints.Count > 0 + ? NativeEndpointRegistry.ResolveCollisions(nativeCatalog.Endpoints, endpoints.Keys, CraftSettings.Endpoints.OnCollision, logger) + : []; + // ── HTTP-role endpoints + middleware ────────────────────────────────────────────────────────────── // A node without the Http role maps NONE of these, so /api and auth paths fall through to static serving // (a Frontend node can expose them from its own static dir) and finally to MapFallback (404 for /api|/.auth). @@ -429,7 +439,8 @@ static bool IsEventsPath(HttpContext c) => // Normalise the EasyAuth principal into the SWA shape the hosted PS app expects, then map the two // auth-adjacent routes. See Services/Hosting/CraftAuthMiddleware.cs and Endpoints/AuthEndpoints.cs. app.UseCraftAuth(CraftSettings, authService, logger); - app.MapCraftAuthEndpoints(CraftSettings, logger); + app.MapCraftAuthEndpoints(CraftSettings, logger, mappableNative.Any(e => + string.Equals(e.Route, AuthEndpoints.MeRoute(CraftSettings), StringComparison.OrdinalIgnoreCase))); } // end HTTP-role block (auth middleware). Bridges below run for any PS role; the // setup/jobs/PS-dispatch routes are re-gated in a second `if (capHttp)` block further down. @@ -490,12 +501,8 @@ static bool IsEventsPath(HttpContext c) => // Native C# endpoints. Mapped before the PowerShell dispatcher, though ASP.NET route precedence // would put a literal segment ahead of /API/{endpoint} regardless — which is what lets an app // migrate one endpoint at a time with the PowerShell function still loaded as the rollback. - if (nativeCatalog.Endpoints.Count > 0) - { - var mappable = NativeEndpointRegistry.ResolveCollisions( - nativeCatalog.Endpoints, endpoints.Keys, CraftSettings.Endpoints.OnCollision, logger); - app.MapCraftNativeEndpoints(mappable, activeRequests, logger, CraftSettings.Endpoints); - } + if (mappableNative.Count > 0) + app.MapCraftNativeEndpoints(mappableNative, activeRequests, logger, CraftSettings.Endpoints, AuthEndpoints.MeRoute(CraftSettings)); // Dispatch /API/{endpoint} to the discovered PowerShell function. Owns the response cache, // stale-while-revalidate, and post-response trigger handling. diff --git a/appsettings.example.jsonc b/appsettings.example.jsonc index a3ef7fa..f0c53d5 100644 --- a/appsettings.example.jsonc +++ b/appsettings.example.jsonc @@ -108,10 +108,11 @@ // // so you can run accounting-only first and watch real numbers before rejecting anything. // // The counter lives in memory (no per-request IO) and is flushed to egress-ledger.json in the same // // directory as the log files every FlushSeconds and on shutdown, so a restart doesn't reset the day. - // // The file holds per-client daily totals + the cap-hit state (the source of truth for the cap). + // // The file holds per-client daily totals (per endpoint and per caller IP) + the cap-hit state (the + // // source of truth for the cap), plus the finished days within RetentionDays under "days". // // In addition, per-client + instance rows are mirrored to the TableName table — 15-minute buckets // // (the time-series) and a daily audit row (cap in force, when the cap was first hit, how many - // // requests were shed) — so the product can chart usage and show cap hits. The table is the same + // // requests were shed, top endpoints and caller IPs) — so the product can chart usage and show cap hits. The table is the same // // storage account the hosted app reads with Get-CIPPTable, retained RetentionDays and purged // // periodically. A restart reseeds the current bucket from the table so its row doesn't regress. // // Env overrides: CRAFT_API_EGRESS_LIMIT_BYTES, CRAFT_API_EGRESS_FLUSH_SECONDS, CRAFT_API_EGRESS_TABLE @@ -122,7 +123,7 @@ // "FlushSeconds": 60, // how often the counter is written to disk + mirrored to the table (also on shutdown) // "TableName": "CraftEgressAccounting", // table for the bucketed history + daily audit ("" = file only) // "BucketMinutes": 15, // width of each history bucket (→ 96 buckets/day) - // "RetentionDays": 7 // days of table rows kept before the periodic purge + // "RetentionDays": 7 // days of table rows (and of file history, today included) kept // } // }, diff --git a/build/Dockerfile b/build/Dockerfile index e801793..8e5d35d 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -177,7 +177,8 @@ ENV DOTNET_CLI_TELEMETRY_OPTOUT=1 # ── .NET GC tuning ── ENV DOTNET_gcServer=0 ENV DOTNET_GCDynamicAdaptationMode=1 -ENV DOTNET_GCHeapHardLimit=0x95E00000 +# Downstream images should set their own GC heap hard limit if needed. +#ENV DOTNET_GCHeapHardLimit=0x95E00000 ENV DOTNET_GCConserveMemory=7 # App port only. Runs as the image's default non-root user (APP_UID=1654). diff --git a/tests/Craft.Tests/ApiEgressWireCounterMiddlewareTests.cs b/tests/Craft.Tests/ApiEgressWireCounterMiddlewareTests.cs index 6806c57..8dd6279 100644 --- a/tests/Craft.Tests/ApiEgressWireCounterMiddlewareTests.cs +++ b/tests/Craft.Tests/ApiEgressWireCounterMiddlewareTests.cs @@ -247,6 +247,29 @@ public async Task TagEndpoint_SuffixesHandlerHeader_StripsIt_AndCountsCacheHits( Assert.Equal(1L, stats.CacheHits); } + [Fact] + public async Task RecordsTheResponseContentEncoding() + { + var (ledger, store) = TableLedger(); + var mw = WireCounter(async ctx => + { + Flag(ctx); + ApiEgressWireCounterMiddleware.TagEndpoint(ctx, "ListUsers"); + ctx.Response.Headers.ContentEncoding = "br"; + await ctx.Response.Body.WriteAsync(new byte[40]); + }, ledger); + await mw.InvokeAsync(Context()); + await WireCounter(async ctx => + { + Flag(ctx); + ApiEgressWireCounterMiddleware.TagEndpoint(ctx, "ListUsers"); + await ctx.Response.Body.WriteAsync(new byte[100]); + }, ledger).InvokeAsync(Context()); + + var stats = (await DailyEndpoints(ledger, store, App))["ListUsers"]; + Assert.Equal((2L, 1L, 40L, 0L, 0L), (stats.Requests, stats.BrRequests, stats.BrBytes, stats.GzipRequests, stats.GzipBytes)); + } + [Fact] public async Task InteractiveCaller_IsRecordedForReporting_ButNeverBilled() { diff --git a/tests/Craft.Tests/EgressLedgerTableTests.cs b/tests/Craft.Tests/EgressLedgerTableTests.cs index 91e468d..f46242a 100644 --- a/tests/Craft.Tests/EgressLedgerTableTests.cs +++ b/tests/Craft.Tests/EgressLedgerTableTests.cs @@ -149,7 +149,7 @@ public async Task Endpoints_BrokenOutOnDailyRows_AndInstanceBucket_NotClientBuck await ledger.SyncToTableAsync(CancellationToken.None); var users = Endpoints(Row(store, App, EgressTableSchema.DailyPrefix))["ListUsers"]; - Assert.Equal([400L, 2, 300, 1, 0, 0], users.ToArray()); // Bytes, Requests, MaxBytes, CacheHits, Errors, Shed + Assert.Equal([400L, 2, 300, 1, 0, 0, 0, 0, 0, 0], users.ToArray()); // Bytes, Requests, MaxBytes, CacheHits, Errors, Shed, Br/Gzip requests+bytes var instanceDay = Endpoints(Row(store, EgressTableSchema.SystemPartition, EgressTableSchema.DailyPrefix)); Assert.Equal(1L, instanceDay["ListUsers"].Shed); @@ -162,6 +162,30 @@ public async Task Endpoints_BrokenOutOnDailyRows_AndInstanceBucket_NotClientBuck Assert.Null(Row(store, App, EgressTableSchema.BucketPrefix)[EgressTableSchema.PropEndpoints]); } + [Fact] + public async Task Ips_OnDailyRows_PerClient_Interactive_AndInstanceMerge_NotBuckets() + { + var store = new FakeTableStore(); + var clock = new DateTime(2026, 9, 13, 14, 22, 0, DateTimeKind.Utc); + var ledger = New(store, cap: 1_000_000, () => clock); + + ledger.Record(100, App, "ListUsers", ip: "203.0.113.7"); + ledger.Record(50, App2, "ListUsers", ip: "203.0.113.7"); + ledger.Record(25, App2, "ListUsers", ip: "198.51.100.1"); + ledger.RecordInteractive(10, "ListLogs", ip: "192.0.2.9"); + await ledger.SyncToTableAsync(CancellationToken.None); + + Dictionary Ips(StoreRow row) => EgressTableSchema.ParseEndpoints(row.GetString(EgressTableSchema.PropIps)); + Assert.Equal(100L, Ips(Row(store, App, EgressTableSchema.DailyPrefix))["203.0.113.7"].Bytes); + Assert.Equal(25L, Ips(Row(store, App2, EgressTableSchema.DailyPrefix))["198.51.100.1"].Bytes); + var instance = Ips(Row(store, EgressTableSchema.SystemPartition, EgressTableSchema.DailyPrefix)); + Assert.Equal([150L, 2, 100, 0, 0, 0, 0, 0, 0, 0], instance["203.0.113.7"].ToArray()); + Assert.DoesNotContain("192.0.2.9", instance.Keys); // user traffic stays in its own partition + Assert.Equal(10L, Ips(Row(store, EgressTableSchema.InteractivePartition, EgressTableSchema.DailyPrefix))["192.0.2.9"].Bytes); + Assert.All(store.All(Table).Where(r => r.RowKey.StartsWith(EgressTableSchema.BucketPrefix, StringComparison.Ordinal)), + r => Assert.Null(r[EgressTableSchema.PropIps])); + } + [Fact] public async Task Interactive_ReportedInOwnPartition_NeverBilled() { diff --git a/tests/Craft.Tests/EgressLedgerTests.cs b/tests/Craft.Tests/EgressLedgerTests.cs index fe15a20..b72b370 100644 --- a/tests/Craft.Tests/EgressLedgerTests.cs +++ b/tests/Craft.Tests/EgressLedgerTests.cs @@ -249,6 +249,98 @@ public void Load_NegativeStoredBytes_Ignored() Assert.Equal(0L, ledger.CurrentBytes); // a corrupt/negative total can't unlock a false budget } + // ── Caller addresses + finished-day history (v4) ─────────────────────────────────────────────── + + [Fact] + public void Record_TracksCallerAddresses_PerClient_InFile() + { + var ledger = New(cap: 0); + ledger.Record(100, App, "ListUsers", ip: "203.0.113.7", encoding: "br"); + ledger.Record(50, App, "ListUsers", ip: "203.0.113.7", encoding: "gzip"); + ledger.Record(25, App, "ListUsers", ip: "198.51.100.1"); + ledger.RecordShed(App, "ListUsers", ip: "203.0.113.7"); + ledger.RecordInteractive(10, "ListLogs", ip: "192.0.2.9"); + ledger.Flush(); + + var root = ReadFile().RootElement; + Assert.Equal(4, root.GetProperty("version").GetInt32()); + var ips = root.GetProperty("clients").GetProperty(App).GetProperty("ips"); + // [Bytes, Requests, MaxBytes, CacheHits, Errors, Shed, BrRequests, BrBytes, GzipRequests, GzipBytes] + Assert.Equal([150L, 2L, 100L, 0L, 0L, 1L, 1L, 100L, 1L, 50L], ips.GetProperty("203.0.113.7").EnumerateArray().Select(v => v.GetInt64())); + Assert.Equal(25L, ips.GetProperty("198.51.100.1")[0].GetInt64()); + Assert.Equal(10L, root.GetProperty("interactive").GetProperty("ips").GetProperty("192.0.2.9")[0].GetInt64()); + } + + [Fact] + public void CallerAddresses_RestoredAcrossRestart_SameDay() + { + var clock = new DateTime(2026, 9, 11, 6, 0, 0, DateTimeKind.Utc); + var first = New(cap: 0, () => clock); + first.Record(100, App, ip: "203.0.113.7"); + first.Flush(); + + var restarted = New(cap: 0, () => clock); + restarted.Record(1, App, ip: "203.0.113.7"); + restarted.Flush(); + Assert.Equal(101L, ReadFile().RootElement.GetProperty("clients").GetProperty(App).GetProperty("ips").GetProperty("203.0.113.7")[0].GetInt64()); + } + + [Fact] + public void Rollover_KeepsFinishedDays_NewestFirst_WithinRetention() + { + var clock = new DateTime(2026, 9, 1, 12, 0, 0, DateTimeKind.Utc); + var ledger = New(cap: 0, () => clock); + for (var day = 1; day <= 10; day++) + { + ledger.Record(day, App, ip: "203.0.113.7"); + ledger.Flush(); + if (day < 10) clock = clock.AddDays(1); + } + + var root = ReadFile().RootElement; + Assert.Equal("2026-09-10", root.GetProperty("dateUtc").GetString()); + Assert.Equal(10L, root.GetProperty("bytes").GetInt64()); + var days = root.GetProperty("days").EnumerateArray().ToList(); + // 7-day retention = today + six finished days. + Assert.Equal(["2026-09-09", "2026-09-08", "2026-09-07", "2026-09-06", "2026-09-05", "2026-09-04"], + days.Select(d => d.GetProperty("dateUtc").GetString())); + Assert.Equal(9L, days[0].GetProperty("bytes").GetInt64()); + Assert.Equal(9L, days[0].GetProperty("clients").GetProperty(App).GetProperty("ips").GetProperty("203.0.113.7")[0].GetInt64()); + Assert.All(days, d => Assert.False(d.TryGetProperty("days", out var nested) && nested.ValueKind != JsonValueKind.Null)); + } + + [Fact] + public void Reload_OnNewUtcDay_MovesStoredDayIntoHistory() + { + var day1 = new DateTime(2026, 9, 11, 6, 0, 0, DateTimeKind.Utc); + var first = New(cap: 0, () => day1); + first.Record(1234, App); + first.Flush(); + + var restarted = New(cap: 0, () => day1.AddDays(2)); // down over a whole day + Assert.Equal(0L, restarted.CurrentBytes); + restarted.Flush(); // the reload alone makes the file worth rewriting + + var root = ReadFile().RootElement; + Assert.Equal("2026-09-13", root.GetProperty("dateUtc").GetString()); + var stored = Assert.Single(root.GetProperty("days").EnumerateArray()); + Assert.Equal("2026-09-11", stored.GetProperty("dateUtc").GetString()); + Assert.Equal(1234L, stored.GetProperty("bytes").GetInt64()); + } + + [Fact] + public void Reload_DropsHistoryPastRetention() + { + var day1 = new DateTime(2026, 9, 1, 6, 0, 0, DateTimeKind.Utc); + var first = New(cap: 0, () => day1); + first.Record(1, App); + first.Flush(); + + var restarted = New(cap: 0, () => day1.AddDays(7)); // day1 is now outside today + six days + restarted.Flush(); + Assert.Empty(ReadFile().RootElement.GetProperty("days").EnumerateArray()); + } + // ── Thread-safety + hosted-service lifecycle ──────────────────────────────────────────────────── [Fact] diff --git a/tests/Craft.Tests/GcHeapLimitTests.cs b/tests/Craft.Tests/GcHeapLimitTests.cs index e4f2d53..a0806cf 100644 --- a/tests/Craft.Tests/GcHeapLimitTests.cs +++ b/tests/Craft.Tests/GcHeapLimitTests.cs @@ -53,6 +53,7 @@ public void ProfileZero_DisablesTheCap() Assert.True(applied); Assert.Equal(0UL, requested); Assert.Contains(logs, l => l.Contains("disabled by SkuProfile", StringComparison.Ordinal)); + Assert.Equal("disabled by SkuProfile", GcHeapLimit.Applied); } [Fact] @@ -67,6 +68,7 @@ public void ConfiguredLimit_IsAppliedInBytes_AndLogged() Assert.True(applied); Assert.Equal(5120UL * 1024 * 1024, requested); Assert.Contains(logs, l => l.Contains("GC heap hard limit set to 5120 MB", StringComparison.Ordinal)); + Assert.Equal("set by SkuProfile", GcHeapLimit.Applied); } [Fact] diff --git a/tests/Craft.Tests/HostConfigurationTests.cs b/tests/Craft.Tests/HostConfigurationTests.cs index 7f480c8..c3b1d09 100644 --- a/tests/Craft.Tests/HostConfigurationTests.cs +++ b/tests/Craft.Tests/HostConfigurationTests.cs @@ -103,6 +103,22 @@ public void NothingIdentifying_StillProducesAStableKey() { Assert.Equal("anonymous", RateLimitPartitionKey.Resolve(new DefaultHttpContext())); } + + [Theory] + [InlineData("203.0.113.7:54321", "203.0.113.7")] + [InlineData("[2001:db8::1]:443, 10.0.0.9", "2001:db8::1")] + [InlineData("2001:db8::1", "2001:db8::1")] + [InlineData("not-an-address, 10.0.0.9", "10.0.0.1")] + public void ClientAddress_DropsThePort_AndIgnoresAHopThatIsNotAnAddress(string forwardedFor, string expected) + { + Assert.Equal(expected, RateLimitPartitionKey.ClientAddress(Request(forwardedFor: forwardedFor, remoteIp: "10.0.0.1"))); + } + + [Fact] + public void ClientAddress_NullWhenNothingIsKnown() + { + Assert.Null(RateLimitPartitionKey.ClientAddress(new DefaultHttpContext())); + } } /// diff --git a/tests/Craft.Tests/NativeEndpointCacheTests.cs b/tests/Craft.Tests/NativeEndpointCacheTests.cs new file mode 100644 index 0000000..bcfa0b9 --- /dev/null +++ b/tests/Craft.Tests/NativeEndpointCacheTests.cs @@ -0,0 +1,133 @@ +using System.Net; +using System.Text; +using Craft.Caching; +using Craft.Configuration; +using Craft.Endpoints; +using Craft.Hosting; +using Craft.Hosting.Endpoints; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Craft.Tests; + +/// +/// Native GETs share the PowerShell path's response cache: per caller, stale-while-revalidate, cleared by writes, +/// and never in front of the central handler. +/// +public class NativeEndpointCacheTests +{ + internal abstract class Counting : ICraftEndpoint + { + public int Calls; + public ValueTask HandleAsync(CraftRequest r, CancellationToken ct) => + new(CraftResult.Json(new { Call = Interlocked.Increment(ref Calls) })); + } + + [CraftEndpoint("ListCacheThings", Methods = ["GET"])] + internal sealed class ListThings : Counting; + + [CraftEndpoint("GetCacheThing", Methods = ["GET"], Caching = EndpointCaching.Cached)] + internal sealed class GetThing : Counting; + + [CraftEndpoint("GetCacheLive", Methods = ["GET"])] + internal sealed class GetLive : Counting; + + [CraftEndpoint("ListCacheLive", Methods = ["GET"], Caching = EndpointCaching.Uncached)] + internal sealed class ListLive : Counting; + + [CraftEndpoint("ExecCacheWrite", Methods = ["POST"])] + internal sealed class ExecWrite : Counting; + + /// Refuses any request carrying x-deny, as an application's authorization would. + internal sealed class DenyHandler : ICraftEndpointHandler + { + public ValueTask HandleAsync(CraftRequest request, CraftEndpointNext invokeEndpoint, CancellationToken ct) => + request.Http.Request.Headers.ContainsKey("x-deny") ? new(CraftResult.Problem(403, "denied")) : invokeEndpoint(); + } + + private static readonly Type[] s_types = [typeof(ListThings), typeof(GetThing), typeof(GetLive), typeof(ListLive), typeof(ExecWrite)]; + + private static async Task<(WebApplication App, HttpClient Client)> StartAsync() + { + var builder = WebApplication.CreateBuilder(); + builder.WebHost.UseTestServer(); + builder.Logging.ClearProviders(); + foreach (var type in s_types) builder.Services.AddSingleton(type); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(new CacheService(NullLogger.Instance, new CraftSettings(), enabled: true)); + var app = builder.Build(); + var descriptors = s_types.Select(t => new NativeEndpointDescriptor( + ((CraftEndpointAttribute)Attribute.GetCustomAttribute(t, typeof(CraftEndpointAttribute))!).Route, t, + (CraftEndpointAttribute)Attribute.GetCustomAttribute(t, typeof(CraftEndpointAttribute))!)).ToList(); + app.MapCraftNativeEndpoints(descriptors, new RequestCounter(), NullLogger.Instance); + await app.StartAsync(); + var client = app.GetTestClient(); + // A distinct caller per test, so tests sharing the on-disk cache never share an entry + client.DefaultRequestHeaders.Add("x-ms-client-principal", + Convert.ToBase64String(Encoding.UTF8.GetBytes($$"""{"userId":"{{Guid.NewGuid()}}"}"""))); + return (app, client); + } + + private static async Task<(string Cache, string Body)> GetAsync(HttpClient client, string path, bool deny = false) + { + using var message = new HttpRequestMessage(HttpMethod.Get, path); + if (deny) message.Headers.Add("x-deny", "1"); + using var response = await client.SendAsync(message); + var cache = response.Headers.TryGetValues("X-Cache", out var v) ? v.Single() : ""; + return (response.StatusCode == HttpStatusCode.OK ? cache : ((int)response.StatusCode).ToString(System.Globalization.CultureInfo.InvariantCulture), await response.Content.ReadAsStringAsync()); + } + + [Fact] + public async Task AListGetIsServedFromTheCacheThenRefreshedInTheBackground() + { + var (app, client) = await StartAsync(); + await using var _ = app; + + Assert.Equal(("MISS", """{"call":1}"""), await GetAsync(client, "/API/ListCacheThings")); + Assert.Equal(("HIT", """{"call":1}"""), await GetAsync(client, "/API/ListCacheThings")); + + // The hit started a refresh that replaces the entry with a later call's result + var body = ""; + for (var i = 0; i < 100 && (body = (await GetAsync(client, "/API/ListCacheThings")).Body) == """{"call":1}"""; i++) + await Task.Delay(20); + Assert.NotEqual("""{"call":1}""", body); + } + + [Fact] + public async Task TheCentralHandlerStillDecidesOnAHit() + { + var (app, client) = await StartAsync(); + await using var _ = app; + + Assert.Equal("MISS", (await GetAsync(client, "/API/ListCacheThings")).Cache); + Assert.Equal("403", (await GetAsync(client, "/API/ListCacheThings", deny: true)).Cache); + } + + [Fact] + public async Task AWriteClearsTheCache() + { + var (app, client) = await StartAsync(); + await using var _ = app; + + await GetAsync(client, "/API/ListCacheThings"); + (await client.PostAsync("/API/ExecCacheWrite", null)).EnsureSuccessStatusCode(); + Assert.Equal("MISS", (await GetAsync(client, "/API/ListCacheThings")).Cache); + } + + [Fact] + public async Task CachingFollowsTheEndpointsDeclaration() + { + var (app, client) = await StartAsync(); + await using var _ = app; + + foreach (var (path, cached) in new[] { ("/API/GetCacheThing", true), ("/API/GetCacheLive", false), ("/API/ListCacheLive", false) }) + { + await GetAsync(client, path); + var second = await GetAsync(client, path); + Assert.Equal(cached ? ("HIT", """{"call":1}""") : ("", """{"call":2}"""), second); + } + } +} diff --git a/tests/Craft.Tests/NativeMeEndpointTests.cs b/tests/Craft.Tests/NativeMeEndpointTests.cs new file mode 100644 index 0000000..e7f845c --- /dev/null +++ b/tests/Craft.Tests/NativeMeEndpointTests.cs @@ -0,0 +1,83 @@ +using Craft.Configuration; +using Craft.Endpoints; +using Craft.Hosting; +using Craft.Hosting.Endpoints; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Craft.Tests; + +[CraftEndpoint("WhoAmI", Methods = ["GET"])] +internal sealed class WhoAmIEndpoint : ICraftEndpoint +{ + public ValueTask HandleAsync(CraftRequest r, CancellationToken ct) => + new(CraftResult.RawJson("""{"clientPrincipal":{"userDetails":"native"},"permissions":["*"]}""")); +} + +/// +/// An application with no PowerShell workers can still answer the SPA's /api/me: a native endpoint on +/// the configured me route serves it, and the PowerShell route is not mapped at all. +/// +public class NativeMeEndpointTests +{ + [Fact] + public async Task ANativeEndpointOnTheMeRouteAnswersApiMe() + { + var settings = new CraftSettings { Auth = { MeEndpointFunction = "WhoAmI" } }; + var builder = WebApplication.CreateBuilder(); + builder.WebHost.UseTestServer(); + builder.Logging.ClearProviders(); + builder.Services.AddSingleton(); + await using var app = builder.Build(); + var descriptor = new NativeEndpointDescriptor("WhoAmI", typeof(WhoAmIEndpoint), + (CraftEndpointAttribute)Attribute.GetCustomAttribute(typeof(WhoAmIEndpoint), typeof(CraftEndpointAttribute))!); + + // No PowerShellRunnerService is registered: mapping must not need one when /api/me is native + app.MapCraftAuthEndpoints(settings, NullLogger.Instance, meIsNative: true); + app.MapCraftNativeEndpoints([descriptor], new RequestCounter(), NullLogger.Instance, meRoute: AuthEndpoints.MeRoute(settings)); + await app.StartAsync(); + using var client = app.GetTestClient(); + + var me = await client.GetStringAsync("/api/me"); + Assert.Contains("\"userDetails\":\"native\"", me); + Assert.Equal(me, await client.GetStringAsync("/API/WhoAmI")); + Assert.Contains("clientPrincipal", await client.GetStringAsync("/.auth/me")); + } + + [CraftEndpoint("me", Methods = ["GET"])] + internal sealed class MeEndpoint : ICraftEndpoint + { + public ValueTask HandleAsync(CraftRequest r, CancellationToken ct) => + new(CraftResult.RawJson("""{"clientPrincipal":{"userDetails":"me"},"permissions":[]}""")); + } + + [Fact] + public async Task ANativeEndpointNamedMeIsMappedOnce() + { + var builder = WebApplication.CreateBuilder(); + builder.WebHost.UseTestServer(); + builder.Logging.ClearProviders(); + builder.Services.AddSingleton(); + await using var app = builder.Build(); + var descriptor = new NativeEndpointDescriptor("me", typeof(MeEndpoint), + (CraftEndpointAttribute)Attribute.GetCustomAttribute(typeof(MeEndpoint), typeof(CraftEndpointAttribute))!); + + app.MapCraftAuthEndpoints(new CraftSettings(), NullLogger.Instance, meIsNative: true); + app.MapCraftNativeEndpoints([descriptor], new RequestCounter(), NullLogger.Instance, meRoute: "me"); + await app.StartAsync(); + using var client = app.GetTestClient(); + + // Mapped twice, this would throw an ambiguous match + Assert.Contains("\"userDetails\":\"me\"", await client.GetStringAsync("/api/me")); + } + + [Fact] + public void TheMeRouteDefaultsToMe() + { + Assert.Equal("me", AuthEndpoints.MeRoute(new CraftSettings())); + Assert.Equal("WhoAmI", AuthEndpoints.MeRoute(new CraftSettings { Auth = { MeEndpointFunction = "WhoAmI" } })); + } +} diff --git a/tests/Craft.Tests/QueueTelemetryFileTests.cs b/tests/Craft.Tests/QueueTelemetryFileTests.cs new file mode 100644 index 0000000..45450d7 --- /dev/null +++ b/tests/Craft.Tests/QueueTelemetryFileTests.cs @@ -0,0 +1,61 @@ +using System.Text.Json; +using Craft.Configuration; +using Craft.Hosting; +using Craft.Orchestration; +using Craft.PowerShellHost; +using Craft.Services; +using Craft.Storage; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Craft.Tests; + +/// +/// queue-telemetry.json is read by external monitoring straight off the file system, so it must sit beside the +/// egress ledger and carry the run counts from storage, not just what this process has claimed. +/// +public class QueueTelemetryFileTests : IDisposable +{ + private readonly string _dir = Path.Combine(Path.GetTempPath(), "craft-queue-telemetry-" + Guid.NewGuid().ToString("N")[..8]); + + public void Dispose() + { + try { Directory.Delete(_dir, recursive: true); } catch { /* temp dir */ } + GC.SuppressFinalize(this); + } + + [Fact] + public async Task Write_PutsMetricsAndDurableRunCountsBesideTheLedger() + { + var settings = new CraftSettings(); + settings.FileLogging.Directory = _dir; + var config = new ConfigurationBuilder().AddInMemoryCollection([]).Build(); + var repo = new ScriptRepository(NullLogger.Instance, settings); + var pool = new PowerShellWorkerPool(repo, NullLogger.Instance, config, settings); + var limiter = new BackgroundTaskLimiter(NullLogger.Instance, config, settings, pool); + var jobs = new JobManager(NullLogger.Instance, settings, limiter); + var store = new WorkStore(NullLogger.Instance, settings, new MemoryTableStore()); + var started = new DateTime(2026, 10, 22, 3, 0, 0, DateTimeKind.Utc); + await store.CreateRunAsync(new RunHeader + { + RunKey = WorkStore.RunKeyFor("Standards-AllTenants", started), + Name = "Standards-AllTenants", + StartedUtc = started, + TaskScriptName = "Invoke-CraftTask", + }, Enumerable.Range(0, 3).Select(i => new WorkStore.NewTask($"t{i}", [])).ToList()); + var service = new StatsHistoryService(NullLogger.Instance, settings, + new JobQueueStatusReader(NullLogger.Instance, jobs, store)); + + await service.WriteQueueTelemetryAsync(new WorkerMetricsSnapshot { TimestampUtc = started, Jobs = new JobMetrics { QueuedDurable = 3 } }, CancellationToken.None); + + var path = Path.Combine(_dir, "queue-telemetry.json"); + Assert.False(File.Exists(path + ".tmp")); + var root = JsonDocument.Parse(File.ReadAllText(path)).RootElement; + Assert.Equal(1, root.GetProperty("version").GetInt32()); + Assert.Equal(3, root.GetProperty("metrics").GetProperty("jobs").GetProperty("queuedDurable").GetInt32()); + var run = Assert.Single(root.GetProperty("runs").EnumerateArray()); + Assert.Equal("Standards-AllTenants", run.GetProperty("name").GetString()); + Assert.Equal(3, run.GetProperty("total").GetInt32()); + Assert.Equal(3, run.GetProperty("queued").GetInt32()); + } +}