perf: Set readytorun for pre-compiled PS SDK and retry building workers in next cycle

Craft.csproj

@@ -4,6 +4,7 @@
   <PropertyGroup>
     <TargetFramework>net8.0</TargetFramework>
     <RootNamespace>Craft</RootNamespace>
+    <PublishReadyToRun>true</PublishReadyToRun>
   </PropertyGroup>
 
   <ItemGroup>

Services/Configuration/SchedulerSettings.cs

@@ -29,4 +29,12 @@ public class SchedulerSettings
     /// Examples: "America/New_York", "Europe/London", "Eastern Standard Time"
     /// </summary>
     public string Timezone { get; set; } = "";
+
+    /// <summary>
+    /// Table that records each timer's state so the hosted app can show it: one row per timer
+    /// (PartitionKey "Timer", RowKey = task Id) with Command, Cron, LastFired, NextFire (jitter included),
+    /// JitterMinutes, JitterSeconds and Timezone. Rewritten at startup, after each firing and on a timezone
+    /// change; rows for timers no longer in the file are removed at startup. Empty (default) = not recorded.
+    /// </summary>
+    public string StateTable { get; set; } = "";
 }

Services/Orchestration/SchedulerService.cs

@@ -28,9 +28,12 @@ public class SchedulerService : BackgroundService
     private readonly StorageHealthMonitor _storageHealth;
     private readonly NativeScheduledTasks _nativeTasks;
     private readonly IServiceProvider _services;
+    private readonly ICraftTableStore? _store;
     private List<SchedulerTask> _tasks = [];
     private readonly Dictionary<string, DateTimeOffset> _lastRun = new();
+    private readonly Dictionary<string, DateTimeOffset> _lastFired = new();
     private TimeZoneInfo _configuredTz = TimeZoneInfo.Utc;
+    private const string StatePartition = "Timer";
 
     /// <summary>Expose loaded tasks for the API layer.</summary>
     public IReadOnlyList<SchedulerTask> Tasks => _tasks;
@@ -45,8 +48,10 @@ public SchedulerService(
         PowerShellWorkerPool pool,
         StorageHealthMonitor storageHealth,
         NativeScheduledTasks nativeTasks,
-        IServiceProvider services)
+        IServiceProvider services,
+        ICraftTableStore? store = null)
     {
+        _store = store;
         _logger = logger;
         _psRunner = psRunner;
         _limiter = limiter;
@@ -97,6 +102,12 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
             _orchestrator.MarkRecoveryDone();
         }
 
+        try { await LoadStateAsync(stoppingToken); }
+        catch (Exception ex) when (ex is not OperationCanceledException)
+        {
+            _logger.LogWarning(ex, "[Scheduler] Could not load timer state from {Table}", _settings.Scheduler.StateTable);
+        }
+
         // Retention sweeps for the rest of the process lifetime; recovery ran the first one. Fire and
         // forget is deliberate: the loop handles its own failures and ends with the stopping token.
         _ = _orchestrator.RunRetentionLoopAsync(stoppingToken);
@@ -190,6 +201,8 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
                 }
             }
 
+            if (due.Count > 0) _ = WriteStateAsync(due, stoppingToken);
+
             // Check at configured interval
             await Task.Delay(TimeSpan.FromSeconds(_settings.Scheduler.CheckIntervalSeconds), stoppingToken);
         }
@@ -332,6 +345,7 @@ internal void SetTimezone(string timezoneId, DateTimeOffset now)
         {
             _lastRun[task.Id] = now - task.JitterOffset;
         }
+        _ = WriteStateAsync(_tasks);
     }
 
     /// <summary>Loads the timer file, resolves the timezone and jitter, and seeds every timer at <paramref name="startup"/>.</summary>
@@ -362,6 +376,7 @@ internal List<SchedulerTask> CollectDue(DateTimeOffset now)
                 if (IsDue(cronExpression, TimeZoneFor(task), _lastRun.GetValueOrDefault(task.Id, DateTimeOffset.MinValue), task.JitterOffset, now))
                 {
                     _lastRun[task.Id] = now;
+                    _lastFired[task.Id] = now;
                     due.Add(task);
                 }
             }
@@ -373,6 +388,62 @@ internal List<SchedulerTask> CollectDue(DateTimeOffset now)
         return due;
     }
 
+    /// <summary>Reads each timer's last firing back from the state table, removes rows for timers no longer in the file, and rewrites the rest.</summary>
+    internal async Task LoadStateAsync(CancellationToken ct)
+    {
+        var table = _settings.Scheduler.StateTable;
+        if (_store is null || string.IsNullOrWhiteSpace(table)) return;
+
+        await _store.EnsureTableAsync(table, ct);
+        var ids = _tasks.Select(t => t.Id).ToHashSet(StringComparer.Ordinal);
+        var stale = new List<string>();
+        await foreach (var row in _store.QueryPartitionAsync(table, StatePartition, ct))
+        {
+            if (!ids.Contains(row.RowKey)) stale.Add(row.RowKey);
+            else if (row.GetDateTimeOffset("LastFired") is { } fired) _lastFired.TryAdd(row.RowKey, fired);
+        }
+        if (stale.Count > 0) await _store.DeleteBatchAsync(table, StatePartition, stale, ct);
+        await WriteStateAsync(_tasks, ct);
+    }
+
+    /// <summary>Records <paramref name="tasks"/> in the state table. Never throws; the rows are built before the first await, on the caller's thread.</summary>
+    internal async Task WriteStateAsync(IEnumerable<SchedulerTask> tasks, CancellationToken ct = default)
+    {
+        var table = _settings.Scheduler.StateTable;
+        if (_store is null || string.IsNullOrWhiteSpace(table)) return;
+
+        var rows = tasks.Select(StateRow).ToList();
+        try { await _store.UpsertBatchAsync(table, StatePartition, rows, ct); }
+        catch (Exception ex)
+        {
+            try { _logger.LogWarning(ex, "[Scheduler] Could not record timer state in {Table}", table); } catch { }
+        }
+    }
+
+    private StoreRow StateRow(SchedulerTask task)
+    {
+        var tz = TimeZoneFor(task);
+        var row = new StoreRow(StatePartition, task.Id)
+        {
+            ["Command"] = task.Command,
+            ["Cron"] = task.Cron,
+            ["JitterMinutes"] = task.JitterMinutes,
+            ["JitterSeconds"] = (int)task.JitterOffset.TotalSeconds,
+            ["Timezone"] = tz.Id,
+        };
+        if (_lastFired.TryGetValue(task.Id, out var fired)) row["LastFired"] = fired;
+        try
+        {
+            if (_lastRun.TryGetValue(task.Id, out var lastRun)
+                && CronExpression.Parse(task.Cron, CronFormat.IncludeSeconds).GetNextOccurrence(lastRun, tz) is { } next)
+            {
+                row["NextFire"] = next + task.JitterOffset;
+            }
+        }
+        catch (CronFormatException) { }
+        return row;
+    }
+
     private TimeZoneInfo TimeZoneFor(SchedulerTask task) =>
         (task.TZOffset || _settings.Scheduler.ApplyTZOffset) ? _configuredTz : TimeZoneInfo.Utc;
 

Services/PowerShellHost/PowerShellWorkerPool.cs

@@ -375,7 +375,8 @@ private void AddToBgPool(PowerShellWorker worker)
     /// <summary>
     /// A user request found every HTTP worker busy: build one more in the background, within HttpBurstMax, two at a
     /// time and only while the GC heap is under 80% of its limit. The waiting request takes whichever
-    /// worker frees first, so it never waits on the build itself.
+    /// worker frees first, so it never waits on the build itself. Each finished build starts the next while more
+    /// user requests wait than are being built for.
     /// </summary>
     private void GrowHttpBurst()
     {
@@ -411,6 +412,8 @@ private void GrowHttpBurst()
             _httpPool.Add(worker);
             _logger.LogInformation("[Pool] HTTP burst: added W{Id} for interactive requests ({Workers}/{Max} HTTP workers)",
                 worker.Id, workers, _httpPoolSize + _httpBurstMax);
+            // Requests that arrived while two builds were running skipped their own build.
+            if (_httpPool.PriorityWaiting > Volatile.Read(ref _httpBurstBuilding)) GrowHttpBurst();
         });
     }
 

Services/PowerShellHost/TieredHandoff.cs

@@ -14,6 +14,8 @@ public sealed class TieredHandoff<T> where T : class
 
     public int Count { get { lock (_lock) return _free.Count; } }
 
+    public int PriorityWaiting { get { lock (_lock) return _priority.Count; } }
+
     public void Add(T item)
     {
         TaskCompletionSource<T> waiter;

appsettings.example.jsonc

@@ -462,7 +462,9 @@
       // IANA or Windows timezone ID for timezone-aware cron evaluation.
       // Overridden by env var CraftTZ if set. Empty = UTC.
       // Examples: "America/New_York", "Europe/London"
-      "Timezone": ""
+      "Timezone": "",
+      // Table recording each timer's LastFired / NextFire (jitter included). Empty = not recorded.
+      "StateTable": ""
     },
 
     "Orchestrator": {

docs/configuration.md

@@ -335,7 +335,11 @@ Drives the cron-based background task system.
   "ConfigFile": "CIPPTimers.json",
 
   // How often (seconds) the scheduler checks for due tasks.
-  "CheckIntervalSeconds": 30
+  "CheckIntervalSeconds": 30,
+
+  // Table recording each timer's LastFired / NextFire (jitter included), one row per task Id
+  // in partition "Timer". Empty = not recorded.
+  "StateTable": ""
 }
 ```
 

tests/Craft.Tests/HttpBurstTests.cs

@@ -69,6 +69,27 @@ public void BurstStopsAtHttpBurstMax()
         foreach (var worker in held) _pool.Reclaim(worker, isHttp: true);
     }
 
+    [Fact]
+    public async Task UsersArrivingTogetherAllGetBurstWorkersBeyondTwoConcurrentBuilds()
+    {
+        var settings = new CraftSettings();
+        settings.Worker.HttpPoolSize = 1;
+        settings.Worker.BgPoolSize = 1;
+        settings.Worker.HttpBurstMax = 4;
+        var repo = new ScriptRepository(NullLogger<ScriptRepository>.Instance, settings);
+        using var pool = new PowerShellWorkerPool(repo, NullLogger<PowerShellWorkerPool>.Instance, new ConfigurationBuilder().Build(), settings);
+        pool.Initialize(enableHttp: true, enableBg: false);
+        var busy = pool.CheckoutHttp(Long)!;
+
+        var users = Enumerable.Range(0, 4).Select(_ => Task.Run(() => pool.CheckoutHttp(TimeSpan.FromSeconds(30)))).ToArray();
+        var workers = await Task.WhenAll(users);
+
+        Assert.All(workers, Assert.NotNull);
+        Assert.Equal(5, pool.HttpWorkers);
+        foreach (var worker in workers) pool.Reclaim(worker!, isHttp: true);
+        pool.Reclaim(busy, isHttp: true);
+    }
+
     [Fact]
     public void IdleBurstWorkersAreTrimmedBackToThePoolOneAtATime()
     {

tests/Craft.Tests/SchedulerTimezoneJitterE2ETests.cs

@@ -1,6 +1,7 @@
 using Craft.Configuration;
 using Craft.Endpoints;
 using Craft.Orchestration;
+using Craft.Storage;
 using Cronos;
 using Microsoft.Extensions.Logging.Abstractions;
 
@@ -159,4 +160,33 @@ public void TheWarmupsTimezoneSwitchMovesTimersWithoutDoubleFiring()
         Assert.InRange(nightly.At, Utc(10, 20, 0, 30) + NightlyOffset(), Utc(10, 20, 0, 30) + NightlyOffset() + Tick);
         Assert.Equal([Utc(10, 20, 3)], fired.Where(f => f.Command == "Flagged").Select(f => f.At));
     }
+
+    [Fact]
+    public async Task TheStateTableRecordsEachFiringAndTheNextWithItsJitter()
+    {
+        var store = new MemoryTableStore();
+        await store.UpsertAsync("Timers", new StoreRow("Timer", "nightly") { ["LastFired"] = Utc(10, 17, 0, 40) });
+        await store.UpsertAsync("Timers", new StoreRow("Timer", "removed"));
+        var scheduler = new SchedulerService(NullLogger<SchedulerService>.Instance, null!, null!, null!, null!,
+            new CraftSettings { Scheduler = new SchedulerSettings { ConfigFile = _timerFile, ApplyTZOffset = true, StateTable = "Timers" } },
+            null!, null!, NativeScheduledTasks.Empty, null!, store);
+        scheduler.Start(Utc(10, 18));
+
+        await scheduler.LoadStateAsync(CancellationToken.None);
+
+        // 01:30 BST is 00:30 UTC; a restart keeps the last firing and drops timers no longer in the file
+        var nightly = await store.GetAsync("Timers", "Timer", "nightly");
+        Assert.Equal(Utc(10, 17, 0, 40), nightly!.GetDateTimeOffset("LastFired"));
+        Assert.Equal(Utc(10, 18, 0, 30) + NightlyOffset(), nightly.GetDateTimeOffset("NextFire"));
+        Assert.Null(await store.GetAsync("Timers", "Timer", "removed"));
+
+        var now = Utc(10, 18);
+        List<SchedulerTask> due;
+        while (!(due = scheduler.CollectDue(now)).Exists(t => t.Id == "nightly")) now += Tick;
+        await scheduler.WriteStateAsync(due);
+
+        nightly = await store.GetAsync("Timers", "Timer", "nightly");
+        Assert.Equal(now, nightly!.GetDateTimeOffset("LastFired"));
+        Assert.Equal(Utc(10, 19, 0, 30) + NightlyOffset(), nightly.GetDateTimeOffset("NextFire"));
+    }
 }