From 951793191d8bd0fc2c4b427097443fa5b5080384 Mon Sep 17 00:00:00 2001
From: Zacgoose <107489668+Zacgoose@users.noreply.github.com>
Date: Fri, 30 Oct 2026 13:34:22 +0800
Subject: [PATCH 1/3] perf(build): publish ReadyToRun so workers start without
JIT-compiling the host
---
Craft.csproj | 1 +
1 file changed, 1 insertion(+)
diff --git a/Craft.csproj b/Craft.csproj
index 8f02e81..cb57bfc 100644
--- a/Craft.csproj
+++ b/Craft.csproj
@@ -4,6 +4,7 @@
net8.0
Craft
+ true
From 2f6a4eb650f787a76665679dc3fc16cdeac89f52 Mon Sep 17 00:00:00 2001
From: Zacgoose <107489668+Zacgoose@users.noreply.github.com>
Date: Fri, 30 Oct 2026 13:34:23 +0800
Subject: [PATCH 2/3] fix(pool): keep building burst workers while interactive
requests still wait
---
.../PowerShellHost/PowerShellWorkerPool.cs | 5 ++++-
Services/PowerShellHost/TieredHandoff.cs | 2 ++
tests/Craft.Tests/HttpBurstTests.cs | 21 +++++++++++++++++++
3 files changed, 27 insertions(+), 1 deletion(-)
diff --git a/Services/PowerShellHost/PowerShellWorkerPool.cs b/Services/PowerShellHost/PowerShellWorkerPool.cs
index 93301dc..9815a0c 100644
--- a/Services/PowerShellHost/PowerShellWorkerPool.cs
+++ b/Services/PowerShellHost/PowerShellWorkerPool.cs
@@ -375,7 +375,8 @@ private void AddToBgPool(PowerShellWorker worker)
///
/// A user request found every HTTP worker busy: build one more in the background, within HttpBurstMax, two at a
/// time and only while the GC heap is under 80% of its limit. The waiting request takes whichever
- /// worker frees first, so it never waits on the build itself.
+ /// worker frees first, so it never waits on the build itself. Each finished build starts the next while more
+ /// user requests wait than are being built for.
///
private void GrowHttpBurst()
{
@@ -411,6 +412,8 @@ private void GrowHttpBurst()
_httpPool.Add(worker);
_logger.LogInformation("[Pool] HTTP burst: added W{Id} for interactive requests ({Workers}/{Max} HTTP workers)",
worker.Id, workers, _httpPoolSize + _httpBurstMax);
+ // Requests that arrived while two builds were running skipped their own build.
+ if (_httpPool.PriorityWaiting > Volatile.Read(ref _httpBurstBuilding)) GrowHttpBurst();
});
}
diff --git a/Services/PowerShellHost/TieredHandoff.cs b/Services/PowerShellHost/TieredHandoff.cs
index f988689..dd777be 100644
--- a/Services/PowerShellHost/TieredHandoff.cs
+++ b/Services/PowerShellHost/TieredHandoff.cs
@@ -14,6 +14,8 @@ public sealed class TieredHandoff where T : class
public int Count { get { lock (_lock) return _free.Count; } }
+ public int PriorityWaiting { get { lock (_lock) return _priority.Count; } }
+
public void Add(T item)
{
TaskCompletionSource waiter;
diff --git a/tests/Craft.Tests/HttpBurstTests.cs b/tests/Craft.Tests/HttpBurstTests.cs
index 2756edd..e9114ec 100644
--- a/tests/Craft.Tests/HttpBurstTests.cs
+++ b/tests/Craft.Tests/HttpBurstTests.cs
@@ -69,6 +69,27 @@ public void BurstStopsAtHttpBurstMax()
foreach (var worker in held) _pool.Reclaim(worker, isHttp: true);
}
+ [Fact]
+ public async Task UsersArrivingTogetherAllGetBurstWorkersBeyondTwoConcurrentBuilds()
+ {
+ var settings = new CraftSettings();
+ settings.Worker.HttpPoolSize = 1;
+ settings.Worker.BgPoolSize = 1;
+ settings.Worker.HttpBurstMax = 4;
+ var repo = new ScriptRepository(NullLogger.Instance, settings);
+ using var pool = new PowerShellWorkerPool(repo, NullLogger.Instance, new ConfigurationBuilder().Build(), settings);
+ pool.Initialize(enableHttp: true, enableBg: false);
+ var busy = pool.CheckoutHttp(Long)!;
+
+ var users = Enumerable.Range(0, 4).Select(_ => Task.Run(() => pool.CheckoutHttp(TimeSpan.FromSeconds(30)))).ToArray();
+ var workers = await Task.WhenAll(users);
+
+ Assert.All(workers, Assert.NotNull);
+ Assert.Equal(5, pool.HttpWorkers);
+ foreach (var worker in workers) pool.Reclaim(worker!, isHttp: true);
+ pool.Reclaim(busy, isHttp: true);
+ }
+
[Fact]
public void IdleBurstWorkersAreTrimmedBackToThePoolOneAtATime()
{
From 4f68f36571cf242c36c3bfce31a7cc87e5fc660e Mon Sep 17 00:00:00 2001
From: Zacgoose <107489668+Zacgoose@users.noreply.github.com>
Date: Fri, 30 Oct 2026 17:25:51 +0800
Subject: [PATCH 3/3] feat(scheduler): record each timer's last and next firing
in an optional state table
---
Services/Configuration/SchedulerSettings.cs | 8 ++
Services/Orchestration/SchedulerService.cs | 73 ++++++++++++++++++-
appsettings.example.jsonc | 4 +-
docs/configuration.md | 6 +-
.../SchedulerTimezoneJitterE2ETests.cs | 30 ++++++++
5 files changed, 118 insertions(+), 3 deletions(-)
diff --git a/Services/Configuration/SchedulerSettings.cs b/Services/Configuration/SchedulerSettings.cs
index 37c0225..96d83bc 100644
--- a/Services/Configuration/SchedulerSettings.cs
+++ b/Services/Configuration/SchedulerSettings.cs
@@ -29,4 +29,12 @@ public class SchedulerSettings
/// Examples: "America/New_York", "Europe/London", "Eastern Standard Time"
///
public string Timezone { get; set; } = "";
+
+ ///
+ /// Table that records each timer's state so the hosted app can show it: one row per timer
+ /// (PartitionKey "Timer", RowKey = task Id) with Command, Cron, LastFired, NextFire (jitter included),
+ /// JitterMinutes, JitterSeconds and Timezone. Rewritten at startup, after each firing and on a timezone
+ /// change; rows for timers no longer in the file are removed at startup. Empty (default) = not recorded.
+ ///
+ public string StateTable { get; set; } = "";
}
diff --git a/Services/Orchestration/SchedulerService.cs b/Services/Orchestration/SchedulerService.cs
index 346c3cd..7602eec 100644
--- a/Services/Orchestration/SchedulerService.cs
+++ b/Services/Orchestration/SchedulerService.cs
@@ -28,9 +28,12 @@ public class SchedulerService : BackgroundService
private readonly StorageHealthMonitor _storageHealth;
private readonly NativeScheduledTasks _nativeTasks;
private readonly IServiceProvider _services;
+ private readonly ICraftTableStore? _store;
private List _tasks = [];
private readonly Dictionary _lastRun = new();
+ private readonly Dictionary _lastFired = new();
private TimeZoneInfo _configuredTz = TimeZoneInfo.Utc;
+ private const string StatePartition = "Timer";
/// Expose loaded tasks for the API layer.
public IReadOnlyList Tasks => _tasks;
@@ -45,8 +48,10 @@ public SchedulerService(
PowerShellWorkerPool pool,
StorageHealthMonitor storageHealth,
NativeScheduledTasks nativeTasks,
- IServiceProvider services)
+ IServiceProvider services,
+ ICraftTableStore? store = null)
{
+ _store = store;
_logger = logger;
_psRunner = psRunner;
_limiter = limiter;
@@ -97,6 +102,12 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
_orchestrator.MarkRecoveryDone();
}
+ try { await LoadStateAsync(stoppingToken); }
+ catch (Exception ex) when (ex is not OperationCanceledException)
+ {
+ _logger.LogWarning(ex, "[Scheduler] Could not load timer state from {Table}", _settings.Scheduler.StateTable);
+ }
+
// Retention sweeps for the rest of the process lifetime; recovery ran the first one. Fire and
// forget is deliberate: the loop handles its own failures and ends with the stopping token.
_ = _orchestrator.RunRetentionLoopAsync(stoppingToken);
@@ -190,6 +201,8 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
}
}
+ if (due.Count > 0) _ = WriteStateAsync(due, stoppingToken);
+
// Check at configured interval
await Task.Delay(TimeSpan.FromSeconds(_settings.Scheduler.CheckIntervalSeconds), stoppingToken);
}
@@ -332,6 +345,7 @@ internal void SetTimezone(string timezoneId, DateTimeOffset now)
{
_lastRun[task.Id] = now - task.JitterOffset;
}
+ _ = WriteStateAsync(_tasks);
}
/// Loads the timer file, resolves the timezone and jitter, and seeds every timer at .
@@ -362,6 +376,7 @@ internal List CollectDue(DateTimeOffset now)
if (IsDue(cronExpression, TimeZoneFor(task), _lastRun.GetValueOrDefault(task.Id, DateTimeOffset.MinValue), task.JitterOffset, now))
{
_lastRun[task.Id] = now;
+ _lastFired[task.Id] = now;
due.Add(task);
}
}
@@ -373,6 +388,62 @@ internal List CollectDue(DateTimeOffset now)
return due;
}
+ /// Reads each timer's last firing back from the state table, removes rows for timers no longer in the file, and rewrites the rest.
+ internal async Task LoadStateAsync(CancellationToken ct)
+ {
+ var table = _settings.Scheduler.StateTable;
+ if (_store is null || string.IsNullOrWhiteSpace(table)) return;
+
+ await _store.EnsureTableAsync(table, ct);
+ var ids = _tasks.Select(t => t.Id).ToHashSet(StringComparer.Ordinal);
+ var stale = new List();
+ await foreach (var row in _store.QueryPartitionAsync(table, StatePartition, ct))
+ {
+ if (!ids.Contains(row.RowKey)) stale.Add(row.RowKey);
+ else if (row.GetDateTimeOffset("LastFired") is { } fired) _lastFired.TryAdd(row.RowKey, fired);
+ }
+ if (stale.Count > 0) await _store.DeleteBatchAsync(table, StatePartition, stale, ct);
+ await WriteStateAsync(_tasks, ct);
+ }
+
+ /// Records in the state table. Never throws; the rows are built before the first await, on the caller's thread.
+ internal async Task WriteStateAsync(IEnumerable tasks, CancellationToken ct = default)
+ {
+ var table = _settings.Scheduler.StateTable;
+ if (_store is null || string.IsNullOrWhiteSpace(table)) return;
+
+ var rows = tasks.Select(StateRow).ToList();
+ try { await _store.UpsertBatchAsync(table, StatePartition, rows, ct); }
+ catch (Exception ex)
+ {
+ try { _logger.LogWarning(ex, "[Scheduler] Could not record timer state in {Table}", table); } catch { }
+ }
+ }
+
+ private StoreRow StateRow(SchedulerTask task)
+ {
+ var tz = TimeZoneFor(task);
+ var row = new StoreRow(StatePartition, task.Id)
+ {
+ ["Command"] = task.Command,
+ ["Cron"] = task.Cron,
+ ["JitterMinutes"] = task.JitterMinutes,
+ ["JitterSeconds"] = (int)task.JitterOffset.TotalSeconds,
+ ["Timezone"] = tz.Id,
+ };
+ if (_lastFired.TryGetValue(task.Id, out var fired)) row["LastFired"] = fired;
+ try
+ {
+ if (_lastRun.TryGetValue(task.Id, out var lastRun)
+ && CronExpression.Parse(task.Cron, CronFormat.IncludeSeconds).GetNextOccurrence(lastRun, tz) is { } next)
+ {
+ row["NextFire"] = next + task.JitterOffset;
+ }
+ }
+ catch (CronFormatException) { }
+ return row;
+ }
+
private TimeZoneInfo TimeZoneFor(SchedulerTask task) =>
(task.TZOffset || _settings.Scheduler.ApplyTZOffset) ? _configuredTz : TimeZoneInfo.Utc;
diff --git a/appsettings.example.jsonc b/appsettings.example.jsonc
index 49712c2..7e3ef87 100644
--- a/appsettings.example.jsonc
+++ b/appsettings.example.jsonc
@@ -462,7 +462,9 @@
// IANA or Windows timezone ID for timezone-aware cron evaluation.
// Overridden by env var CraftTZ if set. Empty = UTC.
// Examples: "America/New_York", "Europe/London"
- "Timezone": ""
+ "Timezone": "",
+ // Table recording each timer's LastFired / NextFire (jitter included). Empty = not recorded.
+ "StateTable": ""
},
"Orchestrator": {
diff --git a/docs/configuration.md b/docs/configuration.md
index 92b5a40..239ca4e 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -335,7 +335,11 @@ Drives the cron-based background task system.
"ConfigFile": "CIPPTimers.json",
// How often (seconds) the scheduler checks for due tasks.
- "CheckIntervalSeconds": 30
+ "CheckIntervalSeconds": 30,
+
+ // Table recording each timer's LastFired / NextFire (jitter included), one row per task Id
+ // in partition "Timer". Empty = not recorded.
+ "StateTable": ""
}
```
diff --git a/tests/Craft.Tests/SchedulerTimezoneJitterE2ETests.cs b/tests/Craft.Tests/SchedulerTimezoneJitterE2ETests.cs
index 5f25fac..051b83e 100644
--- a/tests/Craft.Tests/SchedulerTimezoneJitterE2ETests.cs
+++ b/tests/Craft.Tests/SchedulerTimezoneJitterE2ETests.cs
@@ -1,6 +1,7 @@
using Craft.Configuration;
using Craft.Endpoints;
using Craft.Orchestration;
+using Craft.Storage;
using Cronos;
using Microsoft.Extensions.Logging.Abstractions;
@@ -159,4 +160,33 @@ public void TheWarmupsTimezoneSwitchMovesTimersWithoutDoubleFiring()
Assert.InRange(nightly.At, Utc(10, 20, 0, 30) + NightlyOffset(), Utc(10, 20, 0, 30) + NightlyOffset() + Tick);
Assert.Equal([Utc(10, 20, 3)], fired.Where(f => f.Command == "Flagged").Select(f => f.At));
}
+
+ [Fact]
+ public async Task TheStateTableRecordsEachFiringAndTheNextWithItsJitter()
+ {
+ var store = new MemoryTableStore();
+ await store.UpsertAsync("Timers", new StoreRow("Timer", "nightly") { ["LastFired"] = Utc(10, 17, 0, 40) });
+ await store.UpsertAsync("Timers", new StoreRow("Timer", "removed"));
+ var scheduler = new SchedulerService(NullLogger.Instance, null!, null!, null!, null!,
+ new CraftSettings { Scheduler = new SchedulerSettings { ConfigFile = _timerFile, ApplyTZOffset = true, StateTable = "Timers" } },
+ null!, null!, NativeScheduledTasks.Empty, null!, store);
+ scheduler.Start(Utc(10, 18));
+
+ await scheduler.LoadStateAsync(CancellationToken.None);
+
+ // 01:30 BST is 00:30 UTC; a restart keeps the last firing and drops timers no longer in the file
+ var nightly = await store.GetAsync("Timers", "Timer", "nightly");
+ Assert.Equal(Utc(10, 17, 0, 40), nightly!.GetDateTimeOffset("LastFired"));
+ Assert.Equal(Utc(10, 18, 0, 30) + NightlyOffset(), nightly.GetDateTimeOffset("NextFire"));
+ Assert.Null(await store.GetAsync("Timers", "Timer", "removed"));
+
+ var now = Utc(10, 18);
+ List due;
+ while (!(due = scheduler.CollectDue(now)).Exists(t => t.Id == "nightly")) now += Tick;
+ await scheduler.WriteStateAsync(due);
+
+ nightly = await store.GetAsync("Timers", "Timer", "nightly");
+ Assert.Equal(now, nightly!.GetDateTimeOffset("LastFired"));
+ Assert.Equal(Utc(10, 19, 0, 30) + NightlyOffset(), nightly.GetDateTimeOffset("NextFire"));
+ }
}