From 577b2a75fa9aae9ab1072ff73959be9e8ee90782 Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Sun, 15 Nov 2026 17:11:02 +0100 Subject: [PATCH 1/5] feat(schedule): DueTimes picks the latest due time since the last firing, within 5 minutes A tick that comes late, after a takeover or a restart, still fires the due time it missed, up to 5 minutes late; when it missed several, only the latest starts a run. The old matcher's cron cases move into DueTimesTest. --- .../com/bmscomp/kates/schedule/DueTimes.java | 37 +++++ .../bmscomp/kates/schedule/DueTimesTest.java | 156 ++++++++++++++++++ .../kates/schedule/TestSchedulerCronTest.java | 136 --------------- 3 files changed, 193 insertions(+), 136 deletions(-) create mode 100644 kates/src/main/java/com/bmscomp/kates/schedule/DueTimes.java create mode 100644 kates/src/test/java/com/bmscomp/kates/schedule/DueTimesTest.java delete mode 100644 kates/src/test/java/com/bmscomp/kates/schedule/TestSchedulerCronTest.java diff --git a/kates/src/main/java/com/bmscomp/kates/schedule/DueTimes.java b/kates/src/main/java/com/bmscomp/kates/schedule/DueTimes.java new file mode 100644 index 000000000..dcdfd3c38 --- /dev/null +++ b/kates/src/main/java/com/bmscomp/kates/schedule/DueTimes.java @@ -0,0 +1,37 @@ +package com.bmscomp.kates.schedule; + +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; +import java.util.Optional; + +import com.cronutils.model.time.ExecutionTime; + +/** + * Which due time of a schedule fires now. A tick that came late, after a + * takeover or a restart, still fires the due time it missed, up to + * {@link #CATCH_UP} late; when it missed several, only the latest starts a run. + */ +public final class DueTimes { + + /** How late a due time may still fire. */ + static final Duration CATCH_UP = Duration.ofMinutes(5); + + private DueTimes() {} + + /** + * The latest due time {@code d} of {@code cron} (UNIX, UTC) with + * {@code since < d <= now} and {@code d >= now - CATCH_UP}, or empty. + * Throws IllegalArgumentException when {@code cron} does not parse. + */ + public static Optional toFire(String cron, Instant since, Instant now) { + ExecutionTime times = ExecutionTime.forCron(TestScheduler.CRON.parse(cron)); + ZonedDateTime minute = now.atZone(ZoneOffset.UTC).truncatedTo(ChronoUnit.MINUTES); + // lastExecution is strictly before its argument, so a due time at this very minute is asked first. + Optional latest = times.isMatch(minute) ? Optional.of(minute) : times.lastExecution(minute); + return latest.map(ZonedDateTime::toInstant) + .filter(due -> due.isAfter(since) && !due.isBefore(now.minus(CATCH_UP))); + } +} diff --git a/kates/src/test/java/com/bmscomp/kates/schedule/DueTimesTest.java b/kates/src/test/java/com/bmscomp/kates/schedule/DueTimesTest.java new file mode 100644 index 000000000..3f440af8a --- /dev/null +++ b/kates/src/test/java/com/bmscomp/kates/schedule/DueTimesTest.java @@ -0,0 +1,156 @@ +package com.bmscomp.kates.schedule; + +import static org.junit.jupiter.api.Assertions.*; + +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; +import java.util.Optional; + +import org.junit.jupiter.api.Test; + +class DueTimesTest { + + private static Instant at(String time) { + return Instant.parse("2026-11-15T" + time + "Z"); + } + + private static Optional toFire(String cron, String since, String now) { + return DueTimes.toFire(cron, at(since), at(now)); + } + + @Test + void aMinuteJustPastFires() { + assertEquals(Optional.of(at("10:37:00")), toFire("* * * * *", "10:36:00", "10:37:05")); + } + + @Test + void aDueTimeAlreadyFiredDoesNotFireAgain() { + assertEquals(Optional.empty(), toFire("* * * * *", "10:37:00", "10:37:55")); + } + + @Test + void aDueTimeExactlyNowFires() { + assertEquals(Optional.of(at("10:37:00")), toFire("* * * * *", "10:36:00", "10:37:00.000")); + } + + @Test + void onlyTheLatestMissedDueTimeFires() { + assertEquals(Optional.of(at("10:37:00")), toFire("* * * * *", "10:33:00", "10:37:05")); + } + + @Test + void aDueTimeWithinTheCatchUpFires() { + assertEquals( + Optional.of(at("10:30:00")), + DueTimes.toFire("30 10 * * *", Instant.parse("2026-11-14T10:30:00Z"), at("10:34:59"))); + } + + @Test + void aDueTimeOlderThanTheCatchUpIsPassedOver() { + assertEquals( + Optional.empty(), + DueTimes.toFire("30 10 * * *", Instant.parse("2026-11-14T10:30:00Z"), at("10:35:01"))); + assertEquals(Duration.ofMinutes(5), DueTimes.CATCH_UP); + } + + @Test + void aScheduleCreatedAfterADueTimeWaitsForTheNext() { + assertEquals(Optional.empty(), toFire("* * * * *", "10:36:30", "10:36:50")); + assertEquals(Optional.of(at("10:37:00")), toFire("* * * * *", "10:36:30", "10:37:01")); + } + + // ------------------------------------------------ cron semantics, from the old matcher's tests + + /** Whether {@code at}'s minute is a due time: it fires with since one minute before it. */ + private static boolean fires(String cron, ZonedDateTime at) { + Instant now = at.toInstant(); + return DueTimes.toFire(cron, now.minus(Duration.ofMinutes(1)), now) + .equals(Optional.of(now.truncatedTo(ChronoUnit.MINUTES))); + } + + private static ZonedDateTime utc(int year, int month, int day, int hour, int minute) { + return ZonedDateTime.of(year, month, day, hour, minute, 0, 0, ZoneOffset.UTC); + } + + @Test + void wildcardAndFixedMinute() { + assertTrue(fires("* * * * *", utc(2026, 2, 16, 14, 30))); + assertTrue(fires("30 * * * *", utc(2026, 2, 16, 14, 30))); + assertFalse(fires("30 * * * *", utc(2026, 2, 16, 14, 15))); + } + + @Test + void stepValues() { + String cron = "*/15 * * * *"; + assertTrue(fires(cron, utc(2026, 2, 16, 14, 0))); + assertTrue(fires(cron, utc(2026, 2, 16, 14, 15))); + assertTrue(fires(cron, utc(2026, 2, 16, 14, 45))); + assertFalse(fires(cron, utc(2026, 2, 16, 14, 7))); + } + + @Test + void commaLists() { + assertTrue(fires("0,30 * * * *", utc(2026, 2, 16, 14, 30))); + assertFalse(fires("0,30 * * * *", utc(2026, 2, 16, 14, 10))); + } + + @Test + void ranges() { + String cron = "* 9-17 * * *"; + assertTrue(fires(cron, utc(2026, 2, 16, 9, 0))); + assertTrue(fires(cron, utc(2026, 2, 16, 17, 0))); + assertFalse(fires(cron, utc(2026, 2, 16, 8, 0))); + assertFalse(fires(cron, utc(2026, 2, 16, 18, 0))); + } + + @Test + void stepWithinARange() { + String cron = "0-30/5 * * * *"; + assertTrue(fires(cron, utc(2026, 2, 16, 14, 25))); + assertFalse(fires(cron, utc(2026, 2, 16, 14, 35))); + assertFalse(fires(cron, utc(2026, 2, 16, 14, 7))); + } + + @Test + void secondsWithinTheMinuteStillFire() { + assertTrue(fires("30 14 * * *", ZonedDateTime.of(2026, 2, 16, 14, 30, 42, 0, ZoneOffset.UTC))); + } + + /** Feb 16, 2026 is a Monday, Feb 15 a Sunday. */ + @Test + void dayNamesAndSundayAsSeven() { + assertTrue(fires("0 9 * * MON", utc(2026, 2, 16, 9, 0))); + assertFalse(fires("0 9 * * MON", utc(2026, 2, 17, 9, 0))); + assertTrue(fires("0 0 * * 7", utc(2026, 2, 15, 0, 0))); + assertTrue(fires("0 0 * * 0", utc(2026, 2, 15, 0, 0))); + } + + /** A month step counts from January. */ + @Test + void monthStepStartsInJanuary() { + String cron = "0 0 1 */2 *"; + assertTrue(fires(cron, utc(2026, 1, 1, 0, 0))); + assertFalse(fires(cron, utc(2026, 2, 1, 0, 0))); + assertTrue(fires(cron, utc(2026, 3, 1, 0, 0))); + } + + /** With both days set, cron fires on either one. Thursday Oct 1, Monday Oct 5, Tuesday Oct 6, 2026. */ + @Test + void bothDayFieldsFireOnEither() { + String cron = "0 9 1 * 1"; + assertTrue(fires(cron, utc(2026, 10, 1, 9, 0))); + assertTrue(fires(cron, utc(2026, 10, 5, 9, 0))); + assertFalse(fires(cron, utc(2026, 10, 6, 9, 0))); + } + + @Test + void aCronThatDoesNotParseThrows() { + Instant now = at("10:37:00"); + assertThrows(IllegalArgumentException.class, () -> DueTimes.toFire("* *", now, now)); + assertThrows(IllegalArgumentException.class, () -> DueTimes.toFire("@daily", now, now)); + assertThrows(IllegalArgumentException.class, () -> DueTimes.toFire("60 * * * *", now, now)); + } +} diff --git a/kates/src/test/java/com/bmscomp/kates/schedule/TestSchedulerCronTest.java b/kates/src/test/java/com/bmscomp/kates/schedule/TestSchedulerCronTest.java deleted file mode 100644 index aaa38757f..000000000 --- a/kates/src/test/java/com/bmscomp/kates/schedule/TestSchedulerCronTest.java +++ /dev/null @@ -1,136 +0,0 @@ -package com.bmscomp.kates.schedule; - -import static org.junit.jupiter.api.Assertions.*; - -import java.time.ZoneOffset; -import java.time.ZonedDateTime; - -import org.junit.jupiter.api.Test; - -class TestSchedulerCronTest { - - @Test - void wildcardMatchesAny() { - ZonedDateTime now = ZonedDateTime.of(2026, 2, 16, 14, 30, 0, 0, ZoneOffset.UTC); - assertTrue(TestScheduler.matchesCron("* * * * *", now)); - } - - @Test - void fixedMinuteMatches() { - ZonedDateTime now = ZonedDateTime.of(2026, 2, 16, 14, 30, 0, 0, ZoneOffset.UTC); - assertTrue(TestScheduler.matchesCron("30 * * * *", now)); - } - - @Test - void fixedMinuteDoesNotMatch() { - ZonedDateTime now = ZonedDateTime.of(2026, 2, 16, 14, 15, 0, 0, ZoneOffset.UTC); - assertFalse(TestScheduler.matchesCron("30 * * * *", now)); - } - - @Test - void stepValueMatches() { - ZonedDateTime at0 = ZonedDateTime.of(2026, 2, 16, 14, 0, 0, 0, ZoneOffset.UTC); - ZonedDateTime at15 = ZonedDateTime.of(2026, 2, 16, 14, 15, 0, 0, ZoneOffset.UTC); - ZonedDateTime at30 = ZonedDateTime.of(2026, 2, 16, 14, 30, 0, 0, ZoneOffset.UTC); - ZonedDateTime at45 = ZonedDateTime.of(2026, 2, 16, 14, 45, 0, 0, ZoneOffset.UTC); - ZonedDateTime at7 = ZonedDateTime.of(2026, 2, 16, 14, 7, 0, 0, ZoneOffset.UTC); - - String cron = "*/15 * * * *"; - assertTrue(TestScheduler.matchesCron(cron, at0)); - assertTrue(TestScheduler.matchesCron(cron, at15)); - assertTrue(TestScheduler.matchesCron(cron, at30)); - assertTrue(TestScheduler.matchesCron(cron, at45)); - assertFalse(TestScheduler.matchesCron(cron, at7)); - } - - @Test - void commaListMatches() { - ZonedDateTime at0 = ZonedDateTime.of(2026, 2, 16, 14, 0, 0, 0, ZoneOffset.UTC); - ZonedDateTime at30 = ZonedDateTime.of(2026, 2, 16, 14, 30, 0, 0, ZoneOffset.UTC); - ZonedDateTime at10 = ZonedDateTime.of(2026, 2, 16, 14, 10, 0, 0, ZoneOffset.UTC); - - String cron = "0,30 * * * *"; - assertTrue(TestScheduler.matchesCron(cron, at0)); - assertTrue(TestScheduler.matchesCron(cron, at30)); - assertFalse(TestScheduler.matchesCron(cron, at10)); - } - - @Test - void rangeMatches() { - ZonedDateTime at9 = ZonedDateTime.of(2026, 2, 16, 9, 0, 0, 0, ZoneOffset.UTC); - ZonedDateTime at13 = ZonedDateTime.of(2026, 2, 16, 13, 0, 0, 0, ZoneOffset.UTC); - ZonedDateTime at17 = ZonedDateTime.of(2026, 2, 16, 17, 0, 0, 0, ZoneOffset.UTC); - - String cron = "* 9-17 * * *"; - assertTrue(TestScheduler.matchesCron(cron, at9)); - assertTrue(TestScheduler.matchesCron(cron, at13)); - assertTrue(TestScheduler.matchesCron(cron, at17)); - } - - @Test - void rangeExcludes() { - ZonedDateTime at8 = ZonedDateTime.of(2026, 2, 16, 8, 0, 0, 0, ZoneOffset.UTC); - ZonedDateTime at18 = ZonedDateTime.of(2026, 2, 16, 18, 0, 0, 0, ZoneOffset.UTC); - - String cron = "* 9-17 * * *"; - assertFalse(TestScheduler.matchesCron(cron, at8)); - assertFalse(TestScheduler.matchesCron(cron, at18)); - } - - @Test - void invalidCronThrows() { - ZonedDateTime now = ZonedDateTime.of(2026, 2, 16, 14, 30, 0, 0, ZoneOffset.UTC); - assertThrows(IllegalArgumentException.class, () -> TestScheduler.matchesCron("* *", now)); - assertThrows(IllegalArgumentException.class, () -> TestScheduler.matchesCron("@daily", now)); - assertThrows(IllegalArgumentException.class, () -> TestScheduler.matchesCron("60 * * * *", now)); - } - - @Test - void secondsWithinTheMinuteStillMatch() { - assertTrue( - TestScheduler.matchesCron("30 14 * * *", ZonedDateTime.of(2026, 2, 16, 14, 30, 42, 0, ZoneOffset.UTC))); - } - - /** Day names were refused by the old matcher, which saved them and never fired. Feb 16, 2026 is a Monday. */ - @Test - void dayNamesMatch() { - assertTrue(TestScheduler.matchesCron("0 9 * * MON", ZonedDateTime.of(2026, 2, 16, 9, 0, 0, 0, ZoneOffset.UTC))); - assertFalse( - TestScheduler.matchesCron("0 9 * * MON", ZonedDateTime.of(2026, 2, 17, 9, 0, 0, 0, ZoneOffset.UTC))); - } - - @Test - void stepWithinARangeMatches() { - String cron = "0-30/5 * * * *"; - assertTrue(TestScheduler.matchesCron(cron, ZonedDateTime.of(2026, 2, 16, 14, 25, 0, 0, ZoneOffset.UTC))); - assertFalse(TestScheduler.matchesCron(cron, ZonedDateTime.of(2026, 2, 16, 14, 35, 0, 0, ZoneOffset.UTC))); - assertFalse(TestScheduler.matchesCron(cron, ZonedDateTime.of(2026, 2, 16, 14, 7, 0, 0, ZoneOffset.UTC))); - } - - /** Day-of-week 7 is Sunday, as 0 is. Feb 15, 2026 is a Sunday. */ - @Test - void daySevenIsSunday() { - ZonedDateTime sunday = ZonedDateTime.of(2026, 2, 15, 0, 0, 0, 0, ZoneOffset.UTC); - assertTrue(TestScheduler.matchesCron("0 0 * * 7", sunday)); - assertTrue(TestScheduler.matchesCron("0 0 * * 0", sunday)); - } - - /** A month step counts from January: the old matcher's value % 2 picked February, April, ... */ - @Test - void monthStepStartsInJanuary() { - String cron = "0 0 1 */2 *"; - assertTrue(TestScheduler.matchesCron(cron, ZonedDateTime.of(2026, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC))); - assertFalse(TestScheduler.matchesCron(cron, ZonedDateTime.of(2026, 2, 1, 0, 0, 0, 0, ZoneOffset.UTC))); - assertTrue(TestScheduler.matchesCron(cron, ZonedDateTime.of(2026, 3, 1, 0, 0, 0, 0, ZoneOffset.UTC))); - } - - /** With both days set, cron fires on either one, not only when both hold. */ - @Test - void dayOfMonthOrDayOfWeek() { - String cron = "0 9 1 * 1"; - // Thursday Oct 1, Monday Oct 5, Tuesday Oct 6, 2026. - assertTrue(TestScheduler.matchesCron(cron, ZonedDateTime.of(2026, 10, 1, 9, 0, 0, 0, ZoneOffset.UTC))); - assertTrue(TestScheduler.matchesCron(cron, ZonedDateTime.of(2026, 10, 5, 9, 0, 0, 0, ZoneOffset.UTC))); - assertFalse(TestScheduler.matchesCron(cron, ZonedDateTime.of(2026, 10, 6, 9, 0, 0, 0, ZoneOffset.UTC))); - } -} From 806b037e5e3e67e7a7e173625214cb59f4abcd54 Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Sun, 15 Nov 2026 17:14:15 +0100 Subject: [PATCH 2/5] feat(schedule): a due time is claimed by one conditional update of last_due V31 adds last_due to scheduled_test_runs and disruption_schedules. ScheduleFirings.claim moves it forward to a due time in one UPDATE ... WHERE last_due IS NULL OR last_due < :due, and says whether this call moved it: two members that both believe they lead fire a due time once. JPA never writes last_due, so a PUT that loaded a schedule before a claim and commits after it cannot write the old value back. It is not in API responses. --- .../disruption/DisruptionScheduleEntity.java | 11 ++ .../kates/schedule/ScheduleFirings.java | 47 ++++++ .../kates/schedule/ScheduledTestRun.java | 11 ++ .../db/migration/V31__schedule_last_due.sql | 5 + .../bmscomp/kates/it/ScheduleFiringsIT.java | 135 ++++++++++++++++++ 5 files changed, 209 insertions(+) create mode 100644 kates/src/main/java/com/bmscomp/kates/schedule/ScheduleFirings.java create mode 100644 kates/src/main/resources/db/migration/V31__schedule_last_due.sql create mode 100644 kates/src/test/java/com/bmscomp/kates/it/ScheduleFiringsIT.java diff --git a/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionScheduleEntity.java b/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionScheduleEntity.java index 74e3529c1..04d0b70bd 100644 --- a/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionScheduleEntity.java +++ b/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionScheduleEntity.java @@ -3,6 +3,8 @@ import java.time.Instant; import jakarta.persistence.*; +import com.fasterxml.jackson.annotation.JsonIgnore; + /** * JPA entity for a scheduled/recurring disruption test. * Supports either a playbook name or a raw plan JSON. @@ -36,6 +38,11 @@ public class DisruptionScheduleEntity { @Column(name = "last_run_at") private Instant lastRunAt; + /** Written only by ScheduleFirings, in SQL, so a save of a stale copy never takes a claim back. */ + @JsonIgnore + @Column(name = "last_due", insertable = false, updatable = false) + private Instant lastDue; + @Column(name = "created_at", nullable = false) private Instant createdAt; @@ -114,4 +121,8 @@ public Instant getCreatedAt() { public void setCreatedAt(Instant createdAt) { this.createdAt = createdAt; } + + public Instant getLastDue() { + return lastDue; + } } diff --git a/kates/src/main/java/com/bmscomp/kates/schedule/ScheduleFirings.java b/kates/src/main/java/com/bmscomp/kates/schedule/ScheduleFirings.java new file mode 100644 index 000000000..f44d289e3 --- /dev/null +++ b/kates/src/main/java/com/bmscomp/kates/schedule/ScheduleFirings.java @@ -0,0 +1,47 @@ +package com.bmscomp.kates.schedule; + +import java.time.Instant; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import jakarta.persistence.EntityManager; +import jakarta.transaction.Transactional; + +/** + * Claims a schedule's due time in the database. Two members that both believe + * they lead, a paused leader and the one that took over, may both tick; the + * conditional update lets only one of them move last_due to the due time, and + * only that one fires it. + */ +@ApplicationScoped +public class ScheduleFirings { + + /** The tables that hold schedules; the only table names the claim's SQL takes. */ + public enum Kind { + TESTS("scheduled_test_runs"), + DISRUPTIONS("disruption_schedules"); + + private final String table; + + Kind(String table) { + this.table = table; + } + } + + @Inject + EntityManager em; + + /** + * Moves the schedule's last_due forward to {@code due}; true when this call + * moved it, and so fires that due time. A database error propagates: the + * due time then fires nowhere, rather than perhaps twice. + */ + @Transactional(Transactional.TxType.REQUIRES_NEW) + public boolean claim(Kind kind, String id, Instant due) { + return em.createNativeQuery("UPDATE " + kind.table + " SET last_due = :due" + + " WHERE id = :id AND (last_due IS NULL OR last_due < :due)") + .setParameter("due", due) + .setParameter("id", id) + .executeUpdate() + > 0; + } +} diff --git a/kates/src/main/java/com/bmscomp/kates/schedule/ScheduledTestRun.java b/kates/src/main/java/com/bmscomp/kates/schedule/ScheduledTestRun.java index ed05e8157..7d369be4c 100644 --- a/kates/src/main/java/com/bmscomp/kates/schedule/ScheduledTestRun.java +++ b/kates/src/main/java/com/bmscomp/kates/schedule/ScheduledTestRun.java @@ -6,6 +6,8 @@ import jakarta.persistence.Id; import jakarta.persistence.Table; +import com.fasterxml.jackson.annotation.JsonIgnore; + /** * JPA entity for a scheduled/recurring test configuration. * Stores the cron expression and the serialized CreateTestRequest. @@ -36,6 +38,11 @@ public class ScheduledTestRun { @Column(name = "last_run_at") private Instant lastRunAt; + /** Written only by ScheduleFirings, in SQL, so a save of a stale copy never takes a claim back. */ + @JsonIgnore + @Column(name = "last_due", insertable = false, updatable = false) + private Instant lastDue; + @Column(name = "created_at", nullable = false) private Instant createdAt; @@ -106,4 +113,8 @@ public Instant getCreatedAt() { public void setCreatedAt(Instant createdAt) { this.createdAt = createdAt; } + + public Instant getLastDue() { + return lastDue; + } } diff --git a/kates/src/main/resources/db/migration/V31__schedule_last_due.sql b/kates/src/main/resources/db/migration/V31__schedule_last_due.sql new file mode 100644 index 000000000..70358f3f8 --- /dev/null +++ b/kates/src/main/resources/db/migration/V31__schedule_last_due.sql @@ -0,0 +1,5 @@ +-- The latest due time a schedule fired, or passed over. A tick claims a due +-- time by moving last_due forward in one conditional UPDATE, and fires only +-- when that changed a row, so each due time starts at most one run. +ALTER TABLE scheduled_test_runs ADD COLUMN last_due TIMESTAMPTZ; +ALTER TABLE disruption_schedules ADD COLUMN last_due TIMESTAMPTZ; diff --git a/kates/src/test/java/com/bmscomp/kates/it/ScheduleFiringsIT.java b/kates/src/test/java/com/bmscomp/kates/it/ScheduleFiringsIT.java new file mode 100644 index 000000000..a65f9dfb5 --- /dev/null +++ b/kates/src/test/java/com/bmscomp/kates/it/ScheduleFiringsIT.java @@ -0,0 +1,135 @@ +package com.bmscomp.kates.it; + +import static org.junit.jupiter.api.Assertions.*; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import jakarta.inject.Inject; +import jakarta.persistence.EntityManager; + +import io.quarkus.narayana.jta.QuarkusTransaction; +import io.quarkus.test.common.QuarkusTestResource; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.TestProfile; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import com.bmscomp.kates.disruption.DisruptionScheduleEntity; +import com.bmscomp.kates.schedule.ScheduleFirings; +import com.bmscomp.kates.schedule.ScheduleFirings.Kind; +import com.bmscomp.kates.schedule.ScheduledTestRun; +import com.bmscomp.kates.schedule.ScheduledTestRunRepository; + +/** + * A schedule's due time is claimed by one conditional update of last_due, on + * the Flyway table: two members that both believe they lead fire it once. + */ +@QuarkusTest +@TestProfile(NoSchedulersTestProfile.class) +@QuarkusTestResource(value = PostgresTestResource.class, restrictToAnnotatedClass = true) +class ScheduleFiringsIT { + + private static final Instant DUE = Instant.parse("2026-11-15T10:37:00Z"); + + @Inject + EntityManager em; + + @Inject + ScheduleFirings firings; + + @Inject + ScheduledTestRunRepository repository; + + @BeforeEach + void resetSchedules() { + ItSupport.truncate(em, "scheduled_test_runs", "disruption_schedules"); + } + + @Test + void concurrentClaimsOfOneDueTimeSucceedOnce() throws Exception { + String id = testSchedule(); + assertEquals(1, claimsWon(Kind.TESTS, id)); + } + + @Test + void concurrentClaimsOfADisruptionDueTimeSucceedOnce() throws Exception { + DisruptionScheduleEntity schedule = new DisruptionScheduleEntity(); + schedule.setId("dsched01"); + schedule.setName("every-minute"); + schedule.setCronExpression("* * * * *"); + schedule.setPlaybookName("az-failure"); + QuarkusTransaction.requiringNew().run(() -> em.persist(schedule)); + + assertEquals(1, claimsWon(Kind.DISRUPTIONS, "dsched01")); + } + + @Test + void anEarlierDueTimeIsNotClaimedAfterALaterOne() { + String id = testSchedule(); + assertTrue(firings.claim(Kind.TESTS, id, DUE)); + assertFalse(firings.claim(Kind.TESTS, id, DUE.minus(Duration.ofMinutes(1)))); + assertFalse(firings.claim(Kind.TESTS, id, DUE)); + assertTrue(firings.claim(Kind.TESTS, id, DUE.plus(Duration.ofMinutes(1)))); + } + + /** + * A PUT that loaded the schedule before a tick's claim, and commits after + * it, must not write its stale last_due back, or the due time fires again. + */ + @Test + void aSaveAfterAClaimKeepsTheClaim() { + String id = testSchedule(); + + QuarkusTransaction.requiringNew().run(() -> { + ScheduledTestRun loaded = em.find(ScheduledTestRun.class, id); + assertTrue(firings.claim(Kind.TESTS, id, DUE)); + loaded.setName("renamed"); + }); + + ScheduledTestRun reread = QuarkusTransaction.requiringNew() + .call(() -> repository.findById(id).orElseThrow()); + assertEquals("renamed", reread.getName()); + assertEquals(DUE, reread.getLastDue()); + } + + private String testSchedule() { + ScheduledTestRun schedule = new ScheduledTestRun(); + schedule.setId("sched001"); + schedule.setName("every-minute"); + schedule.setCronExpression("* * * * *"); + schedule.setEnabled(false); + schedule.setRequestJson("{\"type\":\"LOAD\"}"); + repository.save(schedule); + return schedule.getId(); + } + + /** How many of 8 callers, released together, win the claim of DUE. */ + private int claimsWon(Kind kind, String id) throws Exception { + int callers = 8; + CountDownLatch go = new CountDownLatch(1); + ExecutorService pool = Executors.newFixedThreadPool(callers); + try { + List> claims = new ArrayList<>(); + for (int i = 0; i < callers; i++) { + claims.add(pool.submit(() -> { + go.await(); + return firings.claim(kind, id, DUE); + })); + } + go.countDown(); + int won = 0; + for (Future claim : claims) { + won += claim.get() ? 1 : 0; + } + return won; + } finally { + pool.shutdownNow(); + } + } +} From 2bd74fe8a232e209844034fd6d94d582bd333c05 Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Sun, 15 Nov 2026 17:28:39 +0100 Subject: [PATCH 3/5] fix(schedule): each due time fires once, on the leader, up to 5 minutes late; the scheduler lease is gone Both evaluators matched the cron against the current minute under a 55 s PostgreSQL lease held by a per-process id. A leader killed at 10:36:52 left its lease to block the new leader's next tick, and the 10:37 run never started (s12 of the failure study); ticks 60 s apart could also straddle a minute and skip it. The evaluators now tick every 10 s. For each enabled schedule DueTimes picks the latest due time since last_due (or since the schedule was created), up to 5 minutes late, and ScheduleFirings claims it; only the member whose claim moved last_due starts the run. A refused start still counts as the firing. A test run carries the Idempotency-Key schedule::. A PUT that turns a schedule on or changes its cron passes over the due times before it, so re-enabling does not fire at once. SchedulerLeaseService and its callers are removed; scheduler_leases stays one release, for pods of the previous version during a rolling upgrade. --- .../DisruptionScheduleResource.java | 9 ++ .../kates/disruption/DisruptionScheduler.java | 40 +++-- .../kates/schedule/ScheduleResource.java | 10 ++ .../bmscomp/kates/schedule/TestScheduler.java | 62 ++++--- .../kates/service/SchedulerLeaseService.java | 59 ------- .../DisruptionReportPersistenceTest.java | 8 +- .../com/bmscomp/kates/it/PersistenceIT.java | 30 ---- .../kates/schedule/ScheduleResourceTest.java | 12 +- .../schedule/TestSchedulerFiringTest.java | 153 ++++++++++++++++++ 9 files changed, 232 insertions(+), 151 deletions(-) delete mode 100644 kates/src/main/java/com/bmscomp/kates/service/SchedulerLeaseService.java create mode 100644 kates/src/test/java/com/bmscomp/kates/schedule/TestSchedulerFiringTest.java diff --git a/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionScheduleResource.java b/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionScheduleResource.java index 4c57a8da4..a0c8c34e6 100644 --- a/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionScheduleResource.java +++ b/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionScheduleResource.java @@ -46,6 +46,9 @@ public class DisruptionScheduleResource { @Inject ObjectMapper objectMapper; + @Inject + com.bmscomp.kates.schedule.ScheduleFirings firings; + @GET @Operation(summary = "List disruption schedules") public Response listSchedules( @@ -154,6 +157,8 @@ public Response updateSchedule( return badCron.get(); } } + boolean passOver = (request.enabled == Boolean.TRUE && !entity.isEnabled()) + || (request.cronExpression != null && !request.cronExpression.equals(entity.getCronExpression())); if (request.name != null) entity.setName(request.name); if (request.cronExpression != null) entity.setCronExpression(request.cronExpression); if (request.enabled != null) entity.setEnabled(request.enabled); @@ -168,6 +173,10 @@ public Response updateSchedule( } } em.merge(entity); + if (passOver) { + // Turning a schedule on or re-timing it passes over the due times before now. + firings.claim(com.bmscomp.kates.schedule.ScheduleFirings.Kind.DISRUPTIONS, id, java.time.Instant.now()); + } return Response.ok(entity).build(); } diff --git a/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionScheduler.java b/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionScheduler.java index 2122347bf..b1472d3a4 100644 --- a/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionScheduler.java +++ b/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionScheduler.java @@ -1,8 +1,6 @@ package com.bmscomp.kates.disruption; import java.time.Instant; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; import java.util.List; import java.util.UUID; import jakarta.enterprise.context.ApplicationScoped; @@ -14,8 +12,6 @@ import io.quarkus.scheduler.Scheduled; import org.jboss.logging.Logger; -import com.bmscomp.kates.schedule.TestScheduler; - /** * Cron-based scheduler for recurring disruption tests. * Evaluates enabled schedules every 60 seconds, executing matching plans @@ -48,39 +44,41 @@ public class DisruptionScheduler { ObjectMapper objectMapper; @jakarta.inject.Inject - com.bmscomp.kates.service.SchedulerLeaseService leases; + com.bmscomp.kates.schedule.ScheduleFirings firings; @jakarta.inject.Inject com.bmscomp.kates.quorum.Quorum quorum; - @Scheduled(every = "60s", identity = "kates-disruption-schedule-evaluator") + @Scheduled(every = "10s", identity = "kates-disruption-schedule-evaluator") void evaluateSchedules() { - // A draining instance fires nothing, and leaves the lease to one that may; so does a member - // that does not lead its quorum. + evaluate(Instant.now()); + } + + /** + * One tick at {@code now}; package-private, so a test ticks at the time it + * needs. Each due time runs once, claimed as TestScheduler claims a test's. + */ + void evaluate(Instant now) { + // A draining instance fires nothing; nor does a member that does not lead its quorum. if (concurrencyGuard.isDraining() || !quorum.acceptsWrites()) { return; } - // Only the lease holder may trigger disruptions — with replicas > 1 - // each schedule would otherwise inject faults once per replica. - if (!leases.tryAcquire("disruption-scheduler", java.time.Duration.ofSeconds(55))) { - return; - } List schedules = em.createQuery( "SELECT s FROM DisruptionScheduleEntity s WHERE s.enabled = true", DisruptionScheduleEntity.class) .getResultList(); - - if (schedules.isEmpty()) return; - - ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); LOG.debugf("Evaluating %d disruption schedules at %s", schedules.size(), now); for (DisruptionScheduleEntity schedule : schedules) { try { - if (TestScheduler.matchesCron(schedule.getCronExpression(), now)) { - LOG.info("Disruption schedule '" + schedule.getName() + "' triggered at " + now); - executeSchedule(schedule); - } + Instant since = schedule.getLastDue() != null ? schedule.getLastDue() : schedule.getCreatedAt(); + com.bmscomp.kates.schedule.DueTimes.toFire(schedule.getCronExpression(), since, now) + .filter(due -> firings.claim( + com.bmscomp.kates.schedule.ScheduleFirings.Kind.DISRUPTIONS, schedule.getId(), due)) + .ifPresent(due -> { + LOG.info("Disruption schedule '" + schedule.getName() + "' triggered for " + due); + executeSchedule(schedule); + }); } catch (Exception e) { LOG.warn("Failed to evaluate disruption schedule '" + schedule.getName() + "'", e); } diff --git a/kates/src/main/java/com/bmscomp/kates/schedule/ScheduleResource.java b/kates/src/main/java/com/bmscomp/kates/schedule/ScheduleResource.java index 6b13c6e9e..24fe3b0ed 100644 --- a/kates/src/main/java/com/bmscomp/kates/schedule/ScheduleResource.java +++ b/kates/src/main/java/com/bmscomp/kates/schedule/ScheduleResource.java @@ -48,6 +48,9 @@ public class ScheduleResource { @Inject Validator validator; + @Inject + ScheduleFirings firings; + /** * A 400 naming each field of the schedule's test request that the Kates * API would refuse, by its path under {@code testRequest}, or empty when @@ -242,6 +245,9 @@ public Response updateSchedule( return refused.get(); } } + boolean passOver = (request.enabled == Boolean.TRUE && !schedule.isEnabled()) + || (request.cronExpression != null + && !request.cronExpression.equals(schedule.getCronExpression())); if (request.name != null) schedule.setName(request.name); if (request.cronExpression != null) schedule.setCronExpression(request.cronExpression); if (request.enabled != null) schedule.setEnabled(request.enabled); @@ -253,6 +259,10 @@ public Response updateSchedule( } } repository.save(schedule); + if (passOver) { + // Turning a schedule on or re-timing it passes over the due times before now. + firings.claim(ScheduleFirings.Kind.TESTS, id, java.time.Instant.now()); + } return Response.ok(schedule).build(); }) .orElseGet(() -> Response.status(404) diff --git a/kates/src/main/java/com/bmscomp/kates/schedule/TestScheduler.java b/kates/src/main/java/com/bmscomp/kates/schedule/TestScheduler.java index 67f680a9a..0409f6b39 100644 --- a/kates/src/main/java/com/bmscomp/kates/schedule/TestScheduler.java +++ b/kates/src/main/java/com/bmscomp/kates/schedule/TestScheduler.java @@ -1,8 +1,6 @@ package com.bmscomp.kates.schedule; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; -import java.time.temporal.ChronoUnit; +import java.time.Instant; import java.util.List; import java.util.Map; import jakarta.enterprise.context.ApplicationScoped; @@ -10,7 +8,6 @@ import com.cronutils.model.CronType; import com.cronutils.model.definition.CronDefinitionBuilder; -import com.cronutils.model.time.ExecutionTime; import com.cronutils.parser.CronParser; import com.fasterxml.jackson.databind.ObjectMapper; import io.quarkus.scheduler.Scheduled; @@ -24,8 +21,10 @@ import com.bmscomp.kates.util.Result; /** - * Evaluates all enabled schedules every 60 seconds. - * When a cron expression matches the current minute, the associated test is executed. + * Evaluates all enabled schedules every 10 seconds, and starts the test of + * each due time once: DueTimes picks it, up to 5 minutes late, and + * ScheduleFirings claims it in the database, so a second member that believes + * it leads, or the next tick, finds it taken. */ @ApplicationScoped public class TestScheduler { @@ -46,7 +45,7 @@ public class TestScheduler { com.bmscomp.kates.quorum.Quorum quorum; @Inject - com.bmscomp.kates.service.SchedulerLeaseService leases; + ScheduleFirings firings; @Inject SpecLimits specLimits; @@ -54,35 +53,32 @@ public class TestScheduler { @Inject com.bmscomp.kates.service.AuditService auditService; - @Scheduled(every = "60s", identity = "kates-schedule-evaluator") + @Scheduled(every = "10s", identity = "kates-schedule-evaluator") void evaluateSchedules() { - // A draining instance fires nothing, and leaves the lease to one that may. - // The guard's flag, set with the orchestrator's by PUT /api/drain: asking - // the orchestrator would create it on the first tick, and its startup - // recovery with it. - // A member that does not lead fires nothing either, so a follower never holds the lease. + evaluate(Instant.now()); + } + + /** One tick at {@code now}; package-private, so a test ticks at the time it needs. */ + void evaluate(Instant now) { + // A draining instance fires nothing. The guard's flag, set with the + // orchestrator's by PUT /api/drain: asking the orchestrator would create + // it on the first tick, and its startup recovery with it. + // A member that does not lead fires nothing either. if (drain.isDraining() || !quorum.acceptsWrites()) { return; } - // With replicas > 1 every instance fires this — only the lease holder - // may trigger runs, or each schedule would execute once per replica. - if (!leases.tryAcquire("test-scheduler", java.time.Duration.ofSeconds(55))) { - return; - } List schedules = repository.findAllEnabled(); - if (schedules.isEmpty()) { - return; - } - - ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); LOG.debugf("Evaluating %d schedules at %s", schedules.size(), now); for (ScheduledTestRun schedule : schedules) { try { - if (matchesCron(schedule.getCronExpression(), now)) { - LOG.info("Schedule '" + schedule.getName() + "' triggered at " + now); - executeSchedule(schedule); - } + Instant since = schedule.getLastDue() != null ? schedule.getLastDue() : schedule.getCreatedAt(); + DueTimes.toFire(schedule.getCronExpression(), since, now) + .filter(due -> firings.claim(ScheduleFirings.Kind.TESTS, schedule.getId(), due)) + .ifPresent(due -> { + LOG.info("Schedule '" + schedule.getName() + "' triggered for " + due); + executeSchedule(schedule, due); + }); } catch (Exception e) { LOG.warn("Failed to evaluate schedule '" + schedule.getName() + "'", e); } @@ -105,13 +101,16 @@ void evaluateSchedules() { * scenario with a type of its own goes without. A schedule saved before * the Kates API kept only the fields a request sets holds the old Java * defaults, which are all within the limits, so it still fires. + * + *

The run carries the Idempotency-Key {@code schedule::}, so it + * names the schedule and due time that started it. */ - void executeSchedule(ScheduledTestRun schedule) { + void executeSchedule(ScheduledTestRun schedule, Instant due) { try { CreateTestRequest request = JSON.readValue(schedule.getRequestJson(), CreateTestRequest.class); Map outsideLimits = specLimits.violations(request.getSpec()); Result result = outsideLimits.isEmpty() - ? orchestrator.executeTest(request) + ? orchestrator.executeTest(request, "schedule:" + schedule.getId() + ":" + due) : Result.failure(new InvalidTestSpecException(outsideLimits)); if (result.isSuccess()) { var run = result.asSuccess().orElseThrow(); @@ -134,9 +133,4 @@ void executeSchedule(ScheduledTestRun schedule) { /** Standard 5-field UNIX cron; schedules are validated with it on POST and PUT. */ public static final CronParser CRON = new CronParser(CronDefinitionBuilder.instanceDefinitionFor(CronType.UNIX)); - - /** Whether the expression fires in the minute of {@code now}; throws when it doesn't parse. */ - public static boolean matchesCron(String cronExpr, ZonedDateTime now) { - return ExecutionTime.forCron(CRON.parse(cronExpr)).isMatch(now.truncatedTo(ChronoUnit.MINUTES)); - } } diff --git a/kates/src/main/java/com/bmscomp/kates/service/SchedulerLeaseService.java b/kates/src/main/java/com/bmscomp/kates/service/SchedulerLeaseService.java deleted file mode 100644 index 1e1b9d9ba..000000000 --- a/kates/src/main/java/com/bmscomp/kates/service/SchedulerLeaseService.java +++ /dev/null @@ -1,59 +0,0 @@ -package com.bmscomp.kates.service; - -import java.time.Duration; -import java.time.Instant; -import java.util.UUID; -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.inject.Inject; -import jakarta.persistence.EntityManager; -import jakarta.transaction.Transactional; - -import org.jboss.logging.Logger; - -/** - * Cooperative, DB-backed leases for {@code @Scheduled} jobs that must fire on - * exactly one replica (schedule evaluators would otherwise trigger duplicate - * test runs when replicas > 1). Claim is a single atomic upsert: - * insert wins when no row exists; update wins only when the previous lease - * expired or we already hold it. Postgres and H2 (PostgreSQL mode) both - * support {@code ON CONFLICT ... DO UPDATE}. - */ -@ApplicationScoped -public class SchedulerLeaseService { - - private static final Logger LOG = Logger.getLogger(SchedulerLeaseService.class); - - /** Stable per-instance identity for the lifetime of this process. */ - private final String holderId = UUID.randomUUID().toString(); - - @Inject - EntityManager em; - - /** - * @return true when this instance holds the named lease for the next - * {@code ttl}; false when another live replica holds it. - */ - @Transactional(Transactional.TxType.REQUIRES_NEW) - public boolean tryAcquire(String name, Duration ttl) { - try { - Instant expiresAt = Instant.now().plus(ttl); - int updated = em.createNativeQuery("INSERT INTO scheduler_leases (name, holder, expires_at)" - + " VALUES (:name, :holder, :expiresAt)" - + " ON CONFLICT (name) DO UPDATE" - + " SET holder = EXCLUDED.holder, expires_at = EXCLUDED.expires_at" - + " WHERE scheduler_leases.expires_at < NOW()" - + " OR scheduler_leases.holder = EXCLUDED.holder") - .setParameter("name", name) - .setParameter("holder", holderId) - .setParameter("expiresAt", expiresAt) - .executeUpdate(); - return updated > 0; - } catch (Exception e) { - // Fail open: a broken lease table must not silently stop the only - // replica from doing scheduled work. Duplicate work on transient - // DB errors is the lesser evil. - LOG.warn("Scheduler lease acquisition failed for '" + name + "' — proceeding without lease", e); - return true; - } - } -} diff --git a/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionReportPersistenceTest.java b/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionReportPersistenceTest.java index e3d155841..8b52b3140 100644 --- a/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionReportPersistenceTest.java +++ b/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionReportPersistenceTest.java @@ -12,6 +12,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.time.Duration; @@ -238,7 +240,11 @@ void aScheduledRunStoresItsReportOnce() { .extract() .path("id"); try { - scheduler.evaluateSchedules(); + // Two members' ticks after the next minute: the due time runs once. + java.time.Instant late = java.time.Instant.now().plusSeconds(61); + scheduler.evaluate(late); + scheduler.evaluate(late.plusSeconds(10)); + verify(orchestrator, times(1)).execute(any(), any()); String runId = given().when() .get("/api/disruptions/schedules") diff --git a/kates/src/test/java/com/bmscomp/kates/it/PersistenceIT.java b/kates/src/test/java/com/bmscomp/kates/it/PersistenceIT.java index 0c45445cf..9302228ed 100644 --- a/kates/src/test/java/com/bmscomp/kates/it/PersistenceIT.java +++ b/kates/src/test/java/com/bmscomp/kates/it/PersistenceIT.java @@ -4,7 +4,6 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -import java.time.Duration; import java.util.List; import java.util.UUID; import jakarta.inject.Inject; @@ -22,7 +21,6 @@ import com.bmscomp.kates.domain.TestRun; import com.bmscomp.kates.domain.TestSpec; import com.bmscomp.kates.domain.TestType; -import com.bmscomp.kates.service.SchedulerLeaseService; import com.bmscomp.kates.service.TestRunRepository; import com.bmscomp.kates.webhook.WebhookService; @@ -46,9 +44,6 @@ class PersistenceIT { @Inject WebhookService webhookService; - @Inject - SchedulerLeaseService leases; - @Test void flywayMigrationsAllApplied() { Number applied = @@ -108,31 +103,6 @@ void webhookRegistrationsArePersisted() { assertFalse(webhookService.list().stream().anyMatch(r -> r.name().equals("it-hook"))); } - @Test - void schedulerLeaseClaimSemantics() { - String lease = "it-lease-" + UUID.randomUUID(); - - // First claim and same-holder renewal both succeed. - assertTrue(leases.tryAcquire(lease, Duration.ofSeconds(30))); - assertTrue(leases.tryAcquire(lease, Duration.ofSeconds(30))); - - // A live lease held by another replica cannot be stolen... - QuarkusTransaction.requiringNew() - .run(() -> em.createNativeQuery("UPDATE scheduler_leases SET holder = 'other-replica'," - + " expires_at = NOW() + INTERVAL '60 seconds' WHERE name = :name") - .setParameter("name", lease) - .executeUpdate()); - assertFalse(leases.tryAcquire(lease, Duration.ofSeconds(30))); - - // ...but an expired one can. - QuarkusTransaction.requiringNew() - .run(() -> em.createNativeQuery( - "UPDATE scheduler_leases SET expires_at = NOW() - INTERVAL '1 second' WHERE name = :name") - .setParameter("name", lease) - .executeUpdate()); - assertTrue(leases.tryAcquire(lease, Duration.ofSeconds(30))); - } - /** * An INTEGRITY task's integrity result goes into the integrity_json column * V25 adds, and comes back with the run. It was not stored, so a finished diff --git a/kates/src/test/java/com/bmscomp/kates/schedule/ScheduleResourceTest.java b/kates/src/test/java/com/bmscomp/kates/schedule/ScheduleResourceTest.java index d9ab47e72..e2dda1cd4 100644 --- a/kates/src/test/java/com/bmscomp/kates/schedule/ScheduleResourceTest.java +++ b/kates/src/test/java/com/bmscomp/kates/schedule/ScheduleResourceTest.java @@ -534,7 +534,7 @@ void aStoredTestRequestTheApiRefusesStartsNoRun() { ScheduledTestRun s = storedSchedule( "{\"type\":\"STRESS\",\"backend\":\"trogdor\",\"spec\":{\"consumerGroup\":\"perf-cg\"}}"); - scheduler.executeSchedule(s); + scheduler.executeSchedule(s, java.time.Instant.now()); Mockito.verify(repository, Mockito.never()).updateLastRun(anyString(), anyString()); Mockito.verifyNoInteractions(trogdorClient); @@ -553,7 +553,7 @@ void aStoredScenarioOutsideItsLimitsStartsNoRun() { + "{\"name\":\"steady\",\"phaseType\":\"STEADY\",\"durationMs\":60000," + "\"spec\":{\"topic\":\"not a topic!\"}}]}}"); - scheduler.executeSchedule(s); + scheduler.executeSchedule(s, java.time.Instant.now()); Mockito.verify(repository, Mockito.never()).updateLastRun(anyString(), anyString()); Mockito.verifyNoInteractions(trogdorClient); @@ -600,7 +600,7 @@ void aRateOfZeroIsRefusedOnPostOnPutAndAtEachFiring() { + "\"phases\":[{\"name\":\"steady\",\"phaseType\":\"STEADY\",\"durationMs\":60000}]}}", "{\"type\":\"LOAD\",\"backend\":\"trogdor\",\"scenario\":{\"phases\":[{\"name\":\"steady\"," + "\"phaseType\":\"STEADY\",\"durationMs\":60000,\"targetThroughput\":0}]}}")) { - scheduler.executeSchedule(storedSchedule(stored)); + scheduler.executeSchedule(storedSchedule(stored), java.time.Instant.now()); } Mockito.verify(repository, Mockito.never()).updateLastRun(anyString(), anyString()); Mockito.verify(trogdorClient, Mockito.never()).createTask(any()); @@ -639,7 +639,7 @@ void aScheduleRunsTheRequestItWasGiven(TestType type) { ArgumentCaptor saved = ArgumentCaptor.forClass(ScheduledTestRun.class); Mockito.verify(repository).save(saved.capture()); - scheduler.executeSchedule(saved.getValue()); + scheduler.executeSchedule(saved.getValue(), java.time.Instant.now()); ArgumentCaptor runId = ArgumentCaptor.forClass(String.class); Mockito.verify(repository).updateLastRun(eq(saved.getValue().getId()), runId.capture()); @@ -691,7 +691,7 @@ void aStoredSpecOutsideItsLimitsStartsNoRun(String requestJson, String field) { // A run the check let through would fail here, at submission. Mockito.when(trogdorClient.createTask(any())).thenThrow(new IllegalStateException("no coordinator in tests")); - scheduler.executeSchedule(storedSchedule(requestJson)); + scheduler.executeSchedule(storedSchedule(requestJson), java.time.Instant.now()); Mockito.verify(repository, Mockito.never()).updateLastRun(anyString(), anyString()); Mockito.verifyNoInteractions(trogdorClient); @@ -729,7 +729,7 @@ static Stream specsOutsideTheLimits() { void aStoredRequestWithinTheLimitsStillRuns(String name, String requestJson) { Mockito.when(trogdorClient.createTask(any())).thenThrow(new IllegalStateException("no coordinator in tests")); - scheduler.executeSchedule(storedSchedule(requestJson)); + scheduler.executeSchedule(storedSchedule(requestJson), java.time.Instant.now()); ArgumentCaptor runId = ArgumentCaptor.forClass(String.class); Mockito.verify(repository).updateLastRun(eq("s1"), runId.capture()); diff --git a/kates/src/test/java/com/bmscomp/kates/schedule/TestSchedulerFiringTest.java b/kates/src/test/java/com/bmscomp/kates/schedule/TestSchedulerFiringTest.java new file mode 100644 index 000000000..4907b8097 --- /dev/null +++ b/kates/src/test/java/com/bmscomp/kates/schedule/TestSchedulerFiringTest.java @@ -0,0 +1,153 @@ +package com.bmscomp.kates.schedule; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.*; + +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.UUID; +import jakarta.inject.Inject; + +import io.quarkus.test.InjectMock; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.TestProfile; +import io.restassured.RestAssured; +import io.restassured.http.ContentType; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import com.bmscomp.kates.SchedulerOffTestProfile; +import com.bmscomp.kates.domain.TestRun; +import com.bmscomp.kates.domain.TestType; +import com.bmscomp.kates.engine.ConcurrencyLimitException; +import com.bmscomp.kates.engine.TestOrchestrator; +import com.bmscomp.kates.util.Result; + +/** + * Each due time of a schedule starts one run, however many ticks see it, and + * a due time a late tick missed still fires. Every job is off, so only the + * test's own ticks fire anything. + */ +@QuarkusTest +@TestProfile(SchedulerOffTestProfile.class) +class TestSchedulerFiringTest { + + private static final Instant T0 = Instant.parse("2026-11-15T10:36:30Z"); + + @InjectMock + TestOrchestrator orchestrator; + + @Inject + TestScheduler scheduler; + + @Inject + ScheduledTestRunRepository repository; + + private String id; + + @BeforeEach + void runsStart() { + id = UUID.randomUUID().toString().substring(0, 8); + when(orchestrator.executeTest(any(), anyString())) + .thenReturn(Result.success(new TestRun(TestType.LOAD, null).withId("run-" + id))); + } + + @AfterEach + void deleteSchedule() { + repository.delete(id); + } + + @Test + void aDueTimeStartsOneRunAcrossTicks() { + save(true, T0); + + scheduler.evaluate(T0.plusSeconds(35)); + scheduler.evaluate(T0.plusSeconds(45)); + + verify(orchestrator, times(1)).executeTest(any(), eq(key("2026-11-15T10:37:00Z"))); + verify(orchestrator, times(1)).executeTest(any(), anyString()); + } + + @Test + void aRefusedStartStillCountsAsTheFiring() { + when(orchestrator.executeTest(any(), anyString())).thenReturn(Result.failure(new ConcurrencyLimitException(1))); + save(true, T0); + + scheduler.evaluate(T0.plusSeconds(35)); + scheduler.evaluate(T0.plusSeconds(45)); + + verify(orchestrator, times(1)).executeTest(any(), anyString()); + } + + /** A takeover that left two minutes without a tick still fires, once, the latest due time. */ + @Test + void aMissedMinuteFiresAfterATakeover() { + save(true, T0); + + scheduler.evaluate(T0.plusSeconds(35)); + scheduler.evaluate(T0.plusSeconds(155)); + + verify(orchestrator).executeTest(any(), eq(key("2026-11-15T10:37:00Z"))); + verify(orchestrator).executeTest(any(), eq(key("2026-11-15T10:39:00Z"))); + verify(orchestrator, times(2)).executeTest(any(), anyString()); + } + + @Test + void aReEnabledScheduleWaitsForItsNextDueTime() { + Instant now = Instant.now(); + save(false, now.minus(Duration.ofMinutes(10))); + + RestAssured.given() + .contentType(ContentType.JSON) + .body("{\"enabled\":true}") + .when() + .put("/api/schedules/" + id) + .then() + .statusCode(200); + + scheduler.evaluate(Instant.now()); + verify(orchestrator, never()).executeTest(any(), anyString()); + + Instant next = Instant.now().plusSeconds(60); + scheduler.evaluate(next); + verify(orchestrator, times(1)).executeTest(any(), anyString()); + verify(orchestrator) + .executeTest(any(), eq(key(next.truncatedTo(ChronoUnit.MINUTES).toString()))); + } + + /** A PUT that leaves an enabled schedule enabled, as a rename does, keeps its pending due time. */ + @Test + void aPutThatKeepsAScheduleOnKeepsItsDueTime() { + save(true, Instant.now().minus(Duration.ofMinutes(10))); + + RestAssured.given() + .contentType(ContentType.JSON) + .body("{\"enabled\":true,\"name\":\"renamed-" + id + "\"}") + .when() + .put("/api/schedules/" + id) + .then() + .statusCode(200); + + scheduler.evaluate(Instant.now()); + verify(orchestrator, times(1)).executeTest(any(), anyString()); + } + + private void save(boolean enabled, Instant createdAt) { + ScheduledTestRun schedule = new ScheduledTestRun(); + schedule.setId(id); + schedule.setName("every-minute-" + id); + schedule.setCronExpression("* * * * *"); + schedule.setEnabled(enabled); + schedule.setRequestJson("{\"type\":\"LOAD\",\"spec\":{\"numRecords\":1000}}"); + schedule.setCreatedAt(createdAt); + repository.save(schedule); + } + + private String key(String due) { + return "schedule:" + id + ":" + due; + } +} From fd8e0e13b9506b6b11cbe7e6aad10c0822ea6594 Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Sun, 15 Nov 2026 17:29:40 +0100 Subject: [PATCH 4/5] docs(api): each schedule due time fires once, up to 5 minutes late The Scheduling section says each due time of a test or disruption schedule fires once, on the leader, within 10 s or up to 5 minutes late after a restart, a takeover or a drain, that only the latest missed one starts a run, that a PUT turning a schedule on passes over earlier due times, and that a test run carries schedule:: as its Idempotency-Key. The drain note no longer says a due time inside a drain is skipped. Three MCP caveat refs follow the scheduler lines that moved. --- cli/cmd/mcp_tools_runs_activity.go | 6 +++--- docs/book/11-api-reference.md | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cli/cmd/mcp_tools_runs_activity.go b/cli/cmd/mcp_tools_runs_activity.go index 748ed5ac8..8b7326b91 100644 --- a/cli/cmd/mcp_tools_runs_activity.go +++ b/cli/cmd/mcp_tools_runs_activity.go @@ -465,8 +465,8 @@ var mcpCaveatsRunsActivity = []mcpCaveat{ mcpAnchoredRef(mcpJava+"security/GrpcApiKeyInterceptor.java:67-70,100,156-159", "static final Map AUDITED", "audit(action, method, Status.Code.PERMISSION_DENIED", "private void audit("), - mcpAnchoredRef(mcpJava+"schedule/TestScheduler.java:118-124", "Actor.SCHEDULER"), - mcpAnchoredRef(mcpJava+"disruption/DisruptionScheduler.java:124-130", "Actor.SCHEDULER"), + mcpAnchoredRef(mcpJava+"schedule/TestScheduler.java:117-123", "Actor.SCHEDULER"), + mcpAnchoredRef(mcpJava+"disruption/DisruptionScheduler.java:122-128", "Actor.SCHEDULER"), mcpAnchoredRef(mcpJava+"disruption/DisruptionResource.java:93", "auditTrail.done();"), "kates/src/main/resources/db/migration/V26__audit_actor.sql:1-8", }, @@ -488,7 +488,7 @@ var mcpCaveatsRunsActivity = []mcpCaveat{ mcpJava + "disruption/DisruptionReportRepository.java:20-43", mcpJava + "disruption/DisruptionOrphanReconciler.java:334-368", mcpJava + "disruption/DisruptionTemplateResource.java:70-72", - mcpJava + "disruption/DisruptionScheduler.java:112-124", + mcpJava + "disruption/DisruptionScheduler.java:110-122", mcpJava + "disruption/DisruptionReportEntity.java:39-47", mcpAnchoredRef(mcpJava+"disruption/DisruptionAnalysisResource.java:118-203", `@Path("/compound")`, `"results", outcome.results()`), diff --git a/docs/book/11-api-reference.md b/docs/book/11-api-reference.md index c25c122c4..39c88ced5 100644 --- a/docs/book/11-api-reference.md +++ b/docs/book/11-api-reference.md @@ -273,7 +273,7 @@ curl -s -X PUT -H "X-API-Key: $KATES_API_KEY" -H "Content-Type: application/json While the API is draining: - Every request that would start a test run, a disruption, a compound run or a resilience test is answered `503 Service Unavailable` with a `Retry-After: 60` header. Two answers differ: a full engine still answers `429`, and `POST /api/tests/bulk` answers `202` with every item failed. -- Test and disruption schedules don't fire. A firing that falls inside the drain is skipped, not run later. +- Test and disruption schedules don't fire. A due time inside the drain fires when the drain ends, if that is within 5 minutes of it, and is skipped otherwise. - Live runs and faults go on to their end. A resilience test admitted before the drain still starts its benchmark when it reaches it. A dry run, a cancel and every read are answered as usual. The switch is kept in memory: a restarted pod is not draining. A body without `draining` is a `400`. @@ -1379,6 +1379,8 @@ It takes these query parameters: A schedule stores a test request with a cron expression, and while the schedule is enabled the Kates API submits that request as a new run each time the expression fires. `kates schedule` calls every one of these endpoints except `PUT`. +Each due time of a test or disruption schedule fires once, on the member that leads, within 10 seconds of it, or up to 5 minutes late after a restart, a takeover or a drain. When several were missed, only the latest starts a run. A `PUT` that turns a schedule on or changes its cron passes over the due times before it. A test run a schedule starts carries the Idempotency-Key `schedule::`, such as `schedule:b5a9a9bb:2026-11-15T10:37:00Z`, so `GET /api/tests?clientKey=` finds it by that key. + #### POST /api/schedules Create a recurring test schedule. Cron expressions use the standard 5-field Unix format (`minute hour day-of-month month day-of-week`, evaluated in UTC), at most 64 characters: `*`, numbers, ranges (`9-17`), lists (`0,30`) and steps (`*/15`, `0-30/5`), with month and day names such as `JAN` or `MON-FRI`. Day-of-week `0` and `7` are both Sunday, and when both day fields are set the schedule fires on a day that matches either one. Macros such as `@daily`, six-field expressions with seconds, and Quartz's `?` and `L` are refused with a `400` keyed `cronExpression` in `fieldErrors`, and the schedule isn't saved. From cf877e558b46b99fef0371ededaefefce3dbe314 Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Sun, 15 Nov 2026 17:48:47 +0100 Subject: [PATCH 5/5] =?UTF-8?q?fix(schedule):=20review=20fixes=20=E2=80=94?= =?UTF-8?q?=20same-minute=20disruption=20schedules,=20the=20PUT=20race,=20?= =?UTF-8?q?and=20upgrades?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The disruption evaluator ticks with SKIP. A tick runs each plan it fires to its end; a tick started meanwhile fired a schedule due the same minute into the target the first plan held, and spent its due time on a REJECTED report. - A test-schedule PUT that turns a schedule on or changes its cron passes over the earlier due times before it saves, so a tick between the two no longer fires one of them. - Due times are counted after the latest of created_at, last_due and last_run_at. A row from before V31 has no last_due, so the first tick after an upgrade re-fired the due time the previous version had just fired, up to 5 minutes back; last_run_at also covers firings by the previous version during a rolling upgrade. --- cli/cmd/mcp_tools_runs_activity.go | 4 +- .../kates/disruption/DisruptionScheduler.java | 15 +++++-- .../com/bmscomp/kates/schedule/DueTimes.java | 16 ++++++++ .../kates/schedule/ScheduleResource.java | 5 ++- .../bmscomp/kates/schedule/TestScheduler.java | 2 +- .../DisruptionSchedulerTickTest.java | 21 ++++++++++ .../bmscomp/kates/schedule/DueTimesTest.java | 10 +++++ .../schedule/TestSchedulerFiringTest.java | 40 ++++++++++++++++++- 8 files changed, 103 insertions(+), 10 deletions(-) create mode 100644 kates/src/test/java/com/bmscomp/kates/disruption/DisruptionSchedulerTickTest.java diff --git a/cli/cmd/mcp_tools_runs_activity.go b/cli/cmd/mcp_tools_runs_activity.go index 8b7326b91..350896e01 100644 --- a/cli/cmd/mcp_tools_runs_activity.go +++ b/cli/cmd/mcp_tools_runs_activity.go @@ -466,7 +466,7 @@ var mcpCaveatsRunsActivity = []mcpCaveat{ "static final Map AUDITED", "audit(action, method, Status.Code.PERMISSION_DENIED", "private void audit("), mcpAnchoredRef(mcpJava+"schedule/TestScheduler.java:117-123", "Actor.SCHEDULER"), - mcpAnchoredRef(mcpJava+"disruption/DisruptionScheduler.java:122-128", "Actor.SCHEDULER"), + mcpAnchoredRef(mcpJava+"disruption/DisruptionScheduler.java:129-135", "Actor.SCHEDULER"), mcpAnchoredRef(mcpJava+"disruption/DisruptionResource.java:93", "auditTrail.done();"), "kates/src/main/resources/db/migration/V26__audit_actor.sql:1-8", }, @@ -488,7 +488,7 @@ var mcpCaveatsRunsActivity = []mcpCaveat{ mcpJava + "disruption/DisruptionReportRepository.java:20-43", mcpJava + "disruption/DisruptionOrphanReconciler.java:334-368", mcpJava + "disruption/DisruptionTemplateResource.java:70-72", - mcpJava + "disruption/DisruptionScheduler.java:110-122", + mcpJava + "disruption/DisruptionScheduler.java:117-129", mcpJava + "disruption/DisruptionReportEntity.java:39-47", mcpAnchoredRef(mcpJava+"disruption/DisruptionAnalysisResource.java:118-203", `@Path("/compound")`, `"results", outcome.results()`), diff --git a/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionScheduler.java b/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionScheduler.java index b1472d3a4..a3a2b6fb6 100644 --- a/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionScheduler.java +++ b/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionScheduler.java @@ -14,8 +14,8 @@ /** * Cron-based scheduler for recurring disruption tests. - * Evaluates enabled schedules every 60 seconds, executing matching plans - * via DisruptionOrchestrator or from the playbook catalog. + * Evaluates enabled schedules every 10 seconds, and runs the plan of each due + * time once, via DisruptionOrchestrator or from the playbook catalog. */ @ApplicationScoped public class DisruptionScheduler { @@ -49,7 +49,13 @@ public class DisruptionScheduler { @jakarta.inject.Inject com.bmscomp.kates.quorum.Quorum quorum; - @Scheduled(every = "10s", identity = "kates-disruption-schedule-evaluator") + // SKIP: a tick runs each plan it fires to its end. A tick started meanwhile would fire a + // schedule due the same minute into the target the first plan holds, and spend its due time + // on a REJECTED report; with SKIP, the running tick fires it after its plan, as one tick did. + @Scheduled( + every = "10s", + identity = "kates-disruption-schedule-evaluator", + concurrentExecution = Scheduled.ConcurrentExecution.SKIP) void evaluateSchedules() { evaluate(Instant.now()); } @@ -71,7 +77,8 @@ void evaluate(Instant now) { for (DisruptionScheduleEntity schedule : schedules) { try { - Instant since = schedule.getLastDue() != null ? schedule.getLastDue() : schedule.getCreatedAt(); + Instant since = com.bmscomp.kates.schedule.DueTimes.since( + schedule.getCreatedAt(), schedule.getLastDue(), schedule.getLastRunAt()); com.bmscomp.kates.schedule.DueTimes.toFire(schedule.getCronExpression(), since, now) .filter(due -> firings.claim( com.bmscomp.kates.schedule.ScheduleFirings.Kind.DISRUPTIONS, schedule.getId(), due)) diff --git a/kates/src/main/java/com/bmscomp/kates/schedule/DueTimes.java b/kates/src/main/java/com/bmscomp/kates/schedule/DueTimes.java index dcdfd3c38..392a3a773 100644 --- a/kates/src/main/java/com/bmscomp/kates/schedule/DueTimes.java +++ b/kates/src/main/java/com/bmscomp/kates/schedule/DueTimes.java @@ -21,6 +21,22 @@ public final class DueTimes { private DueTimes() {} + /** + * The instant a schedule's due times are counted after: the latest of its + * creation, its last claimed due time and its last run's start. The last + * run counts too, since a row from before V31 has no last_due, and a + * Kates API of the version before it still fires during a rolling upgrade. + */ + public static Instant since(Instant createdAt, Instant lastDue, Instant lastRunAt) { + Instant since = createdAt; + for (Instant later : new Instant[] {lastDue, lastRunAt}) { + if (later != null && later.isAfter(since)) { + since = later; + } + } + return since; + } + /** * The latest due time {@code d} of {@code cron} (UNIX, UTC) with * {@code since < d <= now} and {@code d >= now - CATCH_UP}, or empty. diff --git a/kates/src/main/java/com/bmscomp/kates/schedule/ScheduleResource.java b/kates/src/main/java/com/bmscomp/kates/schedule/ScheduleResource.java index 24fe3b0ed..fdf472f82 100644 --- a/kates/src/main/java/com/bmscomp/kates/schedule/ScheduleResource.java +++ b/kates/src/main/java/com/bmscomp/kates/schedule/ScheduleResource.java @@ -258,11 +258,12 @@ public Response updateSchedule( throw new RuntimeException(e); } } - repository.save(schedule); if (passOver) { - // Turning a schedule on or re-timing it passes over the due times before now. + // Turning a schedule on or re-timing it passes over the due times before now, + // before the save, so a tick between the two still sees it as it was. firings.claim(ScheduleFirings.Kind.TESTS, id, java.time.Instant.now()); } + repository.save(schedule); return Response.ok(schedule).build(); }) .orElseGet(() -> Response.status(404) diff --git a/kates/src/main/java/com/bmscomp/kates/schedule/TestScheduler.java b/kates/src/main/java/com/bmscomp/kates/schedule/TestScheduler.java index 0409f6b39..b435206ac 100644 --- a/kates/src/main/java/com/bmscomp/kates/schedule/TestScheduler.java +++ b/kates/src/main/java/com/bmscomp/kates/schedule/TestScheduler.java @@ -72,7 +72,7 @@ void evaluate(Instant now) { for (ScheduledTestRun schedule : schedules) { try { - Instant since = schedule.getLastDue() != null ? schedule.getLastDue() : schedule.getCreatedAt(); + Instant since = DueTimes.since(schedule.getCreatedAt(), schedule.getLastDue(), schedule.getLastRunAt()); DueTimes.toFire(schedule.getCronExpression(), since, now) .filter(due -> firings.claim(ScheduleFirings.Kind.TESTS, schedule.getId(), due)) .ifPresent(due -> { diff --git a/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionSchedulerTickTest.java b/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionSchedulerTickTest.java new file mode 100644 index 000000000..b552ea83f --- /dev/null +++ b/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionSchedulerTickTest.java @@ -0,0 +1,21 @@ +package com.bmscomp.kates.disruption; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import io.quarkus.scheduler.Scheduled; +import org.junit.jupiter.api.Test; + +class DisruptionSchedulerTickTest { + + /** + * A tick runs each plan it fires to its end. A second tick started while + * one plan runs would fire a schedule due the same minute into the target + * the first holds, and its report would be REJECTED, its due time spent. + */ + @Test + void aTickDoesNotStartWhileAnotherRunsAPlan() throws Exception { + Scheduled tick = + DisruptionScheduler.class.getDeclaredMethod("evaluateSchedules").getAnnotation(Scheduled.class); + assertEquals(Scheduled.ConcurrentExecution.SKIP, tick.concurrentExecution()); + } +} diff --git a/kates/src/test/java/com/bmscomp/kates/schedule/DueTimesTest.java b/kates/src/test/java/com/bmscomp/kates/schedule/DueTimesTest.java index 3f440af8a..6c42aead7 100644 --- a/kates/src/test/java/com/bmscomp/kates/schedule/DueTimesTest.java +++ b/kates/src/test/java/com/bmscomp/kates/schedule/DueTimesTest.java @@ -62,6 +62,16 @@ void aScheduleCreatedAfterADueTimeWaitsForTheNext() { assertEquals(Optional.of(at("10:37:00")), toFire("* * * * *", "10:36:30", "10:37:01")); } + /** A row from before V31 has no last_due; the run the previous version started still counts. */ + @Test + void sinceIsTheLatestOfCreatedLastDueAndLastRun() { + Instant created = at("09:00:00"); + assertEquals(created, DueTimes.since(created, null, null)); + assertEquals(at("10:35:02"), DueTimes.since(created, null, at("10:35:02"))); + assertEquals(at("10:40:00"), DueTimes.since(created, at("10:40:00"), at("10:35:02"))); + assertEquals(at("10:40:03"), DueTimes.since(created, at("10:40:00"), at("10:40:03"))); + } + // ------------------------------------------------ cron semantics, from the old matcher's tests /** Whether {@code at}'s minute is a due time: it fires with since one minute before it. */ diff --git a/kates/src/test/java/com/bmscomp/kates/schedule/TestSchedulerFiringTest.java b/kates/src/test/java/com/bmscomp/kates/schedule/TestSchedulerFiringTest.java index 4907b8097..54bf84777 100644 --- a/kates/src/test/java/com/bmscomp/kates/schedule/TestSchedulerFiringTest.java +++ b/kates/src/test/java/com/bmscomp/kates/schedule/TestSchedulerFiringTest.java @@ -14,6 +14,7 @@ import io.quarkus.test.InjectMock; import io.quarkus.test.junit.QuarkusTest; import io.quarkus.test.junit.TestProfile; +import io.quarkus.test.junit.mockito.InjectSpy; import io.restassured.RestAssured; import io.restassured.http.ContentType; import org.junit.jupiter.api.AfterEach; @@ -44,7 +45,7 @@ class TestSchedulerFiringTest { @Inject TestScheduler scheduler; - @Inject + @InjectSpy ScheduledTestRunRepository repository; private String id; @@ -136,6 +137,43 @@ void aPutThatKeepsAScheduleOnKeepsItsDueTime() { verify(orchestrator, times(1)).executeTest(any(), anyString()); } + /** The version before last_due fired 10:37 and wrote lastRunAt; the first tick after the upgrade does not fire it again. */ + @Test + void aDueTimeThePreviousVersionFiredDoesNotFireAgain() { + save(true, T0); + ScheduledTestRun fired = repository.findById(id).orElseThrow(); + fired.setLastRunAt(Instant.parse("2026-11-15T10:37:01Z")); + repository.save(fired); + + scheduler.evaluate(T0.plusSeconds(45)); + + verify(orchestrator, never()).executeTest(any(), anyString()); + } + + /** A tick that lands between the PUT's save and its pass-over fires nothing from before the PUT. */ + @Test + void aTickDuringTheReEnablingPutFiresNothing() { + Instant now = Instant.now(); + save(false, now.minus(Duration.ofMinutes(10))); + doAnswer(saving -> { + saving.callRealMethod(); + scheduler.evaluate(Instant.now()); + return null; + }) + .when(repository) + .save(any()); + + RestAssured.given() + .contentType(ContentType.JSON) + .body("{\"enabled\":true}") + .when() + .put("/api/schedules/" + id) + .then() + .statusCode(200); + + verify(orchestrator, never()).executeTest(any(), anyString()); + } + private void save(boolean enabled, Instant createdAt) { ScheduledTestRun schedule = new ScheduledTestRun(); schedule.setId(id);