From ff4ba06ab8bca497500d3a5e193a5c62234da772 Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Fri, 20 Nov 2026 11:36:33 -0800 Subject: [PATCH] fix(workflow-operator): correct reservoir sampling replacement odds ReservoirSamplingOpExec draws the replacement index with Random.nextInt(n+1) instead of nextInt(n), where n is the number of tuples seen before the current one. Algorithm R replaces a reservoir slot for the (n+1)-th item with probability count/(n+1); the old nextInt(n) made the first item after the reservoir fills always replace (probability 1.0), biasing the sample toward later rows. A zero-length reservoir (k coerced to 0) now draws nextInt(1) and keeps nothing instead of throwing on nextInt(0). ReservoirSamplingOpExecSpec adds a test that reproduces Algorithm R with the executor's seed and asserts the sampled reservoir matches it, pinning the off-by-one, and updates the k=0 case to expect an empty result rather than an IllegalArgumentException. --- .../ReservoirSamplingOpExec.scala | 3 +- .../ReservoirSamplingOpExecSpec.scala | 36 +++++++++++++++---- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpExec.scala index 4c802bdb9f4..58749fc0674 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpExec.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpExec.scala @@ -49,7 +49,8 @@ class ReservoirSamplingOpExec(descString: String, idx: Int, workerCount: Int) if (n < count) { reservoir(n) = tuple } else { - val i = rand.nextInt(n) + // Algorithm R: the (n+1)-th item (n seen so far) replaces a slot with probability count/(n+1). + val i = rand.nextInt(n + 1) if (i < count) { reservoir(i) = tuple } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpExecSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpExecSpec.scala index d6a9bcd8d51..9a66ee2064c 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpExecSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpExecSpec.scala @@ -163,6 +163,28 @@ class ReservoirSamplingOpExecSpec extends AnyFlatSpec { assert(firstRun != (0 until 7).map(tuple).toList) } + it should "follow Algorithm R replacement odds (draws nextInt(n+1), not nextInt(n))" in { + // Reference Algorithm R using the same seed the exec uses (workerCount); the exec's + // reservoir must match it exactly. A reversion to nextInt(n) draws a different sequence + // and diverges, so this pins the off-by-one fix. + val k = 5 + val workerCount = 1 + val input = (0 until 100).map(tuple).toVector + val rng = new scala.util.Random(workerCount) + val expected = Array.ofDim[Tuple](k) + var seen = 0 + input.foreach { t => + if (seen < k) expected(seen) = t + else { + val j = rng.nextInt(seen + 1) + if (j < k) expected(j) = t + } + seen += 1 + } + val emitted = runFinishTuples(newExec(k = k, workerCount = workerCount), input) + assert(emitted == expected.toList) + } + it should "distribute k across workers via equallyPartitionGoal (k=10, 3 workers -> 4,3,3)" in { // The remainder is handed to the lowest-indexed workers, so worker 0 keeps one extra. val perWorkerSize = (0 until 3).map { idx => @@ -225,14 +247,14 @@ class ReservoirSamplingOpExecSpec extends AnyFlatSpec { } } - it should "coerce a null k to 0 and reject sampling a populated stream" in { - // A null k deserializes to the primitive default 0, yielding a zero-length - // reservoir; the first replacement draw then calls Random.nextInt(0). + it should "coerce a null k to 0 and sample nothing from a populated stream" in { + // A null k deserializes to the primitive default 0, yielding a zero-length reservoir; + // with the Algorithm R bound nextInt(n+1) the replacement draw is always valid, so + // processing keeps nothing instead of throwing on nextInt(0). val exec = new ReservoirSamplingOpExec(descWithNullK, 0, 1) exec.open() - assert(exec.onFinish(0).isEmpty, "an empty reservoir emits nothing") - assertThrows[IllegalArgumentException] { - exec.processTuple(tuple(0), 0) - } + exec.processTuple(tuple(0), 0) + exec.processTuple(tuple(1), 0) + assert(exec.onFinish(0).isEmpty, "a zero-length reservoir emits nothing") } }