Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
package org.apache.texera.amber.operator.reservoirsampling

import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription}
import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaInject
import org.apache.texera.amber.core.executor.OpExecWithClassName
import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity}
import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PhysicalOp}
Expand All @@ -29,8 +30,12 @@ import org.apache.texera.amber.util.JSONUtils.objectMapper

class ReservoirSamplingOpDesc extends LogicalOp with StandaloneCodeGenerator {

// A sample size below 1 is never valid. The bound reaches the property editor only: a k
// from the API, an imported workflow or a loop variable is refused by the executor at
// its first row instead.
@JsonProperty(value = "number of item sampled in reservoir sampling", required = true)
@JsonPropertyDescription("reservoir sampling with k items being kept randomly")
@JsonSchemaInject(json = """{"minimum": 1}""")
var k: Int = _

override def getPhysicalOp(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@

package org.apache.texera.amber.operator.reservoirsampling

import com.fasterxml.jackson.databind.node.IntNode
import com.github.fge.jsonschema.main.JsonSchemaFactory
import org.apache.texera.amber.util.PythonTestSupport.{
NeedsPythonPackages,
canImport,
Expand All @@ -28,7 +30,7 @@ import org.apache.texera.amber.util.PythonTestSupport.{
import org.apache.texera.amber.core.executor.OpExecWithClassName
import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity}
import org.apache.texera.amber.operator.{LogicalOp, SamplingHelpers}
import org.apache.texera.amber.operator.metadata.OperatorGroupConstants
import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorMetadataGenerator}
import org.apache.texera.amber.util.JSONUtils.objectMapper
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers
Expand Down Expand Up @@ -56,6 +58,27 @@ class ReservoirSamplingOpDescSpec extends AnyFlatSpec with Matchers {
info.supportReconfiguration shouldBe false
}

// A sample size below 1 is never valid. Validate through the schema the property editor
// validates against, rather than restating the bound it declares.
private def schemaTakesK(value: Int): Boolean =
JsonSchemaFactory
.byDefault()
.getJsonSchema(
OperatorMetadataGenerator
.generateOperatorJsonSchema(classOf[ReservoirSamplingOpDesc])
.path("properties")
.path(WireKey)
)
.validate(IntNode.valueOf(value))
.isSuccess

"ReservoirSamplingOpDesc" should "let the property editor refuse a sample size below 1" in {
schemaTakesK(-1) shouldBe false
schemaTakesK(0) shouldBe false
schemaTakesK(1) shouldBe true
schemaTakesK(5) shouldBe true
}

"ReservoirSamplingOpDesc" should "serialize k under its wire-key and round-trip it" in {
val d = new ReservoirSamplingOpDesc
d.k = 100
Expand Down