From 701cdff2e1902e3b73c748990ec6e5a5a28cf97d Mon Sep 17 00:00:00 2001 From: suryaacharan Date: Tue, 29 Sep 2026 21:42:53 -0700 Subject: [PATCH 1/3] feat(workflow-operator): make Python UDF templates work after uncommenting one class The Python UDF, 2-in Python UDF and 1-out Python UDF templates were fully commented out, including `from pytexera import *`, and did not say to keep a single class. Uncommenting a class then failed with NameError, and keeping several failed the one-Operator check. - Keep `from pytexera import *` active in all three templates - Add a header: keep exactly one class, the base class to method pairing, `yield` not `return`, open() and super().__init__() - 2-in template: say port 0 (model) is read before port 1 (tuples) - Keep the UiParameter guidance lines - Sync the template copies in the operator reference docs Saved workflows keep their own code and are unaffected. Refs #8736 Co-Authored-By: Claude Opus 5.5 --- .../managers/test_executor_manager.py | 55 +++++++++++++++++++ .../DualInputPortsPythonUDFOpDescV2.scala | 16 ++++-- .../udf/python/PythonUDFOpDescV2.scala | 13 +++-- .../source/PythonUDFSourceOpDescV2.scala | 14 +++-- .../DualInputPortsPythonUDFOpDescV2Spec.scala | 44 +++++++++++++++ .../udf/python/PythonUDFOpDescV2Spec.scala | 43 +++++++++++++++ .../source/PythonUDFSourceOpDescV2Spec.scala | 39 +++++++++++++ .../python/1-out-python-udf.md | 13 ++++- .../python/2-in-python-udf.md | 19 +++++-- .../python/python-udf.md | 16 ++++-- 10 files changed, 247 insertions(+), 25 deletions(-) diff --git a/amber/src/test/python/core/architecture/managers/test_executor_manager.py b/amber/src/test/python/core/architecture/managers/test_executor_manager.py index 07ac054c062..7885af99832 100644 --- a/amber/src/test/python/core/architecture/managers/test_executor_manager.py +++ b/amber/src/test/python/core/architecture/managers/test_executor_manager.py @@ -320,6 +320,61 @@ def test_close_is_noop_when_tmp_dir_never_materialized(self): assert "tmp_dir" not in manager.__dict__ +# Shape of the default Python UDF template (PythonUDFOpDescV2.defaultValue): +# the import is active and every class is commented out. +DEFAULT_TEMPLATE_CODE = """# Keep exactly ONE class below: uncomment it and delete the others. + +from pytexera import * + +# class ProcessTupleOperator(UDFOperatorV2): +# +# @overrides +# def process_tuple(self, tuple_: Tuple, port: int) -> Iterator[Optional[TupleLike]]: +# yield tuple_ +# +# class ProcessTableOperator(UDFTableOperator): +# +# @overrides +# def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: +# yield table +""" + + +def uncomment_class(code: str, class_name: str) -> str: + """Uncomment one `# class ` block of a template, as a user would.""" + lines, in_block = [], False + for line in code.splitlines(): + if line.startswith("# class "): + in_block = line.startswith(f"# class {class_name}") + if in_block and line.startswith("#"): + line = line[1:].removeprefix(" ") + lines.append(line) + return "\n".join(lines) + "\n" + + +class TestDefaultTemplate: + @pytest.fixture + def executor_manager(self): + manager = ExecutorManager() + yield manager + manager.close() + + def test_untouched_template_reports_no_operator(self, executor_manager): + with pytest.raises(AssertionError) as exc_info: + executor_manager.load_executor_definition(DEFAULT_TEMPLATE_CODE) + assert "one and only one Operator" in str(exc_info.value) + + @pytest.mark.parametrize( + "class_name", ["ProcessTupleOperator", "ProcessTableOperator"] + ) + def test_uncommenting_one_class_loads_without_adding_an_import( + self, executor_manager, class_name + ): + code = uncomment_class(DEFAULT_TEMPLATE_CODE, class_name) + executor = executor_manager.load_executor_definition(code) + assert executor.__name__ == class_name + + REPLACEMENT_OPERATOR_CODE = """ from pytexera import * diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/python/DualInputPortsPythonUDFOpDescV2.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/python/DualInputPortsPythonUDFOpDescV2.scala index 14a35faabec..65475d8169b 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/python/DualInputPortsPythonUDFOpDescV2.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/python/DualInputPortsPythonUDFOpDescV2.scala @@ -33,15 +33,23 @@ class DualInputPortsPythonUDFOpDescV2 extends LogicalOp with PythonUdfUiParamete @JsonProperty( required = true, defaultValue = - "# Choose from the following templates:\n" + + "# Keep exactly ONE class below: uncomment it and delete the others.\n" + + "# Its method must match its base class:\n" + + "# UDFOperatorV2 -> process_tuple, UDFBatchOperator -> process_batch,\n" + + "# UDFTableOperator -> process_table\n" + + "# Always `yield` results (never `return`). Put setup code in open();\n" + + "# if you override __init__, call super().__init__() first.\n" + + "# \n" + + "# Port 0 (model) is read completely before port 1 (tuples) starts.\n" + + "# Use the `port` argument to tell them apart, e.g. `if port == 0:`.\n" + "# \n" + "# Define UiParameter inside open() of ProcessTupleOperator, ProcessBatchOperator, or ProcessTableOperator.\n" + "# Example: self.count = self.UiParameter(\"count\", AttributeType.INT).value\n" + "# Add value=Resource.MODEL or Resource.DATASET to pick a version; the value is its mount directory.\n" + "# See the Python UDF operator documentation for supported types and behavior.\n" + - "# \n" + - "# from pytexera import *\n" + - "# \n" + + "\n" + + "from pytexera import *\n" + + "\n" + "# class ProcessTupleOperator(UDFOperatorV2):\n" + "# \n" + "# @overrides\n" + diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/python/PythonUDFOpDescV2.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/python/PythonUDFOpDescV2.scala index 0239ca9145f..e0e0e2a11d8 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/python/PythonUDFOpDescV2.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/python/PythonUDFOpDescV2.scala @@ -35,15 +35,20 @@ class PythonUDFOpDescV2 extends LogicalOp with PythonUdfUiParameterSupport { @JsonProperty( required = true, defaultValue = - "# Choose from the following templates:\n" + + "# Keep exactly ONE class below: uncomment it and delete the others.\n" + + "# Its method must match its base class:\n" + + "# UDFOperatorV2 -> process_tuple, UDFBatchOperator -> process_batch,\n" + + "# UDFTableOperator -> process_table\n" + + "# Always `yield` results (never `return`). Put setup code in open();\n" + + "# if you override __init__, call super().__init__() first.\n" + "# \n" + "# Define UiParameter inside open() of ProcessTupleOperator, ProcessBatchOperator, or ProcessTableOperator.\n" + "# Example: self.count = self.UiParameter(\"count\", AttributeType.INT).value\n" + "# Add value=Resource.MODEL or Resource.DATASET to pick a version; the value is its mount directory.\n" + "# See the Python UDF operator documentation for supported types and behavior.\n" + - "# \n" + - "# from pytexera import *\n" + - "# \n" + + "\n" + + "from pytexera import *\n" + + "\n" + "# class ProcessTupleOperator(UDFOperatorV2):\n" + "# \n" + "# @overrides\n" + diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/python/source/PythonUDFSourceOpDescV2.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/python/source/PythonUDFSourceOpDescV2.scala index 5932c6643cb..202b14832b6 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/python/source/PythonUDFSourceOpDescV2.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/python/source/PythonUDFSourceOpDescV2.scala @@ -33,16 +33,22 @@ class PythonUDFSourceOpDescV2 extends SourceOperatorDescriptor with PythonUdfUiP @JsonProperty( required = true, - defaultValue = "# Define UiParameter inside GenerateOperator.open().\n" + + defaultValue = "# Uncomment the class below. Its method must match its base class:\n" + + "# UDFSourceOperator -> produce\n" + + "# Always `yield` results (never `return`). Put setup code in open();\n" + + "# if you override __init__, call super().__init__() first.\n" + + "# Keep exactly ONE class in this script.\n" + + "# \n" + + "# Define UiParameter inside GenerateOperator.open().\n" + "# Example: self.count = self.UiParameter(\"count\", AttributeType.INT).value\n" + "# Add value=Resource.MODEL or Resource.DATASET to pick a version; the value is its mount directory.\n" + "# See the Python UDF operator documentation for supported types and behavior.\n" + - "# \n" + - "# from pytexera import *\n" + + "\n" + + "from pytexera import *\n" + + "\n" + "# class GenerateOperator(UDFSourceOperator):\n" + "# \n" + "# @overrides\n" + - "# \n" + "# def produce(self) -> Iterator[Union[TupleLike, TableLike, None]]:\n" + "# yield\n" ) diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/DualInputPortsPythonUDFOpDescV2Spec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/DualInputPortsPythonUDFOpDescV2Spec.scala index 3b4f6c3207b..33078ea9a90 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/DualInputPortsPythonUDFOpDescV2Spec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/DualInputPortsPythonUDFOpDescV2Spec.scala @@ -26,6 +26,7 @@ import org.apache.texera.amber.core.workflow.{PortIdentity, UnknownPartition} import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants import org.apache.texera.amber.util.JSONUtils.objectMapper +import com.fasterxml.jackson.annotation.JsonProperty import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -209,4 +210,47 @@ class DualInputPortsPythonUDFOpDescV2Spec extends AnyFlatSpec with Matchers { r.uiParameters.map(_.attribute) shouldBe List(new Attribute("count", AttributeType.INTEGER)) r.uiParameters.map(_.value) shouldBe List("7") } + + private def defaultCode: String = + classOf[DualInputPortsPythonUDFOpDescV2] + .getDeclaredField("code") + .getAnnotation(classOf[JsonProperty]) + .defaultValue() + + private def activeLines(code: String): List[String] = + code.linesIterator.map(_.trim).filterNot(l => l.isEmpty || l.startsWith("#")).toList + + // Uncomment the template lines from `# class ` up to the next commented class. + private def uncommentClass(code: String, className: String): String = { + var inBlock = false + code.linesIterator + .map { line => + if (line.startsWith("# class ")) inBlock = line.startsWith(s"# class $className") + if (inBlock && line.startsWith("#")) line.stripPrefix("#").stripPrefix(" ") else line + } + .mkString("\n") + } + + "DualInputPortsPythonUDFOpDescV2 default code" should "keep the pytexera import active and explain the rules" in { + activeLines(defaultCode) shouldBe List("from pytexera import *") + defaultCode should include("Keep exactly ONE class") + defaultCode should include("never `return`") + defaultCode should include("super().__init__()") + defaultCode should include("UDFOperatorV2 -> process_tuple") + defaultCode should include("UDFBatchOperator -> process_batch") + defaultCode should include("UDFTableOperator -> process_table") + defaultCode should include("Port 0 (model) is read completely before port 1 (tuples)") + defaultCode should include("UiParameter") + } + + it should "leave exactly one active class after uncommenting any one template class" in { + for ( + className <- List("ProcessTupleOperator", "ProcessBatchOperator", "ProcessTableOperator") + ) { + val active = activeLines(uncommentClass(defaultCode, className)) + active.head shouldBe "from pytexera import *" + active.count(_.startsWith("class ")) shouldBe 1 + active.exists(_.startsWith(s"class $className(")) shouldBe true + } + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/PythonUDFOpDescV2Spec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/PythonUDFOpDescV2Spec.scala index bc0f22ff1cb..011ea6d713f 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/PythonUDFOpDescV2Spec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/PythonUDFOpDescV2Spec.scala @@ -26,6 +26,7 @@ import org.apache.texera.amber.core.workflow.{PortIdentity, UnknownPartition} import org.apache.texera.amber.operator.{LogicalOp, PortDescription} import org.apache.texera.amber.operator.metadata.OperatorGroupConstants import org.apache.texera.amber.util.JSONUtils.objectMapper +import com.fasterxml.jackson.annotation.JsonProperty import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -304,4 +305,46 @@ class PythonUDFOpDescV2Spec extends AnyFlatSpec with Matchers { case other => fail(s"expected OpExecWithCode, got $other") } } + + private def defaultCode: String = + classOf[PythonUDFOpDescV2] + .getDeclaredField("code") + .getAnnotation(classOf[JsonProperty]) + .defaultValue() + + private def activeLines(code: String): List[String] = + code.linesIterator.map(_.trim).filterNot(l => l.isEmpty || l.startsWith("#")).toList + + // Uncomment the template lines from `# class ` up to the next commented class. + private def uncommentClass(code: String, className: String): String = { + var inBlock = false + code.linesIterator + .map { line => + if (line.startsWith("# class ")) inBlock = line.startsWith(s"# class $className") + if (inBlock && line.startsWith("#")) line.stripPrefix("#").stripPrefix(" ") else line + } + .mkString("\n") + } + + "PythonUDFOpDescV2 default code" should "keep the pytexera import active and explain the rules" in { + activeLines(defaultCode) shouldBe List("from pytexera import *") + defaultCode should include("Keep exactly ONE class") + defaultCode should include("never `return`") + defaultCode should include("super().__init__()") + defaultCode should include("UDFOperatorV2 -> process_tuple") + defaultCode should include("UDFBatchOperator -> process_batch") + defaultCode should include("UDFTableOperator -> process_table") + defaultCode should include("UiParameter") + } + + it should "leave exactly one active class after uncommenting any one template class" in { + for ( + className <- List("ProcessTupleOperator", "ProcessBatchOperator", "ProcessTableOperator") + ) { + val active = activeLines(uncommentClass(defaultCode, className)) + active.head shouldBe "from pytexera import *" + active.count(_.startsWith("class ")) shouldBe 1 + active.exists(_.startsWith(s"class $className(")) shouldBe true + } + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/source/PythonUDFSourceOpDescV2Spec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/source/PythonUDFSourceOpDescV2Spec.scala index dcb648e2822..e25fcdb43d3 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/source/PythonUDFSourceOpDescV2Spec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/source/PythonUDFSourceOpDescV2Spec.scala @@ -26,6 +26,7 @@ import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants import org.apache.texera.amber.operator.udf.python.UiUDFParameter import org.apache.texera.amber.util.JSONUtils.objectMapper +import com.fasterxml.jackson.annotation.JsonProperty import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -159,4 +160,42 @@ class PythonUDFSourceOpDescV2Spec extends AnyFlatSpec with Matchers { p.uiParameters.map(_.attribute) shouldBe List(new Attribute("count", AttributeType.INTEGER)) p.uiParameters.map(_.value) shouldBe List("7") } + + private def defaultCode: String = + classOf[PythonUDFSourceOpDescV2] + .getDeclaredField("code") + .getAnnotation(classOf[JsonProperty]) + .defaultValue() + + private def activeLines(code: String): List[String] = + code.linesIterator.map(_.trim).filterNot(l => l.isEmpty || l.startsWith("#")).toList + + // Uncomment the template lines from `# class ` up to the next commented class. + private def uncommentClass(code: String, className: String): String = { + var inBlock = false + code.linesIterator + .map { line => + if (line.startsWith("# class ")) inBlock = line.startsWith(s"# class $className") + if (inBlock && line.startsWith("#")) line.stripPrefix("#").stripPrefix(" ") else line + } + .mkString("\n") + } + + "PythonUDFSourceOpDescV2 default code" should "keep the pytexera import active and explain the rules" in { + activeLines(defaultCode) shouldBe List("from pytexera import *") + defaultCode should include("Keep exactly ONE class") + defaultCode should include("never `return`") + defaultCode should include("super().__init__()") + defaultCode should include("UDFSourceOperator -> produce") + defaultCode should include("UiParameter") + } + + it should "leave exactly one active class after uncommenting any one template class" in { + for (className <- List("GenerateOperator")) { + val active = activeLines(uncommentClass(defaultCode, className)) + active.head shouldBe "from pytexera import *" + active.count(_.startsWith("class ")) shouldBe 1 + active.exists(_.startsWith(s"class $className(")) shouldBe true + } + } } diff --git a/docs/reference/operators/user-defined-functions/python/1-out-python-udf.md b/docs/reference/operators/user-defined-functions/python/1-out-python-udf.md index 69f68d25d8b..1196bb2f8c6 100644 --- a/docs/reference/operators/user-defined-functions/python/1-out-python-udf.md +++ b/docs/reference/operators/user-defined-functions/python/1-out-python-udf.md @@ -43,15 +43,22 @@ tags: [user-defined-functions, python] **Python script** ```python +# Uncomment the class below. Its method must match its base class: +# UDFSourceOperator -> produce +# Always `yield` results (never `return`). Put setup code in open(); +# if you override __init__, call super().__init__() first. +# Keep exactly ONE class in this script. +# # Define UiParameter inside GenerateOperator.open(). # Example: self.count = self.UiParameter("count", AttributeType.INT).value +# Add value=Resource.MODEL or Resource.DATASET to pick a version; the value is its mount directory. # See the Python UDF operator documentation for supported types and behavior. -# -# from pytexera import * + +from pytexera import * + # class GenerateOperator(UDFSourceOperator): # # @overrides -# # def produce(self) -> Iterator[Union[TupleLike, TableLike, None]]: # yield diff --git a/docs/reference/operators/user-defined-functions/python/2-in-python-udf.md b/docs/reference/operators/user-defined-functions/python/2-in-python-udf.md index 5c0c398a629..ceafa259e54 100644 --- a/docs/reference/operators/user-defined-functions/python/2-in-python-udf.md +++ b/docs/reference/operators/user-defined-functions/python/2-in-python-udf.md @@ -44,14 +44,23 @@ tags: [user-defined-functions, python] **Python script** ```python -# Choose from the following templates: -# +# Keep exactly ONE class below: uncomment it and delete the others. +# Its method must match its base class: +# UDFOperatorV2 -> process_tuple, UDFBatchOperator -> process_batch, +# UDFTableOperator -> process_table +# Always `yield` results (never `return`). Put setup code in open(); +# if you override __init__, call super().__init__() first. +# +# Port 0 (model) is read completely before port 1 (tuples) starts. +# Use the `port` argument to tell them apart, e.g. `if port == 0:`. +# # Define UiParameter inside open() of ProcessTupleOperator, ProcessBatchOperator, or ProcessTableOperator. # Example: self.count = self.UiParameter("count", AttributeType.INT).value +# Add value=Resource.MODEL or Resource.DATASET to pick a version; the value is its mount directory. # See the Python UDF operator documentation for supported types and behavior. -# -# from pytexera import * -# + +from pytexera import * + # class ProcessTupleOperator(UDFOperatorV2): # # @overrides diff --git a/docs/reference/operators/user-defined-functions/python/python-udf.md b/docs/reference/operators/user-defined-functions/python/python-udf.md index 3f1a1e7f854..35d535669db 100644 --- a/docs/reference/operators/user-defined-functions/python/python-udf.md +++ b/docs/reference/operators/user-defined-functions/python/python-udf.md @@ -44,14 +44,20 @@ tags: [user-defined-functions, python] **Python script** ```python -# Choose from the following templates: -# +# Keep exactly ONE class below: uncomment it and delete the others. +# Its method must match its base class: +# UDFOperatorV2 -> process_tuple, UDFBatchOperator -> process_batch, +# UDFTableOperator -> process_table +# Always `yield` results (never `return`). Put setup code in open(); +# if you override __init__, call super().__init__() first. +# # Define UiParameter inside open() of ProcessTupleOperator, ProcessBatchOperator, or ProcessTableOperator. # Example: self.count = self.UiParameter("count", AttributeType.INT).value +# Add value=Resource.MODEL or Resource.DATASET to pick a version; the value is its mount directory. # See the Python UDF operator documentation for supported types and behavior. -# -# from pytexera import * -# + +from pytexera import * + # class ProcessTupleOperator(UDFOperatorV2): # # @overrides From c14d0a18b2c3d9cb08a24d0c7591ef936f5350a0 Mon Sep 17 00:00:00 2001 From: suryaacharan Date: Thu, 8 Oct 2026 11:41:32 -0700 Subject: [PATCH 2/3] fix(workflow-operator): say "never return a value" and sync the migration prompt - A bare `return` to end a generator early is fine, so the templates now say "never `return` a value" (reference docs synced). - The notebook-migration prompt still embedded the old, fully commented template; update its header and import line. - The Python template test covers the Batch class too and says it is a reduced copy (the Scala specs check the real text). Co-Authored-By: Claude Opus 5.5 --- .../managers/test_executor_manager.py | 15 ++++++++++++--- .../python/DualInputPortsPythonUDFOpDescV2.scala | 2 +- .../operator/udf/python/PythonUDFOpDescV2.scala | 2 +- .../python/source/PythonUDFSourceOpDescV2.scala | 2 +- .../DualInputPortsPythonUDFOpDescV2Spec.scala | 2 +- .../udf/python/PythonUDFOpDescV2Spec.scala | 2 +- .../source/PythonUDFSourceOpDescV2Spec.scala | 2 +- .../python/1-out-python-udf.md | 2 +- .../python/2-in-python-udf.md | 2 +- .../user-defined-functions/python/python-udf.md | 2 +- .../notebook-migration/migration-prompts.ts | 6 +++--- 11 files changed, 24 insertions(+), 15 deletions(-) diff --git a/amber/src/test/python/core/architecture/managers/test_executor_manager.py b/amber/src/test/python/core/architecture/managers/test_executor_manager.py index 7885af99832..d7b9d039cd4 100644 --- a/amber/src/test/python/core/architecture/managers/test_executor_manager.py +++ b/amber/src/test/python/core/architecture/managers/test_executor_manager.py @@ -320,8 +320,9 @@ def test_close_is_noop_when_tmp_dir_never_materialized(self): assert "tmp_dir" not in manager.__dict__ -# Shape of the default Python UDF template (PythonUDFOpDescV2.defaultValue): -# the import is active and every class is commented out. +# A reduced copy of the default Python UDF template (PythonUDFOpDescV2. +# defaultValue): the import is active and every class is commented out. The +# Scala specs check the real template text; this checks how it loads. DEFAULT_TEMPLATE_CODE = """# Keep exactly ONE class below: uncomment it and delete the others. from pytexera import * @@ -332,6 +333,13 @@ def test_close_is_noop_when_tmp_dir_never_materialized(self): # def process_tuple(self, tuple_: Tuple, port: int) -> Iterator[Optional[TupleLike]]: # yield tuple_ # +# class ProcessBatchOperator(UDFBatchOperator): +# BATCH_SIZE = 10 # must be a positive integer +# +# @overrides +# def process_batch(self, batch: Batch, port: int) -> Iterator[Optional[BatchLike]]: +# yield batch +# # class ProcessTableOperator(UDFTableOperator): # # @overrides @@ -365,7 +373,8 @@ def test_untouched_template_reports_no_operator(self, executor_manager): assert "one and only one Operator" in str(exc_info.value) @pytest.mark.parametrize( - "class_name", ["ProcessTupleOperator", "ProcessTableOperator"] + "class_name", + ["ProcessTupleOperator", "ProcessBatchOperator", "ProcessTableOperator"], ) def test_uncommenting_one_class_loads_without_adding_an_import( self, executor_manager, class_name diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/python/DualInputPortsPythonUDFOpDescV2.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/python/DualInputPortsPythonUDFOpDescV2.scala index 65475d8169b..2bb49b00cee 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/python/DualInputPortsPythonUDFOpDescV2.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/python/DualInputPortsPythonUDFOpDescV2.scala @@ -37,7 +37,7 @@ class DualInputPortsPythonUDFOpDescV2 extends LogicalOp with PythonUdfUiParamete "# Its method must match its base class:\n" + "# UDFOperatorV2 -> process_tuple, UDFBatchOperator -> process_batch,\n" + "# UDFTableOperator -> process_table\n" + - "# Always `yield` results (never `return`). Put setup code in open();\n" + + "# Always `yield` results (never `return` a value). Put setup code in open();\n" + "# if you override __init__, call super().__init__() first.\n" + "# \n" + "# Port 0 (model) is read completely before port 1 (tuples) starts.\n" + diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/python/PythonUDFOpDescV2.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/python/PythonUDFOpDescV2.scala index e0e0e2a11d8..6094f28eae1 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/python/PythonUDFOpDescV2.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/python/PythonUDFOpDescV2.scala @@ -39,7 +39,7 @@ class PythonUDFOpDescV2 extends LogicalOp with PythonUdfUiParameterSupport { "# Its method must match its base class:\n" + "# UDFOperatorV2 -> process_tuple, UDFBatchOperator -> process_batch,\n" + "# UDFTableOperator -> process_table\n" + - "# Always `yield` results (never `return`). Put setup code in open();\n" + + "# Always `yield` results (never `return` a value). Put setup code in open();\n" + "# if you override __init__, call super().__init__() first.\n" + "# \n" + "# Define UiParameter inside open() of ProcessTupleOperator, ProcessBatchOperator, or ProcessTableOperator.\n" + diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/python/source/PythonUDFSourceOpDescV2.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/python/source/PythonUDFSourceOpDescV2.scala index 202b14832b6..623b80f2d6c 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/python/source/PythonUDFSourceOpDescV2.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/python/source/PythonUDFSourceOpDescV2.scala @@ -35,7 +35,7 @@ class PythonUDFSourceOpDescV2 extends SourceOperatorDescriptor with PythonUdfUiP required = true, defaultValue = "# Uncomment the class below. Its method must match its base class:\n" + "# UDFSourceOperator -> produce\n" + - "# Always `yield` results (never `return`). Put setup code in open();\n" + + "# Always `yield` results (never `return` a value). Put setup code in open();\n" + "# if you override __init__, call super().__init__() first.\n" + "# Keep exactly ONE class in this script.\n" + "# \n" + diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/DualInputPortsPythonUDFOpDescV2Spec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/DualInputPortsPythonUDFOpDescV2Spec.scala index 33078ea9a90..bd7bd3b9f46 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/DualInputPortsPythonUDFOpDescV2Spec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/DualInputPortsPythonUDFOpDescV2Spec.scala @@ -234,7 +234,7 @@ class DualInputPortsPythonUDFOpDescV2Spec extends AnyFlatSpec with Matchers { "DualInputPortsPythonUDFOpDescV2 default code" should "keep the pytexera import active and explain the rules" in { activeLines(defaultCode) shouldBe List("from pytexera import *") defaultCode should include("Keep exactly ONE class") - defaultCode should include("never `return`") + defaultCode should include("never `return` a value") defaultCode should include("super().__init__()") defaultCode should include("UDFOperatorV2 -> process_tuple") defaultCode should include("UDFBatchOperator -> process_batch") diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/PythonUDFOpDescV2Spec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/PythonUDFOpDescV2Spec.scala index 011ea6d713f..c99d04fcd3e 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/PythonUDFOpDescV2Spec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/PythonUDFOpDescV2Spec.scala @@ -329,7 +329,7 @@ class PythonUDFOpDescV2Spec extends AnyFlatSpec with Matchers { "PythonUDFOpDescV2 default code" should "keep the pytexera import active and explain the rules" in { activeLines(defaultCode) shouldBe List("from pytexera import *") defaultCode should include("Keep exactly ONE class") - defaultCode should include("never `return`") + defaultCode should include("never `return` a value") defaultCode should include("super().__init__()") defaultCode should include("UDFOperatorV2 -> process_tuple") defaultCode should include("UDFBatchOperator -> process_batch") diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/source/PythonUDFSourceOpDescV2Spec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/source/PythonUDFSourceOpDescV2Spec.scala index e25fcdb43d3..3d8016f1d4e 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/source/PythonUDFSourceOpDescV2Spec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/source/PythonUDFSourceOpDescV2Spec.scala @@ -184,7 +184,7 @@ class PythonUDFSourceOpDescV2Spec extends AnyFlatSpec with Matchers { "PythonUDFSourceOpDescV2 default code" should "keep the pytexera import active and explain the rules" in { activeLines(defaultCode) shouldBe List("from pytexera import *") defaultCode should include("Keep exactly ONE class") - defaultCode should include("never `return`") + defaultCode should include("never `return` a value") defaultCode should include("super().__init__()") defaultCode should include("UDFSourceOperator -> produce") defaultCode should include("UiParameter") diff --git a/docs/reference/operators/user-defined-functions/python/1-out-python-udf.md b/docs/reference/operators/user-defined-functions/python/1-out-python-udf.md index 1196bb2f8c6..bcfab91e6b0 100644 --- a/docs/reference/operators/user-defined-functions/python/1-out-python-udf.md +++ b/docs/reference/operators/user-defined-functions/python/1-out-python-udf.md @@ -45,7 +45,7 @@ tags: [user-defined-functions, python] ```python # Uncomment the class below. Its method must match its base class: # UDFSourceOperator -> produce -# Always `yield` results (never `return`). Put setup code in open(); +# Always `yield` results (never `return` a value). Put setup code in open(); # if you override __init__, call super().__init__() first. # Keep exactly ONE class in this script. # diff --git a/docs/reference/operators/user-defined-functions/python/2-in-python-udf.md b/docs/reference/operators/user-defined-functions/python/2-in-python-udf.md index ceafa259e54..a1d9143c55d 100644 --- a/docs/reference/operators/user-defined-functions/python/2-in-python-udf.md +++ b/docs/reference/operators/user-defined-functions/python/2-in-python-udf.md @@ -48,7 +48,7 @@ tags: [user-defined-functions, python] # Its method must match its base class: # UDFOperatorV2 -> process_tuple, UDFBatchOperator -> process_batch, # UDFTableOperator -> process_table -# Always `yield` results (never `return`). Put setup code in open(); +# Always `yield` results (never `return` a value). Put setup code in open(); # if you override __init__, call super().__init__() first. # # Port 0 (model) is read completely before port 1 (tuples) starts. diff --git a/docs/reference/operators/user-defined-functions/python/python-udf.md b/docs/reference/operators/user-defined-functions/python/python-udf.md index 35d535669db..1680c9f3b7e 100644 --- a/docs/reference/operators/user-defined-functions/python/python-udf.md +++ b/docs/reference/operators/user-defined-functions/python/python-udf.md @@ -48,7 +48,7 @@ tags: [user-defined-functions, python] # Its method must match its base class: # UDFOperatorV2 -> process_tuple, UDFBatchOperator -> process_batch, # UDFTableOperator -> process_table -# Always `yield` results (never `return`). Put setup code in open(); +# Always `yield` results (never `return` a value). Put setup code in open(); # if you override __init__, call super().__init__() first. # # Define UiParameter inside open() of ProcessTupleOperator, ProcessBatchOperator, or ProcessTableOperator. diff --git a/frontend/src/app/workspace/service/notebook-migration/migration-prompts.ts b/frontend/src/app/workspace/service/notebook-migration/migration-prompts.ts index 43f0d9dea19..90005677889 100644 --- a/frontend/src/app/workspace/service/notebook-migration/migration-prompts.ts +++ b/frontend/src/app/workspace/service/notebook-migration/migration-prompts.ts @@ -68,9 +68,9 @@ Batch API consumes a batch of tuples at a time. Similar to Table, a Batch is als All three APIs can return an empty iterator by yield None. The template code for a Python UDF follows: MAKE SURE TO USE THE CLASS NAMES AND FUNCTIONS DEFINED, THIS IS A MUST FOR THE PROGRAM TO WORK. SELECT 1 OUT OF THE 3 PROCESSING OPERATOR FUNCTIONS TO BUILD DEPENDING ON THE CONTEXT OF CODE TRANSLATION. -# Choose from the following templates: - # -# from pytexera import * +# Keep exactly ONE class below: uncomment it and delete the others. +# +from pytexera import * # # class ProcessTupleOperator(UDFOperatorV2): # From 992459972f6cc3147e7b210ef241c026121a5d74 Mon Sep 17 00:00:00 2001 From: suryaacharan Date: Wed, 14 Oct 2026 19:49:33 -0700 Subject: [PATCH 3/3] test(workflow-operator): share the Python UDF template spec helpers defaultCode, activeLines and uncommentClass were copied into the three template specs. Move them into PythonUdfTemplateTestUtils, which takes the descriptor class, so a fix lands in one place. Co-Authored-By: Claude Opus 5.5 --- .../DualInputPortsPythonUDFOpDescV2Spec.scala | 28 +++------- .../udf/python/PythonUDFOpDescV2Spec.scala | 28 +++------- .../python/PythonUdfTemplateTestUtils.scala | 51 +++++++++++++++++++ .../source/PythonUDFSourceOpDescV2Spec.scala | 29 +++-------- 4 files changed, 70 insertions(+), 66 deletions(-) create mode 100644 common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/PythonUdfTemplateTestUtils.scala diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/DualInputPortsPythonUDFOpDescV2Spec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/DualInputPortsPythonUDFOpDescV2Spec.scala index bd7bd3b9f46..7945099d009 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/DualInputPortsPythonUDFOpDescV2Spec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/DualInputPortsPythonUDFOpDescV2Spec.scala @@ -26,7 +26,6 @@ import org.apache.texera.amber.core.workflow.{PortIdentity, UnknownPartition} import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants import org.apache.texera.amber.util.JSONUtils.objectMapper -import com.fasterxml.jackson.annotation.JsonProperty import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -211,28 +210,11 @@ class DualInputPortsPythonUDFOpDescV2Spec extends AnyFlatSpec with Matchers { r.uiParameters.map(_.value) shouldBe List("7") } - private def defaultCode: String = - classOf[DualInputPortsPythonUDFOpDescV2] - .getDeclaredField("code") - .getAnnotation(classOf[JsonProperty]) - .defaultValue() - - private def activeLines(code: String): List[String] = - code.linesIterator.map(_.trim).filterNot(l => l.isEmpty || l.startsWith("#")).toList - - // Uncomment the template lines from `# class ` up to the next commented class. - private def uncommentClass(code: String, className: String): String = { - var inBlock = false - code.linesIterator - .map { line => - if (line.startsWith("# class ")) inBlock = line.startsWith(s"# class $className") - if (inBlock && line.startsWith("#")) line.stripPrefix("#").stripPrefix(" ") else line - } - .mkString("\n") - } + private val defaultCode: String = + PythonUdfTemplateTestUtils.defaultCode(classOf[DualInputPortsPythonUDFOpDescV2]) "DualInputPortsPythonUDFOpDescV2 default code" should "keep the pytexera import active and explain the rules" in { - activeLines(defaultCode) shouldBe List("from pytexera import *") + PythonUdfTemplateTestUtils.activeLines(defaultCode) shouldBe List("from pytexera import *") defaultCode should include("Keep exactly ONE class") defaultCode should include("never `return` a value") defaultCode should include("super().__init__()") @@ -247,7 +229,9 @@ class DualInputPortsPythonUDFOpDescV2Spec extends AnyFlatSpec with Matchers { for ( className <- List("ProcessTupleOperator", "ProcessBatchOperator", "ProcessTableOperator") ) { - val active = activeLines(uncommentClass(defaultCode, className)) + val active = PythonUdfTemplateTestUtils.activeLines( + PythonUdfTemplateTestUtils.uncommentClass(defaultCode, className) + ) active.head shouldBe "from pytexera import *" active.count(_.startsWith("class ")) shouldBe 1 active.exists(_.startsWith(s"class $className(")) shouldBe true diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/PythonUDFOpDescV2Spec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/PythonUDFOpDescV2Spec.scala index c99d04fcd3e..bf2d9da4302 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/PythonUDFOpDescV2Spec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/PythonUDFOpDescV2Spec.scala @@ -26,7 +26,6 @@ import org.apache.texera.amber.core.workflow.{PortIdentity, UnknownPartition} import org.apache.texera.amber.operator.{LogicalOp, PortDescription} import org.apache.texera.amber.operator.metadata.OperatorGroupConstants import org.apache.texera.amber.util.JSONUtils.objectMapper -import com.fasterxml.jackson.annotation.JsonProperty import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -306,28 +305,11 @@ class PythonUDFOpDescV2Spec extends AnyFlatSpec with Matchers { } } - private def defaultCode: String = - classOf[PythonUDFOpDescV2] - .getDeclaredField("code") - .getAnnotation(classOf[JsonProperty]) - .defaultValue() - - private def activeLines(code: String): List[String] = - code.linesIterator.map(_.trim).filterNot(l => l.isEmpty || l.startsWith("#")).toList - - // Uncomment the template lines from `# class ` up to the next commented class. - private def uncommentClass(code: String, className: String): String = { - var inBlock = false - code.linesIterator - .map { line => - if (line.startsWith("# class ")) inBlock = line.startsWith(s"# class $className") - if (inBlock && line.startsWith("#")) line.stripPrefix("#").stripPrefix(" ") else line - } - .mkString("\n") - } + private val defaultCode: String = + PythonUdfTemplateTestUtils.defaultCode(classOf[PythonUDFOpDescV2]) "PythonUDFOpDescV2 default code" should "keep the pytexera import active and explain the rules" in { - activeLines(defaultCode) shouldBe List("from pytexera import *") + PythonUdfTemplateTestUtils.activeLines(defaultCode) shouldBe List("from pytexera import *") defaultCode should include("Keep exactly ONE class") defaultCode should include("never `return` a value") defaultCode should include("super().__init__()") @@ -341,7 +323,9 @@ class PythonUDFOpDescV2Spec extends AnyFlatSpec with Matchers { for ( className <- List("ProcessTupleOperator", "ProcessBatchOperator", "ProcessTableOperator") ) { - val active = activeLines(uncommentClass(defaultCode, className)) + val active = PythonUdfTemplateTestUtils.activeLines( + PythonUdfTemplateTestUtils.uncommentClass(defaultCode, className) + ) active.head shouldBe "from pytexera import *" active.count(_.startsWith("class ")) shouldBe 1 active.exists(_.startsWith(s"class $className(")) shouldBe true diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/PythonUdfTemplateTestUtils.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/PythonUdfTemplateTestUtils.scala new file mode 100644 index 00000000000..9b92924f18a --- /dev/null +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/PythonUdfTemplateTestUtils.scala @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.operator.udf.python + +import com.fasterxml.jackson.annotation.JsonProperty + +/** + * Shared helpers for the Python UDF template specs: read a descriptor's default + * code template and simulate a user uncommenting one of its classes. + */ +object PythonUdfTemplateTestUtils { + + /** The `defaultValue` of the descriptor's `code` property, i.e. the template. */ + def defaultCode(descriptorClass: Class[_]): String = + descriptorClass + .getDeclaredField("code") + .getAnnotation(classOf[JsonProperty]) + .defaultValue() + + /** Lines that are neither blank nor comments. */ + def activeLines(code: String): List[String] = + code.linesIterator.map(_.trim).filterNot(l => l.isEmpty || l.startsWith("#")).toList + + /** Uncomment the template lines from `# class ` up to the next commented class. */ + def uncommentClass(code: String, className: String): String = { + var inBlock = false + code.linesIterator + .map { line => + if (line.startsWith("# class ")) inBlock = line.startsWith(s"# class $className") + if (inBlock && line.startsWith("#")) line.stripPrefix("#").stripPrefix(" ") else line + } + .mkString("\n") + } +} diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/source/PythonUDFSourceOpDescV2Spec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/source/PythonUDFSourceOpDescV2Spec.scala index 3d8016f1d4e..b4d32dc0a54 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/source/PythonUDFSourceOpDescV2Spec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/source/PythonUDFSourceOpDescV2Spec.scala @@ -26,7 +26,7 @@ import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants import org.apache.texera.amber.operator.udf.python.UiUDFParameter import org.apache.texera.amber.util.JSONUtils.objectMapper -import com.fasterxml.jackson.annotation.JsonProperty +import org.apache.texera.amber.operator.udf.python.PythonUdfTemplateTestUtils import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -161,28 +161,11 @@ class PythonUDFSourceOpDescV2Spec extends AnyFlatSpec with Matchers { p.uiParameters.map(_.value) shouldBe List("7") } - private def defaultCode: String = - classOf[PythonUDFSourceOpDescV2] - .getDeclaredField("code") - .getAnnotation(classOf[JsonProperty]) - .defaultValue() - - private def activeLines(code: String): List[String] = - code.linesIterator.map(_.trim).filterNot(l => l.isEmpty || l.startsWith("#")).toList - - // Uncomment the template lines from `# class ` up to the next commented class. - private def uncommentClass(code: String, className: String): String = { - var inBlock = false - code.linesIterator - .map { line => - if (line.startsWith("# class ")) inBlock = line.startsWith(s"# class $className") - if (inBlock && line.startsWith("#")) line.stripPrefix("#").stripPrefix(" ") else line - } - .mkString("\n") - } + private val defaultCode: String = + PythonUdfTemplateTestUtils.defaultCode(classOf[PythonUDFSourceOpDescV2]) "PythonUDFSourceOpDescV2 default code" should "keep the pytexera import active and explain the rules" in { - activeLines(defaultCode) shouldBe List("from pytexera import *") + PythonUdfTemplateTestUtils.activeLines(defaultCode) shouldBe List("from pytexera import *") defaultCode should include("Keep exactly ONE class") defaultCode should include("never `return` a value") defaultCode should include("super().__init__()") @@ -192,7 +175,9 @@ class PythonUDFSourceOpDescV2Spec extends AnyFlatSpec with Matchers { it should "leave exactly one active class after uncommenting any one template class" in { for (className <- List("GenerateOperator")) { - val active = activeLines(uncommentClass(defaultCode, className)) + val active = PythonUdfTemplateTestUtils.activeLines( + PythonUdfTemplateTestUtils.uncommentClass(defaultCode, className) + ) active.head shouldBe "from pytexera import *" active.count(_.startsWith("class ")) shouldBe 1 active.exists(_.startsWith(s"class $className(")) shouldBe true