diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregationOperation.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregationOperation.scala index 67146e81af2..f7155c952e9 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregationOperation.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/aggregate/AggregationOperation.scala @@ -133,7 +133,7 @@ class AggregationOperation { res } - private def sumAgg(attributeType: AttributeType): DistributedAggregation[Object] = { + private def requireNumericOrTimestamp(opName: String, attributeType: AttributeType): Unit = if ( attributeType != AttributeType.INTEGER && attributeType != AttributeType.DOUBLE && @@ -141,9 +141,12 @@ class AggregationOperation { attributeType != AttributeType.TIMESTAMP ) { throw new UnsupportedOperationException( - "Unsupported attribute type for sum aggregation: " + attributeType + s"Unsupported attribute type for $opName aggregation: $attributeType" ) } + + private def sumAgg(attributeType: AttributeType): DistributedAggregation[Object] = { + requireNumericOrTimestamp("sum", attributeType) new DistributedAggregation[Object]( () => AttributeTypeUtils.zeroValue(attributeType), (partial, tuple) => { @@ -191,55 +194,32 @@ class AggregationOperation { ) } - private def minAgg(attributeType: AttributeType): DistributedAggregation[Object] = { - if ( - attributeType != AttributeType.INTEGER && - attributeType != AttributeType.DOUBLE && - attributeType != AttributeType.LONG && - attributeType != AttributeType.TIMESTAMP - ) { - throw new UnsupportedOperationException( - "Unsupported attribute type for min aggregation: " + attributeType - ) - } + private def extremumAgg( + opName: String, + sentinel: AttributeType => Object, + isBetter: Int => Boolean, + attributeType: AttributeType + ): DistributedAggregation[Object] = { + requireNumericOrTimestamp(opName, attributeType) new DistributedAggregation[Object]( - () => AttributeTypeUtils.maxValue(attributeType), + () => sentinel(attributeType), (partial, tuple) => { val value = tuple.getField[Object](attribute) val comp = AttributeTypeUtils.compare(value, partial, attributeType) - if (value != null && comp < 0) value else partial + if (value != null && isBetter(comp)) value else partial }, (partial1, partial2) => - if (AttributeTypeUtils.compare(partial1, partial2, attributeType) < 0) partial1 + if (isBetter(AttributeTypeUtils.compare(partial1, partial2, attributeType))) partial1 else partial2, - partial => if (partial == AttributeTypeUtils.maxValue(attributeType)) null else partial + partial => if (partial == sentinel(attributeType)) null else partial ) } - private def maxAgg(attributeType: AttributeType): DistributedAggregation[Object] = { - if ( - attributeType != AttributeType.INTEGER && - attributeType != AttributeType.DOUBLE && - attributeType != AttributeType.LONG && - attributeType != AttributeType.TIMESTAMP - ) { - throw new UnsupportedOperationException( - "Unsupported attribute type for max aggregation: " + attributeType - ) - } - new DistributedAggregation[Object]( - () => AttributeTypeUtils.minValue(attributeType), - (partial, tuple) => { - val value = tuple.getField[Object](attribute) - val comp = AttributeTypeUtils.compare(value, partial, attributeType) - if (value != null && comp > 0) value else partial - }, - (partial1, partial2) => - if (AttributeTypeUtils.compare(partial1, partial2, attributeType) > 0) partial1 - else partial2, - partial => if (partial == AttributeTypeUtils.minValue(attributeType)) null else partial - ) - } + private def minAgg(attributeType: AttributeType): DistributedAggregation[Object] = + extremumAgg("min", AttributeTypeUtils.maxValue, _ < 0, attributeType) + + private def maxAgg(attributeType: AttributeType): DistributedAggregation[Object] = + extremumAgg("max", AttributeTypeUtils.minValue, _ > 0, attributeType) private def getNumericalValue(tuple: Tuple): Option[Double] = { val value: Object = tuple.getField(attribute) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceInferenceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceInferenceOpDesc.scala index cf506fb7606..3dd24ed5880 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceInferenceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceInferenceOpDesc.scala @@ -58,6 +58,7 @@ import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString * `PythonCodeRawInvalidTextSpec`'s contract that arbitrary `@JsonProperty` * values must not leak into generated source. */ +@SerialVersionUID(-3887825915400170939L) class HuggingFaceInferenceOpDesc extends PythonOperatorDescriptor { @JsonProperty(value = "hfApiToken", required = true) @@ -169,15 +170,10 @@ class HuggingFaceInferenceOpDesc extends PythonOperatorDescriptor { * keeps `generatePythonCode` total (it never throws on arbitrary input, * which is required by `PythonCodeRawInvalidTextSpec`). */ - private val registeredCodegens: Map[String, TaskCodegen] = { - val byTask = scala.collection.mutable.Map.empty[String, TaskCodegen] - byTask += (TextGenCodegen.task -> TextGenCodegen) - ImageTaskCodegen.tasks.foreach(t => byTask += (t -> ImageTaskCodegen)) - AudioTaskCodegen.tasks.foreach(t => byTask += (t -> AudioTaskCodegen)) - MediaGenCodegen.tasks.foreach(t => byTask += (t -> MediaGenCodegen)) - QaRankingCodegen.tasks.foreach(t => byTask += (t -> QaRankingCodegen)) - byTask.toMap - } + private val registeredCodegens: Map[String, TaskCodegen] = + Seq(TextGenCodegen, ImageTaskCodegen, AudioTaskCodegen, MediaGenCodegen, QaRankingCodegen) + .flatMap(codegen => codegen.tasks.map(task => task -> codegen)) + .toMap private def codegenForTask(t: String): TaskCodegen = registeredCodegens.getOrElse(t, TextGenCodegen) @@ -200,52 +196,24 @@ class HuggingFaceInferenceOpDesc extends PythonOperatorDescriptor { override def generatePythonCode(): String = { val safeTask: EncodableString = if (task == null || task.trim.isEmpty) "text-generation" else task - val safeModelId: EncodableString = - if (modelId == null) "" else modelId.trim - val safePromptCol: EncodableString = - if (promptColumn == null) "" else promptColumn - val safeResultCol: EncodableString = resolvedResultColumn - val safeSystemPrompt: EncodableString = - if (systemPrompt == null) "" else systemPrompt - val safeToken: EncodableString = - if (hfApiToken == null) "" else hfApiToken - - val safeMaxTokens = - math.max(1, math.min(if (maxNewTokens != null) maxNewTokens.intValue else 256, 4096)) - val safeTemp = - math.max(0.0, math.min(if (temperature != null) temperature.doubleValue else 0.7, 2.0)) - - val safeImageInput: EncodableString = - if (imageInput == null) "" else imageInput - val safeInputImageColumn: EncodableString = - if (inputImageColumn == null) "" else inputImageColumn - val safeAudioInput: EncodableString = - if (audioInput == null) "" else audioInput - val safeInputAudioColumn: EncodableString = - if (inputAudioColumn == null) "" else inputAudioColumn - val safeContextColumn: EncodableString = - if (contextColumn == null) "" else contextColumn - val safeCandidateLabels: EncodableString = - if (candidateLabels == null) "" else candidateLabels - val safeSentencesColumn: EncodableString = - if (sentencesColumn == null) "" else sentencesColumn - val ctx = CodegenContext( - hfApiToken = safeToken, - modelId = safeModelId, - promptColumn = safePromptCol, - resultColumn = safeResultCol, - task = safeTask, - systemPrompt = safeSystemPrompt, - safeMaxTokens = safeMaxTokens, - safeTemp = safeTemp, - imageInput = safeImageInput, - inputImageColumn = safeInputImageColumn, - audioInput = safeAudioInput, - inputAudioColumn = safeInputAudioColumn, - contextColumn = safeContextColumn, - candidateLabels = safeCandidateLabels, - sentencesColumn = safeSentencesColumn + modelId = if (modelId == null) "" else modelId.trim, + promptColumn = if (promptColumn == null) "" else promptColumn, + resultColumn = resolvedResultColumn, + systemPrompt = if (systemPrompt == null) "" else systemPrompt, + hfApiToken = if (hfApiToken == null) "" else hfApiToken, + safeMaxTokens = + math.max(1, math.min(if (maxNewTokens != null) maxNewTokens.intValue else 256, 4096)), + safeTemp = + math.max(0.0, math.min(if (temperature != null) temperature.doubleValue else 0.7, 2.0)), + imageInput = if (imageInput == null) "" else imageInput, + inputImageColumn = if (inputImageColumn == null) "" else inputImageColumn, + audioInput = if (audioInput == null) "" else audioInput, + inputAudioColumn = if (inputAudioColumn == null) "" else inputAudioColumn, + contextColumn = if (contextColumn == null) "" else contextColumn, + candidateLabels = if (candidateLabels == null) "" else candidateLabels, + sentencesColumn = if (sentencesColumn == null) "" else sentencesColumn, + task = safeTask ) HuggingFaceCodegenBase.render(ctx, codegenForTask(safeTask)) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/codegen/AudioTaskCodegen.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/codegen/AudioTaskCodegen.scala index 2b8e260a469..453f89e1e98 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/codegen/AudioTaskCodegen.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/codegen/AudioTaskCodegen.scala @@ -45,35 +45,21 @@ object AudioTaskCodegen extends TaskCodegen { | payload = {"inputs": prompt_value}""".stripMargin override def parsePython(ctx: CodegenContext): String = - """ if task == "text-to-speech": + s""" if task == "text-to-speech": | # Every value below comes from the provider, so each is type-checked | # before use: an unchecked one lands in the result column as a data | # URL nothing can play (e.g. "data:audio/mpeg;base64,None") or as a | # non-string cell, instead of falling back to the body. | if isinstance(body, dict): - | if "output" in body: - | out = body["output"] - | url = out[0] if isinstance(out, list) else out - | if isinstance(url, str) and url.startswith("http"): - | return self._url_to_data_url(url) + |${BinaryResponseFragments.outputUrl} | if "audio" in body: | audio = body["audio"] | if isinstance(audio, dict): - | url = audio.get("url") - | if isinstance(url, str) and url.startswith("http"): - | return self._url_to_data_url(url) - | b64 = audio.get("b64_json") - | if isinstance(b64, str) and b64: - | return f"data:audio/mpeg;base64,{b64}" + |${audioDataUrl("audio")} | if "data" in body: | data = body["data"] | if isinstance(data, list) and data and isinstance(data[0], dict): - | url = data[0].get("url") - | if isinstance(url, str) and url.startswith("http"): - | return self._url_to_data_url(url) - | b64 = data[0].get("b64_json") - | if isinstance(b64, str) and b64: - | return f"data:audio/mpeg;base64,{b64}" + |${audioDataUrl("data[0]")} | return json.dumps(body) | elif task == "automatic-speech-recognition": | if isinstance(body, dict): @@ -86,4 +72,12 @@ object AudioTaskCodegen extends TaskCodegen { | return json.dumps(body) | elif task == "audio-classification": | return json.dumps(body)""".stripMargin + + private def audioDataUrl(value: String): String = + s""" url = ${value}.get("url") + | if isinstance(url, str) and url.startswith("http"): + | return self._url_to_data_url(url) + | b64 = ${value}.get("b64_json") + | if isinstance(b64, str) and b64: + | return f"data:audio/mpeg;base64,{b64}"""".stripMargin } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/codegen/HuggingFaceCodegenBase.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/codegen/HuggingFaceCodegenBase.scala index 5d4bf695ec0..23bf5f13d58 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/codegen/HuggingFaceCodegenBase.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/codegen/HuggingFaceCodegenBase.scala @@ -224,24 +224,14 @@ object HuggingFaceCodegenBase { | img_b64 = "" | if use_raw_binary_body and isinstance(pipeline_payload, bytes): | img_b64 = base64.b64encode(pipeline_payload).decode("utf-8") - | chat_payload = { - | "model": provider_id, - | "messages": [{ - | "role": "user", - | "content": [ - | {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img_b64}"}} if img_b64 else None, - | {"type": "text", "text": self._chat_content_for_task(pipeline_payload, prompt_value) or "What is in this image?"}, - | ], - | }], - | } - | chat_payload["messages"][0]["content"] = [c for c in chat_payload["messages"][0]["content"] if c is not None] + | parts = self._image_chat_parts( + | img_b64, self._chat_content_for_task(pipeline_payload, prompt_value) or "What is in this image?" + | ) + | chat_payload = {"model": provider_id, "messages": [{"role": "user", "content": parts}]} | resp = requests.post(url, headers=json_headers, json=chat_payload, timeout=120) | elif provider_name == "hf-inference": | url = f"https://router.huggingface.co/hf-inference/models/{self.MODEL_ID}" - | if use_raw_binary_body: - | resp = requests.post(url, headers=raw_binary_headers, data=pipeline_payload, timeout=120) - | else: - | resp = requests.post(url, headers=json_headers, json=pipeline_payload, timeout=120) + | resp = self._post_pipeline(url, json_headers, raw_binary_headers, pipeline_payload, use_raw_binary_body) | else: | resp = self._call_provider(provider_name, provider_id, json_headers, raw_binary_headers, pipeline_payload, use_raw_binary_body, prompt_value) | except Exception as e: @@ -373,6 +363,87 @@ object HuggingFaceCodegenBase { | return query or prompt_value | return prompt_value | + | def _post_pipeline(self, url, json_headers, raw_binary_headers, payload, binary): + | if binary: + | return requests.post(url, headers=raw_binary_headers, data=payload, timeout=120) + | return requests.post(url, headers=json_headers, json=payload, timeout=120) + | + | def _image_chat_parts(self, img_b64, text): + | parts = [] + | if img_b64: + | parts.append({"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img_b64}"}}) + | parts.append({"type": "text", "text": text}) + | return parts + | + | def _provider_chat_messages(self, pipeline_payload, prompt_value, img_b64, image_prompt): + | content = self._chat_content_for_task(pipeline_payload, prompt_value) + | if img_b64: + | content = self._image_chat_parts(img_b64, self._chat_content_for_task(pipeline_payload, prompt_value) or image_prompt) + | return [{"role": "user", "content": content}] + | + | def _media_provider_payload(self, task, img_b64, raw_binary_headers, prompt_value, replicate): + | if task == "text-to-speech": + | return {"text": prompt_value} + | elif task in ("text-to-image", "text-to-video") or not img_b64: + | return {"prompt": prompt_value} + | elif task in ("automatic-speech-recognition", "audio-classification") and img_b64: + | if replicate: + | audio_content_type = raw_binary_headers.get("Content-Type", "audio/mpeg") + | return {"audio": f"data:{audio_content_type};base64,{img_b64}"} + | data_url = f"data:image/png;base64,{img_b64}" + | if task == "image-to-image": + | if replicate: + | return {"image": data_url, "images": [data_url], "input_image": data_url, "prompt": prompt_value} + | return {"image_url": data_url, "image_urls": [data_url], "prompt": prompt_value} + | return {("image" if replicate else "image_url"): data_url, "prompt": prompt_value} + | + | def _poll_prediction(self, base, path, headers, poll_resp, attempts, delay, provider, is_terminal): + | import time as _time + | from urllib.parse import urlparse as _urlparse + | poll_url = f"{base}{_urlparse(path).path}" + | for poll_idx in range(attempts): + | _time.sleep(delay) + | poll_resp = requests.get(poll_url, headers=headers, timeout=30) + | if poll_resp.status_code != 200: + | continue + | if is_terminal(poll_resp): + | return poll_resp + | if (poll_idx + 1) % 30 == 0: + | print(f"[hf] {provider} still running for model '{self.MODEL_ID}' after {(poll_idx + 1) * delay}s; will wait up to {attempts * delay}s.") + | return poll_resp + | + | def _mark_failed_prediction(self, poll_resp, error): + | # A successful HTTP poll can report a failed job. Surface it through provider fallback. + | poll_resp.status_code = 502 + | poll_resp._content = json.dumps({"error": error}).encode("utf-8") + | + | def _replicate_terminal(self, poll_resp): + | status = poll_resp.json().get("status", "") + | if status == "succeeded": + | return True + | if status in ("failed", "canceled"): + | body_json = poll_resp.json() if poll_resp.text else {} + | detail = (body_json.get("error") or body_json.get("logs") or status) if isinstance(body_json, dict) else status + | self._mark_failed_prediction(poll_resp, f"Replicate prediction {status}: {detail}") + | return True + | return False + | + | def _wavespeed_terminal(self, poll_resp): + | body_json = poll_resp.json() if poll_resp.text else {} + | data_obj = body_json.get("data", {}) if isinstance(body_json, dict) else {} + | status = data_obj.get("status", "") if isinstance(data_obj, dict) else "" + | if status == "completed": + | return True + | if status == "failed": + | detail = ( + | (data_obj.get("error") if isinstance(data_obj, dict) else None) + | or (body_json.get("error") if isinstance(body_json, dict) else None) + | or "failed" + | ) + | self._mark_failed_prediction(poll_resp, f"Wavespeed job failed: {detail}") + | return True + | return False + | | def _call_provider(self, provider_name, provider_id, json_headers, raw_binary_headers, pipeline_payload, use_raw_binary_body, prompt_value): | '''Route to a third-party provider using its native API format. | Handles OpenAI-compatible chat providers for text-gen, zai-org's @@ -410,89 +481,28 @@ object HuggingFaceCodegenBase { | file_data = f"data:image/png;base64,{img_b64}" if img_b64 else "" | return requests.post(url, headers=zai_headers, json={"model": provider_id, "file": file_data}, timeout=120) | url = f"{base}/api/paas/v4/chat/completions" - | messages = [{"role": "user", "content": self._chat_content_for_task(pipeline_payload, prompt_value)}] - | if img_b64: - | messages = [{"role": "user", "content": [ - | {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img_b64}"}}, - | {"type": "text", "text": self._chat_content_for_task(pipeline_payload, prompt_value) or "What is in this image?"}, - | ]}] + | messages = self._provider_chat_messages(pipeline_payload, prompt_value, img_b64, "What is in this image?") | return requests.post(url, headers=zai_headers, json={"model": provider_id, "messages": messages}, timeout=120) | | # Replicate: synchronous predictions endpoint with polling fallback. | if provider_name == "replicate": | url = f"{base}/v1/models/{provider_id}/predictions" | hdrs = {**json_headers, "Prefer": "wait"} - | if task == "text-to-speech": - | inp = {"text": prompt_value} - | elif task in ("text-to-image", "text-to-video"): - | inp = {"prompt": prompt_value} - | elif task in ("automatic-speech-recognition", "audio-classification") and img_b64: - | audio_content_type = raw_binary_headers.get("Content-Type", "audio/mpeg") - | inp = {"audio": f"data:{audio_content_type};base64,{img_b64}"} - | elif task == "image-to-image" and img_b64: - | data_url = f"data:image/png;base64,{img_b64}" - | inp = {"image": data_url, "images": [data_url], "input_image": data_url, "prompt": prompt_value} - | elif img_b64: - | inp = {"image": f"data:image/png;base64,{img_b64}", "prompt": prompt_value} - | else: - | inp = {"prompt": prompt_value} + | inp = self._media_provider_payload(task, img_b64, raw_binary_headers, prompt_value, replicate=True) | resp = requests.post(url, headers=hdrs, json={"input": inp}, timeout=120) | if resp.status_code == 202: - | import time as _time - | pred = resp.json() - | poll_url = pred.get("urls", {}).get("get", "") - | if not poll_url: - | return resp - | from urllib.parse import urlparse as _urlparse - | poll_path = _urlparse(poll_url).path - | poll_url = f"{base}{poll_path}" - | # Worst case: 300 polls × 2s = ~10 minutes per row before we give - | # up. Sized for text-to-video which legitimately takes minutes on - | # Replicate. process_table is synchronous, so emit a progress - | # line every 30 polls (~1 min) to distinguish slow work from a - | # hang in the worker log. - | for poll_idx in range(300): - | _time.sleep(2) - | poll_resp = requests.get(poll_url, headers=json_headers, timeout=30) - | if poll_resp.status_code != 200: - | continue - | status = poll_resp.json().get("status", "") - | if status == "succeeded": - | return poll_resp - | if status in ("failed", "canceled"): - | # The polling HTTP request itself returned 200, but the - | # Replicate prediction terminally failed. Without this - | # branch, process_table would treat the 200 as success - | # and emit json.dumps(body) (raw error JSON) into the - | # output cell. Convert to a synthetic 502 so - | # _post_with_fallback's non-200 handler surfaces the - | # actual failure detail via _format_error. - | body_json = poll_resp.json() if poll_resp.text else {} - | detail = (body_json.get("error") or body_json.get("logs") or status) \ - | if isinstance(body_json, dict) else status - | poll_resp.status_code = 502 - | poll_resp._content = json.dumps({ - | "error": f"Replicate prediction {status}: {detail}" - | }).encode("utf-8") - | return poll_resp - | if (poll_idx + 1) % 30 == 0: - | print(f"[hf] Replicate still running for model '{self.MODEL_ID}' after {(poll_idx + 1) * 2}s; will wait up to 600s.") - | return poll_resp + | poll_url = resp.json().get("urls", {}).get("get", "") + | if poll_url: + | return self._poll_prediction( + | base, poll_url, json_headers, resp, 300, 2, "Replicate", self._replicate_terminal + | ) | return resp | | # Fal-ai: per-model endpoint. | if provider_name == "fal-ai": | url = f"{base}/{provider_id}" - | if task == "text-to-speech": - | return requests.post(url, headers=json_headers, json={"text": prompt_value}, timeout=120) - | if task in ("text-to-image", "text-to-video"): - | return requests.post(url, headers=json_headers, json={"prompt": prompt_value}, timeout=120) - | if task == "image-to-image" and img_b64: - | data_url = f"data:image/png;base64,{img_b64}" - | return requests.post(url, headers=json_headers, json={"image_url": data_url, "image_urls": [data_url], "prompt": prompt_value}, timeout=120) - | if img_b64: - | return requests.post(url, headers=json_headers, json={"image_url": f"data:image/png;base64,{img_b64}", "prompt": prompt_value}, timeout=120) - | return requests.post(url, headers=json_headers, json={"prompt": prompt_value}, timeout=120) + | payload = self._media_provider_payload(task, img_b64, raw_binary_headers, prompt_value, replicate=False) + | return requests.post(url, headers=json_headers, json=payload, timeout=120) | | # Wavespeed: async submit + poll. | if provider_name == "wavespeed": @@ -507,42 +517,9 @@ object HuggingFaceCodegenBase { | get_path = submit_resp.json().get("data", {}).get("urls", {}).get("get", "") | if not get_path: | return submit_resp - | from urllib.parse import urlparse as _urlparse - | result_url = f"{base}{_urlparse(get_path).path}" - | import time as _time - | poll_resp = submit_resp - | # Worst case: 120 polls × 1s = ~2 minutes per row. Emit a progress - | # line every 30 polls (~30 s) so the worker log distinguishes slow - | # work from a hang. - | for poll_idx in range(120): - | _time.sleep(1) - | poll_resp = requests.get(result_url, headers=json_headers, timeout=30) - | if poll_resp.status_code != 200: - | continue - | body_json = poll_resp.json() if poll_resp.text else {} - | data_obj = body_json.get("data", {}) if isinstance(body_json, dict) else {} - | status = data_obj.get("status", "") if isinstance(data_obj, dict) else "" - | if status == "completed": - | return poll_resp - | if status == "failed": - | # Same shape as Replicate: HTTP 200 + body says "failed". - | # Synthesize a 502 so _post_with_fallback's non-200 handler - | # reports the actual reason instead of process_table - | # parsing the success-shaped body and writing raw error - | # JSON into the result cell. - | detail = ( - | (data_obj.get("error") if isinstance(data_obj, dict) else None) - | or (body_json.get("error") if isinstance(body_json, dict) else None) - | or "failed" - | ) - | poll_resp.status_code = 502 - | poll_resp._content = json.dumps({ - | "error": f"Wavespeed job failed: {detail}" - | }).encode("utf-8") - | return poll_resp - | if (poll_idx + 1) % 30 == 0: - | print(f"[hf] Wavespeed still running for model '{self.MODEL_ID}' after {poll_idx + 1}s; will wait up to 120s.") - | return poll_resp + | return self._poll_prediction( + | base, get_path, json_headers, submit_resp, 120, 1, "Wavespeed", self._wavespeed_terminal + | ) | | if provider_name in self.OPENAI_COMPATIBLE_PROVIDERS: | if task == "text-to-image": @@ -552,12 +529,7 @@ object HuggingFaceCodegenBase { | url = f"{base}/v1/audio/speech" | return requests.post(url, headers=json_headers, json={"model": provider_id, "input": prompt_value}, timeout=120) | url = f"{base}/{self.CHAT_ROUTES.get(provider_name, 'v1/chat/completions')}" - | messages = [{"role": "user", "content": self._chat_content_for_task(pipeline_payload, prompt_value)}] - | if img_b64: - | messages = [{"role": "user", "content": [ - | {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img_b64}"}}, - | {"type": "text", "text": self._chat_content_for_task(pipeline_payload, prompt_value) or "What is in this image?"}, - | ]}] + | messages = self._provider_chat_messages(pipeline_payload, prompt_value, img_b64, "What is in this image?") | return requests.post( | url, | headers=json_headers, @@ -567,18 +539,10 @@ object HuggingFaceCodegenBase { | | # Unknown provider: try pipeline format, then chat completions. | url = f"{base}/{provider_id}" - | if use_raw_binary_body: - | resp = requests.post(url, headers=raw_binary_headers, data=pipeline_payload, timeout=120) - | else: - | resp = requests.post(url, headers=json_headers, json=pipeline_payload, timeout=120) + | resp = self._post_pipeline(url, json_headers, raw_binary_headers, pipeline_payload, use_raw_binary_body) | if resp.status_code in (400, 404, 422): | url = f"{base}/v1/chat/completions" - | messages = [{"role": "user", "content": self._chat_content_for_task(pipeline_payload, prompt_value)}] - | if img_b64: - | messages = [{"role": "user", "content": [ - | {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img_b64}"}}, - | {"type": "text", "text": self._chat_content_for_task(pipeline_payload, prompt_value) or "Describe this image."}, - | ]}] + | messages = self._provider_chat_messages(pipeline_payload, prompt_value, img_b64, "Describe this image.") | resp2 = requests.post( | url, | headers=json_headers, @@ -677,60 +641,44 @@ object HuggingFaceCodegenBase { | str(v) if not pd.isna(v) else "" for v in table[col].tolist() | ] | - | # --- resolve image source (upload or column) for image tasks --- + | # --- resolve upload or column input for image and audio tasks --- | has_image_upload = bool(self.IMAGE_INPUT) and bool(str(self.IMAGE_INPUT).strip()) | use_image_column = not has_image_upload and bool(self.INPUT_IMAGE_COLUMN) and self.INPUT_IMAGE_COLUMN in table.columns - | image_bytes = None - | image_error = None | has_audio_upload = bool(self.AUDIO_INPUT) and bool(str(self.AUDIO_INPUT).strip()) | use_audio_column = not has_audio_upload and bool(self.INPUT_AUDIO_COLUMN) and self.INPUT_AUDIO_COLUMN in table.columns | audio_headers = { | "Authorization": f"Bearer {token}", | "Content-Type": "application/octet-stream" if use_audio_column else self._get_audio_content_type(), | } - | audio_bytes = None - | audio_error = None - | if task in image_tasks and not use_image_column: - | if not has_image_upload: - | if self.INPUT_IMAGE_COLUMN and str(self.INPUT_IMAGE_COLUMN).strip(): - | image_error = ( - | f"Input Image Column '{self.INPUT_IMAGE_COLUMN}' not found in the input table. " - | f"Available columns: {list(table.columns)}" - | ) - | else: - | image_error = "No image source. Set an Input Image Column or upload an image." - | else: - | try: - | image_bytes = self._read_image_input() - | except Exception as e: - | image_error = f"Could not read image input ({type(e).__name__}: {e})" - | if task in audio_only_tasks and not use_audio_column: - | if not has_audio_upload: - | if self.INPUT_AUDIO_COLUMN and str(self.INPUT_AUDIO_COLUMN).strip(): - | audio_error = ( - | f"Input Audio Column '{self.INPUT_AUDIO_COLUMN}' not found in the input table. " + | media_kind = "image" if task in image_tasks else ("audio" if task in audio_only_tasks else None) + | media_bytes = media_error = None + | if media_kind is not None: + | is_image = media_kind == "image" + | has_upload = has_image_upload if is_image else has_audio_upload + | use_media_column = use_image_column if is_image else use_audio_column + | media_column = self.INPUT_IMAGE_COLUMN if is_image else self.INPUT_AUDIO_COLUMN + | if not use_media_column: + | if has_upload: + | try: + | media_bytes = self._read_image_input() if is_image else self._read_audio_input() + | except Exception as e: + | media_error = f"Could not read {media_kind} input ({type(e).__name__}: {e})" + | elif media_column and str(media_column).strip(): + | media_error = ( + | f"Input {media_kind.title()} Column '{media_column}' not found in the input table. " | f"Available columns: {list(table.columns)}" | ) | else: - | audio_error = "No audio source. Set an Input Audio Column or upload audio." - | else: - | try: - | audio_bytes = self._read_audio_input() - | except Exception as e: - | audio_error = f"Could not read audio input ({type(e).__name__}: {e})" + | upload = "an image" if is_image else "audio" + | media_error = f"No {media_kind} source. Set an Input {media_kind.title()} Column or upload {upload}." | | results = [] | for idx, row in table.iterrows(): - | if image_error is not None: - | results.append(self._format_error("Image task configuration error", image_error)) - | continue - | if audio_error is not None: - | results.append(self._format_error("Audio task configuration error", audio_error)) + | if media_error is not None: + | results.append(self._format_error(f"{media_kind.title()} task configuration error", media_error)) | continue | - | if task in image_only_tasks: - | prompt_value = "" - | elif task in audio_only_tasks: + | if task in image_only_tasks or task in audio_only_tasks: | prompt_value = "" | elif task in image_prompt_tasks and prompt_col not in table.columns: | prompt_value = "What is shown in this image?" @@ -741,30 +689,21 @@ object HuggingFaceCodegenBase { | else: | prompt_value = str(prompt_value) | - | # --- resolve per-row image bytes from column --- - | current_image_bytes = image_bytes - | if task in image_tasks and use_image_column: + | # --- resolve per-row media bytes from column --- + | current_media_bytes = media_bytes + | if media_kind is not None and use_media_column: | try: - | raw = self._read_binary_value(row[self.INPUT_IMAGE_COLUMN]) - | if raw is None: - | results.append(self._format_error("Image data error", f"Row {idx}: image column is empty")) + | current_media_bytes = self._read_binary_value(row[media_column]) + | if current_media_bytes is None: + | results.append(self._format_error(f"{media_kind.title()} data error", f"Row {idx}: {media_kind} column is empty")) | continue - | current_image_bytes = self._compress_image_bytes(raw) + | if is_image: + | current_media_bytes = self._compress_image_bytes(current_media_bytes) | except Exception as e: - | results.append(self._format_error("Image data error", f"Row {idx}: {type(e).__name__}: {e}")) - | continue - | - | # --- resolve per-row audio bytes from column --- - | current_audio_bytes = audio_bytes - | if task in audio_only_tasks and use_audio_column: - | try: - | current_audio_bytes = self._read_binary_value(row[self.INPUT_AUDIO_COLUMN]) - | if current_audio_bytes is None: - | results.append(self._format_error("Audio data error", f"Row {idx}: audio column is empty")) - | continue - | except Exception as e: - | results.append(self._format_error("Audio data error", f"Row {idx}: {type(e).__name__}: {e}")) + | results.append(self._format_error(f"{media_kind.title()} data error", f"Row {idx}: {type(e).__name__}: {e}")) | continue + | current_image_bytes = current_media_bytes if media_kind == "image" else None + | current_audio_bytes = current_media_bytes if media_kind == "audio" else None | | # --- build task-specific payload (provided by per-task codegen) --- | use_raw_binary_body = False @@ -809,11 +748,7 @@ object HuggingFaceCodegenBase { | continue | | content_type = resp.headers.get("Content-Type", "") - | if content_type.startswith("image/"): - | b64 = base64.b64encode(resp.content).decode("utf-8") - | results.append(f"data:{content_type};base64,{b64}") - | continue - | if content_type.startswith("audio/") or content_type.startswith("video/"): + | if content_type.startswith(("image/", "audio/", "video/")): | b64 = base64.b64encode(resp.content).decode("utf-8") | results.append(f"data:{content_type};base64,{b64}") | continue @@ -930,19 +865,20 @@ object HuggingFaceCodegenBase { | chunks.append(chunk) | return content_type, b"".join(chunks) | - | def _read_image_input(self): - | image_input = str(self.IMAGE_INPUT or "").strip() - | if image_input.startswith("data:"): - | _, encoded = image_input.split(",", 1) + | def _read_media_upload(self, value, error_message): + | source = str(value or "").strip() + | if source.startswith("data:"): + | _, encoded = source.split(",", 1) | return base64.b64decode(encoded) - | if image_input.startswith("http://") or image_input.startswith("https://"): - | _, data = self._fetch_remote_url(image_input) + | if source.startswith(("http://", "https://")): + | _, data = self._fetch_remote_url(source) | return data - | # Reading arbitrary worker-filesystem paths is intentionally NOT - | # supported: a workflow could otherwise point this at any file on the - | # worker (e.g. /etc/passwd) and exfiltrate it via the inference call. - | # Uploaded images arrive as data URLs; remote images as https URLs. - | raise ValueError( + | # Reject worker-filesystem paths. Remote reads use the hardened URL fetch above. + | raise ValueError(error_message) + | + | def _read_image_input(self): + | return self._read_media_upload( + | self.IMAGE_INPUT, | "Unsupported image input. Upload an image (sent as a data URL) " | "or provide a public https image URL." | ) @@ -980,17 +916,8 @@ object HuggingFaceCodegenBase { | return base64.b64encode(image_bytes).decode("utf-8") | | def _read_audio_input(self): - | audio_input = str(self.AUDIO_INPUT or "").strip() - | if audio_input.startswith("data:"): - | _, encoded = audio_input.split(",", 1) - | return base64.b64decode(encoded) - | if audio_input.startswith("http://") or audio_input.startswith("https://"): - | _, data = self._fetch_remote_url(audio_input) - | return data - | # Reading arbitrary worker-filesystem paths is intentionally NOT - | # supported: uploaded audio arrives as a data URL and remote audio - | # must be fetched through the hardened https-only helper above. - | raise ValueError( + | return self._read_media_upload( + | self.AUDIO_INPUT, | "Unsupported audio input. Upload an audio file (sent as a data URL) " | "or provide a public https audio URL." | ) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/codegen/MediaGenCodegen.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/codegen/MediaGenCodegen.scala index 73047da89c3..feb7b1b4fcf 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/codegen/MediaGenCodegen.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/codegen/MediaGenCodegen.scala @@ -40,24 +40,9 @@ object MediaGenCodegen extends TaskCodegen { """ payload = {"inputs": prompt_value}""".stripMargin override def parsePython(ctx: CodegenContext): String = - """ if task == "text-to-image": + s""" if task == "text-to-image": | if isinstance(body, dict): - | if "output" in body: - | out = body["output"] - | url = out[0] if isinstance(out, list) else out - | if isinstance(url, str) and url.startswith("http"): - | return self._url_to_data_url(url) - | if "images" in body: - | images = body["images"] - | if images and isinstance(images[0], dict) and "url" in images[0]: - | return self._url_to_data_url(images[0]["url"]) - | if "data" in body: - | data = body["data"] - | if isinstance(data, dict) and "outputs" in data: - | outputs = data["outputs"] - | if outputs and isinstance(outputs[0], str) and outputs[0].startswith("http"): - | return self._url_to_data_url(outputs[0]) - | if isinstance(data, list) and data and isinstance(data[0], dict): + |${BinaryResponseFragments.imageEnvelope} | if "b64_json" in data[0]: | return f"data:image/png;base64,{data[0]['b64_json']}" | if "url" in data[0]: @@ -65,11 +50,7 @@ object MediaGenCodegen extends TaskCodegen { | return json.dumps(body) | elif task == "text-to-video": | if isinstance(body, dict): - | if "output" in body: - | out = body["output"] - | url = out[0] if isinstance(out, list) else out - | if isinstance(url, str) and url.startswith("http"): - | return self._url_to_data_url(url) + |${BinaryResponseFragments.outputUrl} | if "video" in body: | video = body["video"] | if isinstance(video, dict) and "url" in video: diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/codegen/QaRankingCodegen.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/codegen/QaRankingCodegen.scala index d4b4da95f17..2e80a70c1de 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/codegen/QaRankingCodegen.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/codegen/QaRankingCodegen.scala @@ -39,7 +39,7 @@ object QaRankingCodegen extends TaskCodegen { ) override def payloadPython(ctx: CodegenContext): String = - """ if task == "question-answering": + s""" if task == "question-answering": | ctx_val = row[self.CONTEXT_COLUMN] | ctx_val = "" if pd.isna(ctx_val) else str(ctx_val) | payload = {"inputs": {"question": prompt_value, "context": ctx_val}} @@ -52,48 +52,19 @@ object QaRankingCodegen extends TaskCodegen { | "parameters": {"candidate_labels": labels}, | } | elif task == "sentence-similarity": - | sent_val = row[self.SENTENCES_COLUMN] - | sent_val = "" if pd.isna(sent_val) else str(sent_val) - | sentences_list = [s.strip() for s in sent_val.split(",") if s.strip()] - | payload = { - | "inputs": { - | "source_sentence": prompt_value, - | "sentences": sentences_list, - | } - | } + |${TextResponsePython.rankingPayload("source_sentence", "sentences")} | elif task == "text-ranking": - | sent_val = row[self.SENTENCES_COLUMN] - | sent_val = "" if pd.isna(sent_val) else str(sent_val) - | sentences_list = [s.strip() for s in sent_val.split(",") if s.strip()] - | payload = { - | "inputs": { - | "query": prompt_value, - | "texts": sentences_list, - | } - | } + |${TextResponsePython.rankingPayload("query", "texts")} | else: | payload = {"inputs": prompt_value}""".stripMargin override def parsePython(ctx: CodegenContext): String = - """ if task == "question-answering": + s""" if task == "question-answering": | # Third-party chat providers answer via choices[0].message; | # hf-inference returns the native {"answer": ...} shape. - | content = self._chat_message_content(body) - | if content is not None: - | return content - | if isinstance(body, dict): - | return body.get("answer", json.dumps(body)) - | return json.dumps(body) + |${TextResponsePython.chatOrAnswer} | elif task == "table-question-answering": - | content = self._chat_message_content(body) - | if content is not None: - | return content - | if isinstance(body, dict): - | return body.get("answer", json.dumps(body)) - | return json.dumps(body) + |${TextResponsePython.chatOrAnswer} | elif task in ("zero-shot-classification", "sentence-similarity", "text-ranking"): - | content = self._chat_message_content(body) - | if content is not None: - | return content - | return json.dumps(body)""".stripMargin + |${TextResponsePython.chatOrBody}""".stripMargin } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/codegen/TaskCodegen.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/codegen/TaskCodegen.scala index 3e40a2085ee..a621073889d 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/codegen/TaskCodegen.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/codegen/TaskCodegen.scala @@ -92,3 +92,55 @@ trait TaskCodegen { */ def parsePython(ctx: CodegenContext): String } + +private[codegen] object BinaryResponseFragments { + val outputUrl: String = + """ if "output" in body: + | out = body["output"] + | url = out[0] if isinstance(out, list) else out + | if isinstance(url, str) and url.startswith("http"): + | return self._url_to_data_url(url)""".stripMargin + + // Each PNG handler continues this envelope with its own base64 validation. + val imageEnvelope: String = + s"""$outputUrl + | if "images" in body: + | images = body["images"] + | if images and isinstance(images[0], dict) and "url" in images[0]: + | return self._url_to_data_url(images[0]["url"]) + | if "data" in body: + | data = body["data"] + | if isinstance(data, dict) and "outputs" in data: + | outputs = data["outputs"] + | if outputs and isinstance(outputs[0], str) and outputs[0].startswith("http"): + | return self._url_to_data_url(outputs[0]) + | if isinstance(data, list) and data and isinstance(data[0], dict):""".stripMargin +} + +private[codegen] object TextResponsePython { + private val chatContent: String = + """ content = self._chat_message_content(body) + | if content is not None: + | return content""".stripMargin + + val chatOrAnswer: String = + s"""$chatContent + | if isinstance(body, dict): + | return body.get("answer", json.dumps(body)) + | return json.dumps(body)""".stripMargin + + val chatOrBody: String = + s"""$chatContent + | return json.dumps(body)""".stripMargin + + def rankingPayload(promptKey: String, sentencesKey: String): String = + s""" sent_val = row[self.SENTENCES_COLUMN] + | sent_val = "" if pd.isna(sent_val) else str(sent_val) + | sentences_list = [s.strip() for s in sent_val.split(",") if s.strip()] + | payload = { + | "inputs": { + | "$promptKey": prompt_value, + | "$sentencesKey": sentences_list, + | } + | }""".stripMargin +} diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/codegen/TextGenCodegen.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/codegen/TextGenCodegen.scala index b1650d5aab8..0f85cda6bd2 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/codegen/TextGenCodegen.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/huggingFace/codegen/TextGenCodegen.scala @@ -51,9 +51,6 @@ object TextGenCodegen extends TaskCodegen { | payload = {"inputs": prompt_value}""".stripMargin override def parsePython(ctx: CodegenContext): String = - """ if task == "text-generation": - | content = self._chat_message_content(body) - | if content is not None: - | return content - | return json.dumps(body)""".stripMargin + s""" if task == "text-generation": + |${TextResponsePython.chatOrBody}""".stripMargin } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpExec.scala index 9aa62a48055..428834c3af3 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpExec.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpExec.scala @@ -97,44 +97,36 @@ class IntervalJoinOpExec(descString: String) extends OperatorExecutor { } + private def pruneCache(cache: ListBuffer[Tuple])(tooSmall: Tuple => Boolean): Unit = { + while (cache.nonEmpty && tooSmall(cache.head)) { + cache.remove(0) + } + } + //if right table has tuple smaller than smallest tuple in left table, delete it private def removeTooSmallTupleInRightCache(leftTableSmallestTuple: Tuple): Unit = { - while (rightTable.nonEmpty) { - if ( - intervalCompare( - leftTableSmallestTuple.getField(desc.leftAttributeName), - rightTable.head.getField(desc.rightAttributeName), - leftTableSmallestTuple.getSchema - .getAttribute(desc.leftAttributeName) - .getType - ) > 0 - ) { - rightTable.remove(0) - } else { - return - } + pruneCache(rightTable) { cached => + intervalCompare( + leftTableSmallestTuple.getField(desc.leftAttributeName), + cached.getField(desc.rightAttributeName), + leftTableSmallestTuple.getSchema + .getAttribute(desc.leftAttributeName) + .getType + ) > 0 } - } //if left table has tuple smaller than smallest tuple in right table, delete it private def removeTooSmallTupleInLeftCache(rightTableSmallestTuple: Tuple): Unit = { - while (leftTable.nonEmpty) { - if ( - intervalCompare( - leftTable.head.getField(desc.leftAttributeName), - rightTableSmallestTuple.getField(desc.rightAttributeName), - rightTableSmallestTuple.getSchema - .getAttribute(desc.rightAttributeName) - .getType - ) < 0 - ) { - leftTable.remove(0) - } else { - return - } + pruneCache(leftTable) { cached => + intervalCompare( + cached.getField(desc.leftAttributeName), + rightTableSmallestTuple.getField(desc.rightAttributeName), + rightTableSmallestTuple.getSchema + .getAttribute(desc.rightAttributeName) + .getType + ) < 0 } - } private def processNumValue[T]( @@ -142,23 +134,16 @@ class IntervalJoinOpExec(descString: String) extends OperatorExecutor { leftBoundValue: T, rightBoundValue: T )(implicit ev$1: T => Ordered[T]): Int = { - if (desc.includeLeftBound && desc.includeRightBound) { - if (pointValue >= leftBoundValue && pointValue <= rightBoundValue) 0 - else if (pointValue < leftBoundValue) -1 - else 1 - } else if (desc.includeLeftBound && !desc.includeRightBound) { - if (pointValue >= leftBoundValue && pointValue < rightBoundValue) 0 - else if (pointValue < leftBoundValue) -1 - else 1 - } else if (!desc.includeLeftBound && desc.includeRightBound) { - if (pointValue > leftBoundValue && pointValue <= rightBoundValue) 0 - else if (pointValue <= leftBoundValue) -1 - else 1 - } else { - if (pointValue > leftBoundValue && pointValue < rightBoundValue) 0 - else if (pointValue <= leftBoundValue) -1 - else 1 - } + val afterLeft = + if (desc.includeLeftBound) pointValue >= leftBoundValue else pointValue > leftBoundValue + if ( + afterLeft && + (if (desc.includeRightBound) pointValue <= rightBoundValue else pointValue < rightBoundValue) + ) 0 + else if ( + if (desc.includeLeftBound) pointValue < leftBoundValue else pointValue <= leftBoundValue + ) -1 + else 1 } private def intervalCompare[K]( @@ -170,33 +155,23 @@ class IntervalJoinOpExec(descString: String) extends OperatorExecutor { if (dataType == AttributeType.LONG) { val pointValue: Long = point.asInstanceOf[Long] val leftBoundValue: Long = leftBound.asInstanceOf[Long] - val constantValue: Long = desc.constant - val rightBoundValue: Long = leftBoundValue + constantValue - result = processNumValue[Long]( - pointValue, - leftBoundValue, - rightBoundValue - ) + result = processNumValue[Long](pointValue, leftBoundValue, leftBoundValue + desc.constant) } else if (dataType == AttributeType.DOUBLE) { val pointValue: Double = point.asInstanceOf[Double] val leftBoundValue: Double = leftBound.asInstanceOf[Double] - val constantValue: Double = desc.constant.asInstanceOf[Double] - val rightBoundValue: Double = leftBoundValue + constantValue result = processNumValue[Double]( pointValue, leftBoundValue, - rightBoundValue + leftBoundValue + desc.constant.asInstanceOf[Double] ) } else if (dataType == AttributeType.INTEGER) { val pointValue: Int = point.asInstanceOf[Int] val leftBoundValue: Int = leftBound.asInstanceOf[Int] - val constantValue: Int = desc.constant.asInstanceOf[Int] - val rightBoundValue: Int = leftBoundValue + constantValue result = processNumValue[Int]( pointValue, leftBoundValue, - rightBoundValue + leftBoundValue + desc.constant.asInstanceOf[Int] ) } else if (dataType == AttributeType.TIMESTAMP) { val pointValue: Timestamp = point.asInstanceOf[Timestamp] diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/KNNTrainer/SklearnAdvancedKNNParameters.java b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/KNNTrainer/SklearnAdvancedKNNParameters.java index 29cd8ec18e9..c357a02b9b2 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/KNNTrainer/SklearnAdvancedKNNParameters.java +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/KNNTrainer/SklearnAdvancedKNNParameters.java @@ -20,52 +20,32 @@ package org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.KNNTrainer; import org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.base.ParamClass; +import org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.base.ParamSpec; public enum SklearnAdvancedKNNParameters implements ParamClass { // Bounds are scikit-learn's own: a neighbour count and a leaf size start at one, and the // Minkowski power is open at zero. - n_neighbors("n_neighbors", "int", "5") { @Override public String getMinimum() { return ">=1"; } }, - p("p", "int", "2") { @Override public String getMinimum() { return ">0"; } }, - weights("weights", "str", "", "uniform", "distance"), - algorithm("algorithm", "str", "", "auto", "ball_tree", "kd_tree", "brute"), - leaf_size("leaf_size", "int", "30") { @Override public String getMinimum() { return ">=1"; } }, + n_neighbors(ParamSpec.of("n_neighbors", "int", "5").withMinimum(">=1")), + p(ParamSpec.of("p", "int", "2").withMinimum(">0")), + weights(ParamSpec.of("weights", "str", "", "uniform", "distance")), + algorithm(ParamSpec.of("algorithm", "str", "", "auto", "ball_tree", "kd_tree", "brute")), + leaf_size(ParamSpec.of("leaf_size", "int", "30").withMinimum(">=1")), // A metric is named, not measured: "minkowski" and the rest are words, so the - // int() this used to declare rejected every value scikit-learn would take. The - // set is the one every `algorithm` above accepts -- the tree algorithms take - // fewer metrics than brute force, and naming a brute-only metric here would - // break the moment the sibling knob is moved off `auto`. - metric("metric", "str", "minkowski", "minkowski", "euclidean", "manhattan", - "chebyshev", "cityblock", "l1", "l2"), + // int() this used to declare rejected every value scikit-learn would take. + metric(ParamSpec.of("metric", "str", "minkowski", "minkowski", "euclidean", "manhattan", + "chebyshev", "cityblock", "l1", "l2")), // The only one that is not a scalar. scikit-learn wants a mapping of extra // keyword arguments for the metric, so the user's text is read as JSON. - metric_params("metric_params", "json.loads", "{}"); + metric_params(ParamSpec.of("metric_params", "json.loads", "{}")); - private final String name; - private final String type; - private final String sampleValue; - private final String[] allowedValues; + private final ParamSpec paramSpec; - SklearnAdvancedKNNParameters( - String name, String type, String sampleValue, String... allowedValues) { - this.name = name; - this.type = type; - this.sampleValue = sampleValue; - this.allowedValues = allowedValues; + SklearnAdvancedKNNParameters(ParamSpec paramSpec) { + this.paramSpec = paramSpec; } - public String getType() { - return this.type; - } - - public String getName() { - return this.name; - } - - public String getSampleValue() { - return this.sampleValue; - } - - public String[] getAllowedValues() { - return this.allowedValues.clone(); + @Override + public ParamSpec paramSpec() { + return this.paramSpec; } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/SVCTrainer/SklearnAdvancedSVCParameters.java b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/SVCTrainer/SklearnAdvancedSVCParameters.java index b85f4f6864c..522d182632e 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/SVCTrainer/SklearnAdvancedSVCParameters.java +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/SVCTrainer/SklearnAdvancedSVCParameters.java @@ -20,58 +20,33 @@ package org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.SVCTrainer; import org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.base.ParamClass; +import org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.base.ParamSpec; public enum SklearnAdvancedSVCParameters implements ParamClass { // Bounds are scikit-learn's own, whose ranges are open at zero for the two below and // closed for degree. - C("C", "float", "1.0") { @Override public String getMinimum() { return ">0"; } }, - kernel("kernel", "str", "", "rbf", "linear", "poly", "sigmoid", "precomputed"), - // gamma takes either of two words or a number, so no converter of a name covers it, and - // the pattern states what this one takes. Digits are [0-9] rather than \d so the browser - // and Python read it alike. It lets a negative through for the estimator to refuse, since - // excluding the sign would also exclude -0.0, which the estimator takes. - gamma( - "gamma", - "(lambda value: value.strip() if value.strip() in (\"scale\", \"auto\") else float(value))", - "scale") { - @Override - public String getPattern() { - return "^\\s*(?:scale|auto|[-+]?(?:(?:[0-9]+(?:_[0-9]+)*)?\\.(?:[0-9]+(?:_[0-9]+)*)" - + "|(?:[0-9]+(?:_[0-9]+)*)\\.?)(?:[eE][-+]?[0-9]+(?:_[0-9]+)*)?)\\s*$"; - } - }, - degree("degree", "int", "3") { @Override public String getMinimum() { return ">=0"; } }, + C(ParamSpec.of("C", "float", "1.0").withMinimum(">0")), + kernel(ParamSpec.of("kernel", "str", "", "rbf", "linear", "poly", "sigmoid", "precomputed")), + gamma(ParamSpec.of( + "gamma", + "(lambda value: value.strip() if value.strip() in (\"scale\", \"auto\") else float(value))", + "scale") + .withPattern(ParamSpec.GammaPattern())), + degree(ParamSpec.of("degree", "int", "3").withMinimum(">=0")), // coef0 is the one parameter here with no bound at either end. - coef0("coef0", "float", "0.0"), - tol("tol", "float", "0.001") { @Override public String getMinimum() { return ">0"; } }, - probability("probability", "(lambda value: value.lower() == \"true\")", "", "false", "true"); + coef0(ParamSpec.of("coef0", "float", "0.0")), + tol(ParamSpec.of("tol", "float", "0.001").withMinimum(">0")), + probability(ParamSpec.of( + "probability", "(lambda value: value.lower() == \"true\")", "", "false", "true")); - private final String name; - private final String type; - private final String sampleValue; - private final String[] allowedValues; + private final ParamSpec paramSpec; - SklearnAdvancedSVCParameters( - String name, String type, String sampleValue, String... allowedValues) { - this.name = name; - this.type = type; - this.sampleValue = sampleValue; - this.allowedValues = allowedValues; + SklearnAdvancedSVCParameters(ParamSpec paramSpec) { + this.paramSpec = paramSpec; } - public String getType() { - return this.type; - } - - public String getName() { - return this.name; - } - - public String getSampleValue() { - return this.sampleValue; - } - - public String[] getAllowedValues() { - return this.allowedValues.clone(); + @Override + public ParamSpec paramSpec() { + return this.paramSpec; } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/SVRTrainer/SklearnAdvancedSVRParameters.java b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/SVRTrainer/SklearnAdvancedSVRParameters.java index c631b95ee48..f6d08f49fbb 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/SVRTrainer/SklearnAdvancedSVRParameters.java +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/SVRTrainer/SklearnAdvancedSVRParameters.java @@ -20,59 +20,38 @@ package org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.SVRTrainer; import org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.base.ParamClass; +import org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.base.ParamSpec; public enum SklearnAdvancedSVRParameters implements ParamClass { // Bounds are scikit-learn's own; see SVC for the shared ones. - C("C", "float", "1.0") { @Override public String getMinimum() { return ">0"; } }, - kernel("kernel", "str", "", "rbf", "linear", "poly", "sigmoid", "precomputed"), - // Same converter and shape as SVC's gamma -- see there for why each is spelled this way. - gamma( - "gamma", - "(lambda value: value.strip() if value.strip() in (\"scale\", \"auto\") else float(value))", - "scale") { - @Override - public String getPattern() { - return "^\\s*(?:scale|auto|[-+]?(?:(?:[0-9]+(?:_[0-9]+)*)?\\.(?:[0-9]+(?:_[0-9]+)*)" - + "|(?:[0-9]+(?:_[0-9]+)*)\\.?)(?:[eE][-+]?[0-9]+(?:_[0-9]+)*)?)\\s*$"; - } - }, - degree("degree", "int", "3") { @Override public String getMinimum() { return ">=0"; } }, - coef0("coef0", "float", "0.0"), - tol("tol", "float", "0.001") { @Override public String getMinimum() { return ">0"; } }, - shrinking("shrinking", "(lambda value: value.lower() == \"true\")", "", "true", "false"), - verbose("verbose", "(lambda value: value.lower() == \"true\")", "", "false", "true"), - epsilon("epsilon", "float", "0.1") { @Override public String getMinimum() { return ">=0"; } }, - cache_size("cache_size", "int", "200") { @Override public String getMinimum() { return ">0"; } }, + C(ParamSpec.of("C", "float", "1.0").withMinimum(">0")), + kernel(ParamSpec.of("kernel", "str", "", "rbf", "linear", "poly", "sigmoid", "precomputed")), + gamma(ParamSpec.of( + "gamma", + "(lambda value: value.strip() if value.strip() in (\"scale\", \"auto\") else float(value))", + "scale") + .withPattern(ParamSpec.GammaPattern())), + degree(ParamSpec.of("degree", "int", "3").withMinimum(">=0")), + coef0(ParamSpec.of("coef0", "float", "0.0")), + tol(ParamSpec.of("tol", "float", "0.001").withMinimum(">0")), + shrinking(ParamSpec.of( + "shrinking", "(lambda value: value.lower() == \"true\")", "", "true", "false")), + verbose(ParamSpec.of( + "verbose", "(lambda value: value.lower() == \"true\")", "", "false", "true")), + epsilon(ParamSpec.of("epsilon", "float", "0.1").withMinimum(">=0")), + cache_size(ParamSpec.of("cache_size", "int", "200").withMinimum(">0")), // -1 is SVR's own value for no iteration limit, which is also why the bound is -1 and not // zero: the sentinel has to remain reachable. - max_iter("max_iter", "int", "-1") { @Override public String getMinimum() { return ">=-1"; } }; + max_iter(ParamSpec.of("max_iter", "int", "-1").withMinimum(">=-1")); - private final String name; - private final String type; - private final String sampleValue; - private final String[] allowedValues; + private final ParamSpec paramSpec; - SklearnAdvancedSVRParameters( - String name, String type, String sampleValue, String... allowedValues) { - this.name = name; - this.type = type; - this.sampleValue = sampleValue; - this.allowedValues = allowedValues; + SklearnAdvancedSVRParameters(ParamSpec paramSpec) { + this.paramSpec = paramSpec; } - public String getType() { - return this.type; - } - - public String getName() { - return this.name; - } - - public String getSampleValue() { - return this.sampleValue; - } - - public String[] getAllowedValues() { - return this.allowedValues.clone(); + @Override + public ParamSpec paramSpec() { + return this.paramSpec; } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/base/SklearnAdvancedBaseDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/base/SklearnAdvancedBaseDesc.scala index 7875a2fbb0a..e8f23d5f61e 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/base/SklearnAdvancedBaseDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/base/SklearnAdvancedBaseDesc.scala @@ -49,10 +49,43 @@ import scala.util.Try * decide which values the estimator can be given. The two below state that so the form can * hold the user to it, rather than leaving it to be discovered when `fit` raises. */ +final case class ParamSpec( + name: String, + valueType: String, + sampleValue: String = "", + allowedValues: Array[String] = Array.empty, + pattern: String = "", + minimum: String = "" +) { + def withPattern(pattern: String): ParamSpec = copy(pattern = pattern) + + def withMinimum(minimum: String): ParamSpec = copy(minimum = minimum) +} + +object ParamSpec { + final val Empty: ParamSpec = ParamSpec("", "") + + // Shared pattern for SVM gamma ("scale", "auto", or floating-point literal). + final val GammaPattern: String = + "^\\s*(?:scale|auto|[-+]?(?:(?:[0-9]+(?:_[0-9]+)*)?\\.(?:[0-9]+(?:_[0-9]+)*)" + + "|(?:[0-9]+(?:_[0-9]+)*)\\.?)(?:[eE][-+]?[0-9]+(?:_[0-9]+)*)?)\\s*$" + + @scala.annotation.varargs + def of( + name: String, + valueType: String, + sampleValue: String, + allowedValues: String* + ): ParamSpec = + ParamSpec(name, valueType, sampleValue, allowedValues.toArray) +} + trait ParamClass { - def getName: String + def paramSpec: ParamSpec = ParamSpec.Empty + + def getName: String = paramSpec.name - def getType: String + def getType: String = paramSpec.valueType /** One value this parameter accepts, offered as an example rather than imposed as a * default: what the estimator itself documents as its default is the obvious choice, but @@ -61,26 +94,26 @@ trait ParamClass { * estimator's own default is a value `getType` cannot convert, since an example the * operator would then reject is worse than none. */ - def getSampleValue: String + def getSampleValue: String = paramSpec.sampleValue /** The values the estimator accepts, where it accepts a fixed set rather than a range, the * estimator's own default first. Empty for a parameter taking any number, which its * converter already constrains. */ - def getAllowedValues: Array[String] + def getAllowedValues: Array[String] = paramSpec.allowedValues.clone() /** The shape the value takes, for a parameter that accepts neither a fixed set nor a plain * number but a choice between them. Empty for every parameter one of the other two * describes, which is most of them. */ - def getPattern: String = "" + def getPattern: String = paramSpec.pattern /** How low the value may go, written the way the estimator's own range reads: `">0"` where * zero itself is refused and `">=1"` where the bound is included. Empty for a parameter * bounded by nothing, which a number alone cannot be told from one whose bound nobody * looked up. */ - def getMinimum: String = "" + def getMinimum: String = paramSpec.minimum } // Every selected column goes into `fit` untouched, so each has to be one that diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpExec.scala index 6b45f40c3eb..adffa8533b0 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpExec.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpExec.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.sort import org.apache.texera.amber.core.executor.OperatorExecutor +import org.apache.texera.amber.core.tuple.AttributeTypeUtils import org.apache.texera.amber.core.tuple.{AttributeType, Schema, Tuple, TupleLike} import org.apache.texera.amber.util.JSONUtils.objectMapper @@ -243,34 +244,16 @@ class StableMergeSortOpExec(descString: String) extends OperatorExecutor { attrType: AttributeType ): Int = attrType match { - case AttributeType.INTEGER => - java.lang.Integer.compare( - leftValue.asInstanceOf[Number].intValue(), - rightValue.asInstanceOf[Number].intValue() - ) - case AttributeType.LONG => - java.lang.Long.compare( - leftValue.asInstanceOf[Number].longValue(), - rightValue.asInstanceOf[Number].longValue() - ) - case AttributeType.DOUBLE => - java.lang.Double.compare( - leftValue.asInstanceOf[Number].doubleValue(), - rightValue.asInstanceOf[Number].doubleValue() - ) - case AttributeType.BOOLEAN => - java.lang.Boolean.compare(leftValue.asInstanceOf[Boolean], rightValue.asInstanceOf[Boolean]) + case AttributeType.INTEGER | AttributeType.LONG | AttributeType.DOUBLE | + AttributeType.BOOLEAN | AttributeType.BINARY => + AttributeTypeUtils.compare(leftValue, rightValue, attrType) + // Keep timestamp precision and strict strings: the shared helper differs here. case AttributeType.TIMESTAMP => leftValue .asInstanceOf[java.sql.Timestamp] .compareTo(rightValue.asInstanceOf[java.sql.Timestamp]) case AttributeType.STRING => leftValue.asInstanceOf[String].compareTo(rightValue.asInstanceOf[String]) - case AttributeType.BINARY => - java.util.Arrays.compareUnsigned( - leftValue.asInstanceOf[Array[Byte]], - rightValue.asInstanceOf[Array[Byte]] - ) case other => throw new IllegalStateException(s"Unsupported attribute type $other in StableMergeSort") } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/apis/twitter/TwitterSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/apis/twitter/TwitterSourceOpDesc.scala index 223b9bbfff3..9678943431e 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/apis/twitter/TwitterSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/apis/twitter/TwitterSourceOpDesc.scala @@ -21,9 +21,13 @@ package org.apache.texera.amber.operator.source.apis.twitter import com.fasterxml.jackson.annotation.{JsonIgnore, JsonProperty} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaDescription, JsonSchemaTitle} -import org.apache.texera.amber.core.workflow.OutputPort +import org.apache.texera.amber.core.executor.OpExecWithClassName +import org.apache.texera.amber.core.tuple.{AttributeType, Schema} +import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} +import org.apache.texera.amber.core.workflow.{OutputPort, PhysicalOp, SchemaPropagationFunc} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.operator.source.SourceOperatorDescriptor +import org.apache.texera.amber.util.JSONUtils.objectMapper @deprecated("Twitter source operator is no longer executable.", "1.1.0-incubating") abstract class TwitterSourceOpDesc extends SourceOperatorDescriptor { @@ -54,4 +58,63 @@ abstract class TwitterSourceOpDesc extends SourceOperatorDescriptor { ) } + protected def tweetSourcePhysicalOp( + workflowId: WorkflowIdentity, + executionId: ExecutionIdentity, + executorClassName: String + ): PhysicalOp = + // TODO: use multiple workers + PhysicalOp + .sourcePhysicalOp( + workflowId, + executionId, + operatorIdentifier, + OpExecWithClassName(executorClassName, objectMapper.writeValueAsString(this)) + ) + .withInputPorts(operatorInfo.inputPorts) + .withOutputPorts(operatorInfo.outputPorts) + .withPropagateSchema( + SchemaPropagationFunc(_ => Map(operatorInfo.outputPorts.head.id -> sourceSchema())) + ) + + override def sourceSchema(): Schema = TweetSchema.create() +} + +private[twitter] object TweetSchema { + def create(): Schema = { + Schema() + .add("id", AttributeType.STRING) + .add("text", AttributeType.STRING) + .add("created_at", AttributeType.TIMESTAMP) + .add("lang", AttributeType.STRING) + .add("tweet_type", AttributeType.STRING) + .add("place_id", AttributeType.STRING) + .add("place_coordinate", AttributeType.STRING) + .add("in_reply_to_status_id", AttributeType.STRING) + .add("in_reply_to_user_id", AttributeType.STRING) + .add("like_count", AttributeType.LONG) + .add("quote_count", AttributeType.LONG) + .add("reply_count", AttributeType.LONG) + .add("retweet_count", AttributeType.LONG) + .add("hashtags", AttributeType.STRING) + .add("symbols", AttributeType.STRING) + .add("urls", AttributeType.STRING) + .add("mentions", AttributeType.STRING) + .add("user_id", AttributeType.STRING) + .add("user_created_at", AttributeType.TIMESTAMP) + .add("user_name", AttributeType.STRING) + .add("user_display_name", AttributeType.STRING) + .add("user_lang", AttributeType.STRING) + .add("user_description", AttributeType.STRING) + .add("user_followers_count", AttributeType.LONG) + .add("user_following_count", AttributeType.LONG) + .add("user_tweet_count", AttributeType.LONG) + .add("user_listed_count", AttributeType.LONG) + .add("user_location", AttributeType.STRING) + .add("user_url", AttributeType.STRING) + .add("user_profile_image_url", AttributeType.STRING) + .add("user_pinned_tweet_id", AttributeType.STRING) + .add("user_protected", AttributeType.BOOLEAN) + .add("user_verified", AttributeType.BOOLEAN) + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/apis/twitter/TwitterSourceOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/apis/twitter/TwitterSourceOpExec.scala index 9577c1f711e..b0312c62204 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/apis/twitter/TwitterSourceOpExec.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/apis/twitter/TwitterSourceOpExec.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.source.apis.twitter import org.apache.texera.amber.core.executor.SourceOperatorExecutor +import org.apache.texera.amber.core.tuple.TupleLike @deprecated("Twitter source operator is no longer executable.", "1.1.0-incubating") abstract class TwitterSourceOpExec( @@ -31,4 +32,6 @@ abstract class TwitterSourceOpExec( ) override def close(): Unit = {} + + override def produceTuple(): Iterator[TupleLike] = Iterator.empty } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/apis/twitter/v2/TwitterFullArchiveSearchSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/apis/twitter/v2/TwitterFullArchiveSearchSourceOpDesc.scala index 2e5722635d8..d156a94b54b 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/apis/twitter/v2/TwitterFullArchiveSearchSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/apis/twitter/v2/TwitterFullArchiveSearchSourceOpDesc.scala @@ -25,13 +25,10 @@ import com.kjetland.jackson.jsonSchema.annotations.{ JsonSchemaInject, JsonSchemaTitle } -import org.apache.texera.amber.core.executor.OpExecWithClassName -import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} -import org.apache.texera.amber.core.workflow.{PhysicalOp, SchemaPropagationFunc} +import org.apache.texera.amber.core.workflow.PhysicalOp import org.apache.texera.amber.operator.metadata.annotations.UIWidget import org.apache.texera.amber.operator.source.apis.twitter.TwitterSourceOpDesc -import org.apache.texera.amber.util.JSONUtils.objectMapper @deprecated("Twitter source operator is no longer executable.", "1.1.0-incubating") class TwitterFullArchiveSearchSourceOpDesc extends TwitterSourceOpDesc { @@ -64,60 +61,9 @@ class TwitterFullArchiveSearchSourceOpDesc extends TwitterSourceOpDesc { workflowId: WorkflowIdentity, executionId: ExecutionIdentity ): PhysicalOp = - // TODO: use multiple workers - PhysicalOp - .sourcePhysicalOp( - workflowId, - executionId, - operatorIdentifier, - OpExecWithClassName( - "org.apache.texera.amber.operator.source.apis.twitter.v2.TwitterFullArchiveSearchSourceOpExec", - objectMapper.writeValueAsString(this) - ) - ) - .withInputPorts(operatorInfo.inputPorts) - .withOutputPorts(operatorInfo.outputPorts) - .withPropagateSchema( - SchemaPropagationFunc(_ => Map(operatorInfo.outputPorts.head.id -> sourceSchema())) - ) - - override def sourceSchema(): Schema = { - - // twitter schema is hard coded for now. V2 API has changed many fields of the Tweet object. - // we are also currently depending on redouane59/twittered client library to parse tweet fields. - Schema() - .add("id", AttributeType.STRING) - .add("text", AttributeType.STRING) - .add("created_at", AttributeType.TIMESTAMP) - .add("lang", AttributeType.STRING) - .add("tweet_type", AttributeType.STRING) - .add("place_id", AttributeType.STRING) - .add("place_coordinate", AttributeType.STRING) - .add("in_reply_to_status_id", AttributeType.STRING) - .add("in_reply_to_user_id", AttributeType.STRING) - .add("like_count", AttributeType.LONG) - .add("quote_count", AttributeType.LONG) - .add("reply_count", AttributeType.LONG) - .add("retweet_count", AttributeType.LONG) - .add("hashtags", AttributeType.STRING) - .add("symbols", AttributeType.STRING) - .add("urls", AttributeType.STRING) - .add("mentions", AttributeType.STRING) - .add("user_id", AttributeType.STRING) - .add("user_created_at", AttributeType.TIMESTAMP) - .add("user_name", AttributeType.STRING) - .add("user_display_name", AttributeType.STRING) - .add("user_lang", AttributeType.STRING) - .add("user_description", AttributeType.STRING) - .add("user_followers_count", AttributeType.LONG) - .add("user_following_count", AttributeType.LONG) - .add("user_tweet_count", AttributeType.LONG) - .add("user_listed_count", AttributeType.LONG) - .add("user_location", AttributeType.STRING) - .add("user_url", AttributeType.STRING) - .add("user_profile_image_url", AttributeType.STRING) - .add("user_pinned_tweet_id", AttributeType.STRING) - .add("user_protected", AttributeType.BOOLEAN) - .add("user_verified", AttributeType.BOOLEAN) - } + tweetSourcePhysicalOp( + workflowId, + executionId, + "org.apache.texera.amber.operator.source.apis.twitter.v2.TwitterFullArchiveSearchSourceOpExec" + ) } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/apis/twitter/v2/TwitterFullArchiveSearchSourceOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/apis/twitter/v2/TwitterFullArchiveSearchSourceOpExec.scala index 64fa77e6cca..59a0a321f80 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/apis/twitter/v2/TwitterFullArchiveSearchSourceOpExec.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/apis/twitter/v2/TwitterFullArchiveSearchSourceOpExec.scala @@ -19,12 +19,9 @@ package org.apache.texera.amber.operator.source.apis.twitter.v2 -import org.apache.texera.amber.core.tuple.TupleLike import org.apache.texera.amber.operator.source.apis.twitter.TwitterSourceOpExec @deprecated("Twitter source operator is no longer executable.", "1.1.0-incubating") class TwitterFullArchiveSearchSourceOpExec( descString: String -) extends TwitterSourceOpExec(descString) { - override def produceTuple(): Iterator[TupleLike] = Iterator.empty -} +) extends TwitterSourceOpExec(descString) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/apis/twitter/v2/TwitterSearchSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/apis/twitter/v2/TwitterSearchSourceOpDesc.scala index c2ae5170d24..adce6b54056 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/apis/twitter/v2/TwitterSearchSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/apis/twitter/v2/TwitterSearchSourceOpDesc.scala @@ -25,13 +25,10 @@ import com.kjetland.jackson.jsonSchema.annotations.{ JsonSchemaInject, JsonSchemaTitle } -import org.apache.texera.amber.core.executor.OpExecWithClassName -import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} -import org.apache.texera.amber.core.workflow.{PhysicalOp, SchemaPropagationFunc} +import org.apache.texera.amber.core.workflow.PhysicalOp import org.apache.texera.amber.operator.metadata.annotations.UIWidget import org.apache.texera.amber.operator.source.apis.twitter.TwitterSourceOpDesc -import org.apache.texera.amber.util.JSONUtils.objectMapper @deprecated("Twitter source operator is no longer executable.", "1.1.0-incubating") class TwitterSearchSourceOpDesc extends TwitterSourceOpDesc { @@ -54,61 +51,9 @@ class TwitterSearchSourceOpDesc extends TwitterSourceOpDesc { workflowId: WorkflowIdentity, executionId: ExecutionIdentity ): PhysicalOp = - // TODO: use multiple workers - PhysicalOp - .sourcePhysicalOp( - workflowId, - executionId, - operatorIdentifier, - OpExecWithClassName( - "org.apache.texera.amber.operator.source.apis.twitter.v2.TwitterSearchSourceOpExec", - objectMapper.writeValueAsString(this) - ) - ) - .withInputPorts(operatorInfo.inputPorts) - .withOutputPorts(operatorInfo.outputPorts) - .withPropagateSchema( - SchemaPropagationFunc(_ => Map(operatorInfo.outputPorts.head.id -> sourceSchema())) - ) - - override def sourceSchema(): Schema = { - - // twitter schema is hard coded for now. V2 API has changed many fields of the Tweet object. - // we are also currently depending on redouane59/twittered client library to parse tweet fields. - - Schema() - .add("id", AttributeType.STRING) - .add("text", AttributeType.STRING) - .add("created_at", AttributeType.TIMESTAMP) - .add("lang", AttributeType.STRING) - .add("tweet_type", AttributeType.STRING) - .add("place_id", AttributeType.STRING) - .add("place_coordinate", AttributeType.STRING) - .add("in_reply_to_status_id", AttributeType.STRING) - .add("in_reply_to_user_id", AttributeType.STRING) - .add("like_count", AttributeType.LONG) - .add("quote_count", AttributeType.LONG) - .add("reply_count", AttributeType.LONG) - .add("retweet_count", AttributeType.LONG) - .add("hashtags", AttributeType.STRING) - .add("symbols", AttributeType.STRING) - .add("urls", AttributeType.STRING) - .add("mentions", AttributeType.STRING) - .add("user_id", AttributeType.STRING) - .add("user_created_at", AttributeType.TIMESTAMP) - .add("user_name", AttributeType.STRING) - .add("user_display_name", AttributeType.STRING) - .add("user_lang", AttributeType.STRING) - .add("user_description", AttributeType.STRING) - .add("user_followers_count", AttributeType.LONG) - .add("user_following_count", AttributeType.LONG) - .add("user_tweet_count", AttributeType.LONG) - .add("user_listed_count", AttributeType.LONG) - .add("user_location", AttributeType.STRING) - .add("user_url", AttributeType.STRING) - .add("user_profile_image_url", AttributeType.STRING) - .add("user_pinned_tweet_id", AttributeType.STRING) - .add("user_protected", AttributeType.BOOLEAN) - .add("user_verified", AttributeType.BOOLEAN) - } + tweetSourcePhysicalOp( + workflowId, + executionId, + "org.apache.texera.amber.operator.source.apis.twitter.v2.TwitterSearchSourceOpExec" + ) } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/apis/twitter/v2/TwitterSearchSourceOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/apis/twitter/v2/TwitterSearchSourceOpExec.scala index e31c1100380..1cb7410b9a2 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/apis/twitter/v2/TwitterSearchSourceOpExec.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/apis/twitter/v2/TwitterSearchSourceOpExec.scala @@ -19,12 +19,9 @@ package org.apache.texera.amber.operator.source.apis.twitter.v2 -import org.apache.texera.amber.core.tuple.TupleLike import org.apache.texera.amber.operator.source.apis.twitter.TwitterSourceOpExec @deprecated("Twitter source operator is no longer executable.", "1.1.0-incubating") class TwitterSearchSourceOpExec( descString: String -) extends TwitterSourceOpExec(descString) { - override def produceTuple(): Iterator[TupleLike] = Iterator.empty -} +) extends TwitterSourceOpExec(descString) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala index 6aa03de0acc..93668f0c78f 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala @@ -22,28 +22,19 @@ package org.apache.texera.amber.operator.source.scan.csv import com.fasterxml.jackson.annotation.{JsonInclude, JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import com.univocity.parsers.csv.{CsvFormat, CsvParser, CsvParserSettings} -import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.storage.DocumentFactory import org.apache.texera.amber.core.tuple.AttributeTypeUtils.inferSchemaFromRows import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} -import org.apache.texera.amber.core.workflow.{PhysicalOp, SchemaPropagationFunc} -import org.apache.texera.amber.operator.{StandaloneCodeGenerator, StandaloneHelpers} -import org.apache.texera.amber.operator.StandaloneCodeGenerator.SourceFilePlaceholder +import org.apache.texera.amber.core.workflow.PhysicalOp import org.apache.texera.amber.operator.metadata.annotations.UIWidget import org.apache.texera.amber.operator.source.scan.ScanSourceOpDesc -import org.apache.texera.amber.operator.source.scan.csv.CSVScanSourceOpExec -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.{StandaloneCodeGenerator, StandaloneHelpers} import java.io.{IOException, InputStreamReader} -import java.net.URI -import scala.util.Try class CSVScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator { - // One character: every reader narrows this with charAt(0), because univocity's - // setDelimiter and scala-csv's DefaultCSVFormat both take a Char. @JsonProperty(defaultValue = ",") @JsonSchemaTitle("Delimiter") @JsonPropertyDescription("single character separating the fields on each line") @@ -63,45 +54,21 @@ class CSVScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator workflowId: WorkflowIdentity, executionId: ExecutionIdentity ): PhysicalOp = { - // fill in default values if (customDelimiter.forall(_.isEmpty)) { customDelimiter = Option(",") } - - PhysicalOp - .sourcePhysicalOp( - workflowId, - executionId, - operatorIdentifier, - OpExecWithClassName( - "org.apache.texera.amber.operator.source.scan.csv.CSVScanSourceOpExec", - objectMapper.writeValueAsString(this) - ) - ) - .withInputPorts(operatorInfo.inputPorts) - .withOutputPorts(operatorInfo.outputPorts) - .withPropagateSchema( - SchemaPropagationFunc(_ => Map(operatorInfo.outputPorts.head.id -> sourceSchema())) - ) + CsvScanSupport.csvPhysicalOp( + this, + workflowId, + executionId, + "org.apache.texera.amber.operator.source.scan.csv.CSVScanSourceOpExec" + ) } override def sourceSchema(): Schema = { - val delimiterChar = customDelimiter.filter(_.nonEmpty).getOrElse(",").charAt(0) - require( - fileResolved(), - "No file selected. Please select a valid .csv file from the 'File' dropdown in the right panel." - ) - - val uri = new URI(fileName.get) - if (uri.getScheme == "file") { - require( - new java.io.File(uri).isFile, - "The selected item is a folder or does not exist. Please select an actual .csv file from the 'File' dropdown." - ) - } + val (uri, delimiterChar) = CsvScanSupport.resolveCsvUriAndDelimiter(this, customDelimiter) val stream = DocumentFactory.openReadonlyDocument(uri).asInputStream() - val inputReader = - new InputStreamReader(stream, fileEncoding.getCharset) + val inputReader = new InputStreamReader(stream, fileEncoding.getCharset) val csvFormat = new CsvFormat() csvFormat.setDelimiter(delimiterChar) @@ -112,10 +79,6 @@ class CSVScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator csvSetting.setMaxColumns(maxColumns) csvSetting.setFormat(csvFormat) csvSetting.setHeaderExtractionEnabled(hasHeader) - // No setNullValue here, so a blank cell reads as null exactly as it does at - // execution time (CSVScanSourceOpExec builds its parser without one). Reading it - // as "" instead made inferField fall through to STRING, which typed a numeric - // column by its one empty cell rather than by its values. val parser = new CsvParser(csvSetting) parser.beginParsing(inputReader) @@ -141,112 +104,29 @@ class CSVScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator else (1 to attributeTypeList.length).map(i => "column-" + i).toArray header.indices.foldLeft(Schema()) { (schema, i) => - // Auto-rename blank header positions to `column-N` so empty CSV headers - // (e.g. a trailing comma) do not propagate empty attribute names to - // downstream Iceberg/Parquet writers, which reject them. val name = Option(header(i)).filter(_.nonEmpty).getOrElse(s"column-${i + 1}") schema.add(name, attributeTypeList(i)) } - } override def standaloneSourcePath(): Option[String] = fileName override def standaloneHelpers(): Seq[String] = Seq(StandaloneHelpers.AttributeCasts) - override def generateStandaloneCode(): String = { - // Resolve the delimiter the same way the parser above does — first character, empty - // means comma — and escape it. Every value the field accepts has to survive this: - // pandas reads a separator longer than one character as a REGULAR EXPRESSION, and a - // backslash spliced raw produced `sep="\"`, which is not valid Python at all. - val sep = customDelimiter.filter(_.nonEmpty).getOrElse(",").charAt(0).toString - // Inferred once: each inference reads the file again. - val schema = Try(sourceSchema()).toOption - // Texera's encoding enum uses values like UTF_8; pandas expects utf-8. - val encoding = fileEncoding.toString.replace("_", "-").toLowerCase - val headerArg = if (hasHeader) "0" else "None" - - val args = scala.collection.mutable.ArrayBuffer[String]() - args += s"filepath_or_buffer=$SourceFilePlaceholder" - args += s"sep=${pyStringLiteral(sep)}" - args += s"""encoding=${pyStringLiteral(encoding)}""" - args += s"header=$headerArg" - - // The parser above sets no null value, so only an empty field is null and every other - // text stands for itself. pandas instead reads a list of words as missing by default, - // "NA" and "null" among them, which turned a column holding the country code NA into - // nulls. Both halves are needed: dropping the default list stops the words, and naming - // the empty string keeps the blank cell null. - args += "keep_default_na=False" - args += """na_values=[""]""" - - // A column holding a null has to be asked for, or pandas widens it to carry - // the hole: an integer through a float, where a LONG past 2^53 comes back - // rounded (9007199254740993 as ...992) and every INTEGER is left a float - // that prints 3 as 3.0, and a boolean to an object column. The nullable - // dtypes carry the hole and keep the type the schema declared. A timestamp - // is read as its text and parsed below, the way the engine parses it. - // - // By position, header or not, because the schema's names are not pandas' - // until the rename below: a blank header is `column-2` here and `Unnamed: 1` - // there, and asking for `column-2` ended the read. pandas takes an integer - // here as a position even where a header spells one. A schema that cannot be - // read (an unresolved file) leaves the argument off rather than failing the - // export. - val nullableDtypes = Map( - AttributeType.INTEGER -> "Int32", - AttributeType.LONG -> "Int64", - AttributeType.BOOLEAN -> "boolean", - AttributeType.TIMESTAMP -> "object" - ) - val typedColumns: Seq[String] = - schema.toSeq.flatMap( - _.getAttributes.zipWithIndex.flatMap { - case (attribute, i) => nullableDtypes.get(attribute.getType).map(d => s"""$i: "$d"""") - } - ) - if (typedColumns.nonEmpty) args += s"dtype={${typedColumns.mkString(", ")}}" - - // Clamped: the property editor refuses a negative, but a plan posted to the API - // can still carry one, and pandas rejects a negative `nrows` outright where the - // executor's `take` simply keeps no rows. - offset.map(_.max(0)).foreach { o => - // With a header, skip offset rows after row 0; without, skip offset rows from the start. - // The end is counted in Long: the largest offset the operator accepts - // overflows an Int on the way past the header, and the range came out - // empty, skipping nothing where the executor's `drop` keeps no rows. Asked - // of each line rather than listed, since pandas makes a set of a list of - // rows to skip, and one of two billion ran the script out of memory. - if (hasHeader) args += s"skiprows=lambda _i: 0 < _i <= ${o.toLong}" - else args += s"skiprows=$o" - } - limit.map(_.max(0)).foreach(l => args += s"nrows=$l") - - // Without a header, an offset at or past the last row left pandas no line to - // count the columns from, and it raised EmptyDataError where the executor - // emits no rows, so the rename and the typing below never ran. Naming the - // columns by position gives it the count and keeps the dtype keys above. - if (!hasHeader) - schema.map(_.getAttributes.size).filter(_ > 0).foreach { n => - args += s"names=[${(0 until n).mkString(", ")}]" - } - - val readCall = s"out1df = pd.read_csv(${args.mkString(", ")})" - - // The schema's own names, which every downstream operator was configured - // against. They differ from pandas' in both directions: a blank header is - // `column-2` here and `Unnamed: 1` there, and a header the user really did - // spell `Unnamed: 1` is kept. Matching the placeholder against the index - // cannot tell those two apart when they coincide; taking the names by - // position can. - schema.filter(_.getAttributes.nonEmpty) match { - case Some(declared) => + override def generateStandaloneCode(): String = + CsvScanSupport.generateStandaloneCode( + this, + customDelimiter, + hasHeader, + dtypeByType = Map( + AttributeType.INTEGER -> "Int32", + AttributeType.LONG -> "Int64", + AttributeType.BOOLEAN -> "boolean", + AttributeType.TIMESTAMP -> "object" + ), + includeNaValuesBlank = true, + includeWindow = true, + declaredSchemaBody = (readCall, declared) => s"$readCall\n${StandaloneCodeGenerator.readAsDeclared("out1df", declared)}" - case None if hasHeader => readCall - case None => - // Unresolved file: fall back to Texera's headerless naming. - s"""$readCall - |out1df.columns = [f"column-{i + 1}" for i in range(len(out1df.columns))]""".stripMargin - } - } + ) } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CsvScanSupport.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CsvScanSupport.scala new file mode 100644 index 00000000000..9c26e24ed70 --- /dev/null +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CsvScanSupport.scala @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.operator.source.scan.csv + +import org.apache.texera.amber.core.executor.OpExecWithClassName +import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema} +import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} +import org.apache.texera.amber.core.workflow.{PhysicalOp, SchemaPropagationFunc} +import org.apache.texera.amber.operator.StandaloneCodeGenerator.SourceFilePlaceholder +import org.apache.texera.amber.operator.source.scan.ScanSourceOpDesc +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral +import org.apache.texera.amber.util.JSONUtils.objectMapper + +import java.net.URI +import scala.util.Try + +/** Shared physical-op wiring, URI/delimiter resolution, schema building, and standalone codegen for CSV scans. */ +private[source] object CsvScanSupport { + + def resolveDelimiterChar(customDelimiter: Option[String]): String = + customDelimiter.filter(_.nonEmpty).getOrElse(",").charAt(0).toString + + def csvPhysicalOp( + desc: ScanSourceOpDesc, + workflowId: WorkflowIdentity, + executionId: ExecutionIdentity, + execClassName: String, + parallelizable: Boolean = false + ): PhysicalOp = { + val base = PhysicalOp + .sourcePhysicalOp( + workflowId, + executionId, + desc.operatorIdentifier, + OpExecWithClassName(execClassName, objectMapper.writeValueAsString(desc)) + ) + .withInputPorts(desc.operatorInfo.inputPorts) + .withOutputPorts(desc.operatorInfo.outputPorts) + val withParallel = if (parallelizable) base.withParallelizable(true) else base + withParallel.withPropagateSchema( + SchemaPropagationFunc(_ => Map(desc.operatorInfo.outputPorts.head.id -> desc.sourceSchema())) + ) + } + + def resolveCsvUriAndDelimiter( + desc: ScanSourceOpDesc, + customDelimiter: Option[String] + ): (URI, Char) = { + val delimiterChar = resolveDelimiterChar(customDelimiter).charAt(0) + require( + desc.fileResolved(), + "No file selected. Please select a valid .csv file from the 'File' dropdown in the right panel." + ) + val uri = new URI(desc.fileName.get) + if (uri.getScheme == "file") { + require( + new java.io.File(uri).isFile, + "The selected item is a folder or does not exist. Please select an actual .csv file from the 'File' dropdown." + ) + } + (uri, delimiterChar) + } + + def buildSchemaFromFirstRow( + firstRow: Array[String], + attributeTypeList: Array[AttributeType], + hasHeader: Boolean + ): Schema = + Schema().add(firstRow.indices.map { i => + new Attribute( + if (hasHeader && firstRow(i).nonEmpty) firstRow(i) else s"column-${i + 1}", + attributeTypeList.lift(i).getOrElse(AttributeType.STRING) + ) + }) + + def generateStandaloneCode( + desc: ScanSourceOpDesc, + customDelimiter: Option[String], + hasHeader: Boolean, + dtypeByType: Map[AttributeType, String], + includeNaValuesBlank: Boolean, + includeWindow: Boolean, + declaredSchemaBody: (String, Schema) => String, + ignoredWindowPrefix: Seq[String] = Seq.empty + ): String = { + val sep = resolveDelimiterChar(customDelimiter) + val schema = Try(desc.sourceSchema()).toOption + val encoding = desc.fileEncoding.toString.replace("_", "-").toLowerCase + val headerArg = if (hasHeader) "0" else "None" + + val args = scala.collection.mutable.ArrayBuffer[String]() + args += s"filepath_or_buffer=$SourceFilePlaceholder" + args += s"sep=${pyStringLiteral(sep)}" + args += s"""encoding=${pyStringLiteral(encoding)}""" + args += s"header=$headerArg" + args += "keep_default_na=False" + if (includeNaValuesBlank) args += """na_values=[""]""" + + val typedColumns: Seq[String] = + schema.toSeq.flatMap( + _.getAttributes.zipWithIndex.flatMap { + case (attribute, i) => dtypeByType.get(attribute.getType).map(d => s"""$i: "$d"""") + } + ) + if (typedColumns.nonEmpty) args += s"dtype={${typedColumns.mkString(", ")}}" + + if (includeWindow) { + desc.offset.map(_.max(0)).foreach { o => + if (hasHeader) args += s"skiprows=lambda _i: 0 < _i <= ${o.toLong}" + else args += s"skiprows=$o" + } + desc.limit.map(_.max(0)).foreach(l => args += s"nrows=$l") + if (!hasHeader) + schema.map(_.getAttributes.size).filter(_ > 0).foreach { n => + args += s"names=[${(0 until n).mkString(", ")}]" + } + } + + val readCall = s"out1df = pd.read_csv(${args.mkString(", ")})" + + val body = schema.filter(_.getAttributes.nonEmpty) match { + case Some(declared) => declaredSchemaBody(readCall, declared) + case None if hasHeader => readCall + case None => + s"""$readCall + |out1df.columns = [f"column-{i + 1}" for i in range(len(out1df.columns))]""".stripMargin + } + + (ignoredWindowPrefix :+ body).mkString("\n") + } +} diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/ParallelCSVScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/ParallelCSVScanSourceOpDesc.scala index 02390ebb568..b403dfec9dd 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/ParallelCSVScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/ParallelCSVScanSourceOpDesc.scala @@ -23,26 +23,20 @@ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.fasterxml.jackson.databind.annotation.JsonDeserialize import com.github.tototoshi.csv.{CSVReader, DefaultCSVFormat} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} -import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.storage.DocumentFactory import org.apache.texera.amber.core.tuple.AttributeTypeUtils.inferSchemaFromRows -import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema} +import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} -import org.apache.texera.amber.core.workflow.{PhysicalOp, SchemaPropagationFunc} -import org.apache.texera.amber.operator.{StandaloneCodeGenerator, StandaloneHelpers} -import org.apache.texera.amber.operator.StandaloneCodeGenerator.SourceFilePlaceholder +import org.apache.texera.amber.core.workflow.PhysicalOp import org.apache.texera.amber.operator.metadata.annotations.UIWidget import org.apache.texera.amber.operator.source.scan.ScanSourceOpDesc +import org.apache.texera.amber.operator.{StandaloneCodeGenerator, StandaloneHelpers} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral -import org.apache.texera.amber.util.JSONUtils.objectMapper import java.io.IOException -import java.net.URI -import scala.util.Try class ParallelCSVScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator { - // One character -- see CSVScanSourceOpDesc. @JsonProperty(defaultValue = ",") @JsonSchemaTitle("Delimiter") @JsonPropertyDescription("single character separating the fields on each line") @@ -62,27 +56,16 @@ class ParallelCSVScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGe workflowId: WorkflowIdentity, executionId: ExecutionIdentity ): PhysicalOp = { - // fill in default values if (customDelimiter.forall(_.isEmpty)) { customDelimiter = Option(",") } - - PhysicalOp - .sourcePhysicalOp( - workflowId, - executionId, - operatorIdentifier, - OpExecWithClassName( - "org.apache.texera.amber.operator.source.scan.csv.ParallelCSVScanSourceOpExec", - objectMapper.writeValueAsString(this) - ) - ) - .withInputPorts(operatorInfo.inputPorts) - .withOutputPorts(operatorInfo.outputPorts) - .withParallelizable(true) - .withPropagateSchema( - SchemaPropagationFunc(_ => Map(operatorInfo.outputPorts.head.id -> sourceSchema())) - ) + CsvScanSupport.csvPhysicalOp( + this, + workflowId, + executionId, + "org.apache.texera.amber.operator.source.scan.csv.ParallelCSVScanSourceOpExec", + parallelizable = true + ) } override def standaloneSourcePath(): Option[String] = fileName @@ -90,135 +73,49 @@ class ParallelCSVScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGe override def standaloneHelpers(): Seq[String] = Seq(StandaloneHelpers.AttributeCasts) override def generateStandaloneCode(): String = { - // First character, empty means comma — the same resolution the reader below does — - // and escaped, so every value the field accepts survives being spliced into Python. - // See CSVScanSourceOpDesc for what handing pandas the raw value did. - val sep = customDelimiter.filter(_.nonEmpty).getOrElse(",").charAt(0).toString - // Inferred once: each inference reads the file again. - val schema = Try(sourceSchema()).toOption - val encoding = fileEncoding.toString.replace("_", "-").toLowerCase - val headerArg = if (hasHeader) "0" else "None" - - val args = scala.collection.mutable.ArrayBuffer[String]() - args += s"filepath_or_buffer=$SourceFilePlaceholder" - args += s"sep=${pyStringLiteral(sep)}" - args += s"""encoding=${pyStringLiteral(encoding)}""" - args += s"header=$headerArg" - - // The block reader nulls an omitted field and leaves every other text alone, - // so only an empty field is missing here. pandas reads a list of words as - // missing by default, "NA" and "null" among them, which turned a column - // holding the country code NA into nulls. See CSVScanSourceOpDesc: both - // halves are needed, one to stop the words and one to keep the blank null. - args += "keep_default_na=False" - args += """na_values=[""]""" - - // Ask for the schema's own type wherever pandas would infer another one. - // The two halves of this operator disagree about a blank: sourceSchema reads - // with scala-csv, where a blank is "" and types its column STRING, while the - // executor nulls it and parses the rest as that STRING. pandas infers a number - // instead, so a column of ids came back as floats, one past 2^53 rounded: - // 9007199254740993 as ...992. A LONG needs the nullable integer for the same - // reason. By position, as in CSVScanSourceOpDesc: under a blank header the - // schema's `column-2` is pandas' `Unnamed: 1`, and pandas passes over a - // name it has no column for, so the type was never applied. - val dtypes: Seq[String] = - schema.toSeq.flatMap( - _.getAttributes.zipWithIndex - .flatMap { - case (a, i) => - val pandasType = a.getType match { - case AttributeType.LONG => Some("Int64") - case AttributeType.STRING => Some("string") - case _ => None - } - pandasType.map(t => s"""$i: "$t"""") - } - ) - if (dtypes.nonEmpty) args += s"dtype={${dtypes.mkString(", ")}}" - - // Limit and offset are inherited fields the parallel reader never reads: - // ParallelCSVScanSourceOpExec.open carves the file into byte ranges and - // leaves both as TODOs. Slicing here gave the export fewer rows than the - // workflow produced, so the window is dropped and said to be dropped. val ignoredWindow = if (offset.isEmpty && limit.isEmpty) Seq.empty else Seq( "# NOTE: this operator's limit and offset are ignored, as the parallel CSV reader ignores them." ) - - val readCall = s"out1df = pd.read_csv(${args.mkString(", ")})" - - // The schema's own names, which every downstream operator was configured - // against: sourceSchema below rewrites a blank header to `column-N`, where - // pandas writes `Unnamed: 1`. Taken by position, so a header the user really - // did spell `Unnamed: 1` survives too. See CSVScanSourceOpDesc. - val body = schema.filter(_.getAttributes.nonEmpty) match { - case Some(declared) => + CsvScanSupport.generateStandaloneCode( + this, + customDelimiter, + hasHeader, + dtypeByType = Map(AttributeType.LONG -> "Int64", AttributeType.STRING -> "string"), + includeNaValuesBlank = true, + includeWindow = false, + declaredSchemaBody = { (readCall, declared) => val names = declared.getAttributes.map(a => pyStringLiteral(a.getName)) Seq( readCall, s"out1df.columns = [${names.mkString(", ")}]", StandaloneCodeGenerator.parseDoubles("out1df", declared) ).filter(_.nonEmpty).mkString("\n") - case None if hasHeader => readCall - case None => - // Unresolved file: fall back to Texera's headerless naming. - s"""$readCall - |out1df.columns = [f"column-{i + 1}" for i in range(len(out1df.columns))]""".stripMargin - } - - (ignoredWindow :+ body).mkString("\n") + }, + ignoredWindowPrefix = ignoredWindow + ) } override def sourceSchema(): Schema = { - val delimiterChar = customDelimiter.filter(_.nonEmpty).getOrElse(",").charAt(0) - require( - fileResolved(), - "No file selected. Please select a valid .csv file from the 'File' dropdown in the right panel." - ) - - val uri = new URI(fileName.get) - if (uri.getScheme == "file") { - require( - new java.io.File(uri).isFile, - "The selected item is a folder or does not exist. Please select an actual .csv file from the 'File' dropdown." - ) - } + val (uri, delimiterChar) = CsvScanSupport.resolveCsvUriAndDelimiter(this, customDelimiter) val file = DocumentFactory.openReadonlyDocument(uri).asFile() implicit object CustomFormat extends DefaultCSVFormat { override val delimiter: Char = delimiterChar - } var reader: CSVReader = CSVReader.open(file)(CustomFormat) val firstRow: Array[String] = reader.iterator.next().toArray reader.close() - // reopen the file to read from the beginning reader = CSVReader.open(file.toPath.toString)(CustomFormat) - if (hasHeader) - reader.readNext() + if (hasHeader) reader.readNext() val attributeTypeList: Array[AttributeType] = inferSchemaFromRows( - reader.iterator - .take(inferSampleSize) - .map(seq => seq.toArray) + reader.iterator.take(inferSampleSize).map(seq => seq.toArray) ) - reader.close() - // build schema based on inferred AttributeTypes. - // Auto-rename blank header positions to `column-N` so empty CSV headers - // (e.g. a trailing comma) do not propagate empty attribute names to - // downstream Iceberg/Parquet writers, which reject them. - Schema().add(firstRow.indices.map { i => - new Attribute( - if (hasHeader && firstRow(i).nonEmpty) firstRow(i) else s"column-${i + 1}", - attributeTypeList(i) - ) - }) - + CsvScanSupport.buildSchemaFromFirstRow(firstRow, attributeTypeList, hasHeader) } - } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDesc.scala index 2c4432976ba..84ae60c9cd6 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDesc.scala @@ -22,26 +22,20 @@ package org.apache.texera.amber.operator.source.scan.csvOld import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.github.tototoshi.csv.{CSVReader, DefaultCSVFormat} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} -import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.storage.DocumentFactory import org.apache.texera.amber.core.tuple.AttributeTypeUtils.inferSchemaFromRows -import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema} +import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} -import org.apache.texera.amber.core.workflow.{PhysicalOp, SchemaPropagationFunc} -import org.apache.texera.amber.operator.{StandaloneCodeGenerator, StandaloneHelpers} -import org.apache.texera.amber.operator.StandaloneCodeGenerator.SourceFilePlaceholder +import org.apache.texera.amber.core.workflow.PhysicalOp import org.apache.texera.amber.operator.metadata.annotations.UIWidget import org.apache.texera.amber.operator.source.scan.ScanSourceOpDesc -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.source.scan.csv.CsvScanSupport +import org.apache.texera.amber.operator.{StandaloneCodeGenerator, StandaloneHelpers} import java.io.IOException -import java.net.URI -import scala.util.Try class CSVOldScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerator { - // One character -- see CSVScanSourceOpDesc. @JsonProperty(defaultValue = ",") @JsonSchemaTitle("Delimiter") @JsonPropertyDescription("single character separating the fields on each line") @@ -60,119 +54,35 @@ class CSVOldScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerat workflowId: WorkflowIdentity, executionId: ExecutionIdentity ): PhysicalOp = { - // fill in default values if (customDelimiter.get.isEmpty) { customDelimiter = Option(",") } - PhysicalOp - .sourcePhysicalOp( - workflowId, - executionId, - operatorIdentifier, - OpExecWithClassName( - "org.apache.texera.amber.operator.source.scan.csvOld.CSVOldScanSourceOpExec", - objectMapper.writeValueAsString(this) - ) - ) - .withInputPorts(operatorInfo.inputPorts) - .withOutputPorts(operatorInfo.outputPorts) - .withPropagateSchema( - SchemaPropagationFunc(_ => Map(operatorInfo.outputPorts.head.id -> sourceSchema())) - ) + CsvScanSupport.csvPhysicalOp( + this, + workflowId, + executionId, + "org.apache.texera.amber.operator.source.scan.csvOld.CSVOldScanSourceOpExec" + ) } override def standaloneSourcePath(): Option[String] = fileName override def standaloneHelpers(): Seq[String] = Seq(StandaloneHelpers.AttributeCasts) - override def generateStandaloneCode(): String = { - // First character, empty means comma — the same resolution the reader below does — - // and escaped, so every value the field accepts survives being spliced into Python. - // See CSVScanSourceOpDesc for what handing pandas the raw value did. - val sep = customDelimiter.filter(_.nonEmpty).getOrElse(",").charAt(0).toString - // Inferred once: each inference reads the file again. - val schema = Try(sourceSchema()).toOption - val encoding = fileEncoding.toString.replace("_", "-").toLowerCase - val headerArg = if (hasHeader) "0" else "None" - - val args = scala.collection.mutable.ArrayBuffer[String]() - args += s"filepath_or_buffer=$SourceFilePlaceholder" - args += s"sep=${pyStringLiteral(sep)}" - args += s"""encoding=${pyStringLiteral(encoding)}""" - args += s"header=$headerArg" - - // This reader has NO missing value at all: scala-csv hands an omitted field - // back as the empty string, and every other text stands for itself, so a - // blank cell is "" and even widens its column to STRING. pandas instead - // reads a list of words as missing by default, "NA" and "null" among them, - // and reads a blank as NaN. Dropping the default list settles both, and no - // na_values is named — where the other CSV readers null a blank, this one - // keeps it. See CSVScanSourceOpDesc. - args += "keep_default_na=False" - - // Read a LONG column as the nullable integer, so a hole does not widen it - // through a float and round the values it carries, and a timestamp as its - // text, parsed below the way the engine parses it. By position, header or - // not, since a blank header is not yet the schema's name. See - // CSVScanSourceOpDesc. - val readAs = Map(AttributeType.LONG -> "Int64", AttributeType.TIMESTAMP -> "object") - val typedColumns: Seq[String] = - schema.toSeq.flatMap( - _.getAttributes.zipWithIndex.flatMap { - case (attribute, i) => readAs.get(attribute.getType).map(d => s"""$i: "$d"""") - } - ) - if (typedColumns.nonEmpty) args += s"dtype={${typedColumns.mkString(", ")}}" - - // Clamped, as in the newer CSV scan: pandas rejects a negative `nrows` where - // the executor's `take` keeps no rows, and only the editor refuses one. - offset.map(_.max(0)).foreach { o => - // Counted in Long past the header and asked of each line, as in the newer - // CSV scan. - if (hasHeader) args += s"skiprows=lambda _i: 0 < _i <= ${o.toLong}" - else args += s"skiprows=$o" - } - limit.map(_.max(0)).foreach(l => args += s"nrows=$l") - - // Without a header, an offset at or past the last row left pandas no line to - // count the columns from, and it raised where the executor emits no rows. - // Named by position, as in the newer CSV scan. - if (!hasHeader) - schema.map(_.getAttributes.size).filter(_ > 0).foreach { n => - args += s"names=[${(0 until n).mkString(", ")}]" - } - - val readCall = s"out1df = pd.read_csv(${args.mkString(", ")})" - - // The schema's own names, which every downstream operator was configured - // against: sourceSchema below rewrites a blank header to `column-N`, where - // pandas writes `Unnamed: 1`. Taken by position, so a header the user really - // did spell `Unnamed: 1` survives too. See CSVScanSourceOpDesc. - schema.filter(_.getAttributes.nonEmpty) match { - case Some(declared) => + override def generateStandaloneCode(): String = + CsvScanSupport.generateStandaloneCode( + this, + customDelimiter, + hasHeader, + dtypeByType = Map(AttributeType.LONG -> "Int64", AttributeType.TIMESTAMP -> "object"), + includeNaValuesBlank = false, + includeWindow = true, + declaredSchemaBody = (readCall, declared) => s"$readCall\n${StandaloneCodeGenerator.readAsDeclared("out1df", declared)}" - case None if hasHeader => readCall - case None => - // Unresolved file: fall back to Texera's headerless naming. - s"""$readCall - |out1df.columns = [f"column-{i + 1}" for i in range(len(out1df.columns))]""".stripMargin - } - } - - override def sourceSchema(): Schema = { - val delimiterChar = customDelimiter.filter(_.nonEmpty).getOrElse(",").charAt(0) - require( - fileResolved(), - "No file selected. Please select a valid .csv file from the 'File' dropdown in the right panel." ) - val uri = new URI(fileName.get) - if (uri.getScheme == "file") { - require( - new java.io.File(uri).isFile, - "The selected item is a folder or does not exist. Please select an actual .csv file from the 'File' dropdown." - ) - } + override def sourceSchema(): Schema = { + val (uri, delimiterChar) = CsvScanSupport.resolveCsvUriAndDelimiter(this, customDelimiter) val file = DocumentFactory.openReadonlyDocument(uri).asFile() implicit object CustomFormat extends DefaultCSVFormat { override val delimiter: Char = delimiterChar @@ -182,12 +92,9 @@ class CSVOldScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerat val firstRow: Array[String] = reader.iterator.next().toArray reader.close() - // reopen the file to read from the beginning reader = CSVReader.open(file, fileEncoding.getCharset.name())(CustomFormat) val header = if (hasHeader) 1 else 0 - // The header and the offset are dropped one after the other, as the executor - // drops them, since their sum is past what an Int holds at the largest offset. val windowRows = reader.iterator .drop(header) .drop(windowOffset) @@ -195,10 +102,7 @@ class CSVOldScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerat .map(_.toArray[Any]) .toSeq reader.close() - // An offset past the last row leaves the window empty, and the types came - // back empty too, so the header below asked a column for a type that was - // never there and the operator threw. The sample is then taken from the - // first row, and a file holding no row at all types every column as text. + val sampleRows = if (windowRows.nonEmpty) windowRows else { @@ -208,17 +112,6 @@ class CSVOldScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerat } val attributeTypeList: Array[AttributeType] = inferSchemaFromRows(sampleRows.iterator) - // build schema based on inferred AttributeTypes. - // Auto-rename blank header positions to `column-N` so empty CSV headers - // (e.g. a trailing comma) do not propagate empty attribute names to - // downstream Iceberg/Parquet writers, which reject them. - Schema().add(firstRow.indices.map { i => - new Attribute( - if (hasHeader && firstRow(i).nonEmpty) firstRow(i) else s"column-${i + 1}", - attributeTypeList.lift(i).getOrElse(AttributeType.STRING) - ) - }) - + CsvScanSupport.buildSchemaFromFirstRow(firstRow, attributeTypeList, hasHeader) } - } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDesc.scala index 5d1db7edfcf..c3ae83f448d 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDesc.scala @@ -21,23 +21,16 @@ package org.apache.texera.amber.operator.source.scan.file import com.fasterxml.jackson.annotation.JsonProperty import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle -import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.state.NoLoopVariable -import org.apache.texera.amber.core.tuple.{AttributeType, Schema} +import org.apache.texera.amber.core.tuple.Schema import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} -import org.apache.texera.amber.core.workflow.{ - InputPort, - OutputPort, - PhysicalOp, - SchemaPropagationFunc -} +import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PhysicalOp} import org.apache.texera.amber.operator.StandaloneCodeGenerator import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.operator.source.SourceOperatorDescriptor -import org.apache.texera.amber.operator.source.scan.{FileAttributeType, FileDecodingMethod} +import org.apache.texera.amber.operator.source.scan.FileDecodingMethod import org.apache.texera.amber.operator.source.scan.text.TextSourceOpDesc import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral -import org.apache.texera.amber.util.JSONUtils.objectMapper class FileScanOpDesc extends SourceOperatorDescriptor @@ -59,31 +52,16 @@ class FileScanOpDesc override def getPhysicalOp( workflowId: WorkflowIdentity, executionId: ExecutionIdentity - ): PhysicalOp = { - PhysicalOp - .sourcePhysicalOp( - workflowId, - executionId, - operatorIdentifier, - OpExecWithClassName( - "org.apache.texera.amber.operator.source.scan.file.FileScanOpExec", - objectMapper.writeValueAsString(this) - ) - ) - .withInputPorts(operatorInfo.inputPorts) - .withOutputPorts(operatorInfo.outputPorts) - .withPropagateSchema( - SchemaPropagationFunc(_ => Map(operatorInfo.outputPorts.head.id -> sourceSchema())) - ) - } + ): PhysicalOp = + TextSourceOpDesc.buildPhysicalOp( + this, + workflowId, + executionId, + "org.apache.texera.amber.operator.source.scan.file.FileScanOpExec" + ) - override def sourceSchema(): Schema = { - var schema = Schema() - if (outputFileName) { - schema = schema.add("filename", AttributeType.STRING) - } - schema.add(attributeName, attributeType.getType) - } + override def sourceSchema(): Schema = + TextSourceOpDesc.buildFileScanSchema(attributeName, attributeType, outputFileName) override def operatorInfo: OperatorInfo = OperatorInfo( @@ -95,18 +73,12 @@ class FileScanOpDesc ) override def generateStandaloneCode(): String = { - val col = attributeName val enc = fileEncoding.toString.replace("_", "-").toLowerCase - val buf = scala.collection.mutable.ArrayBuffer[String]() - - val isBinary = - attributeType == FileAttributeType.BINARY || attributeType == FileAttributeType.LARGE_BINARY val encLit = pyStringLiteral(enc) + val colLit = pyStringLiteral(attributeName) + val buf = scala.collection.mutable.ArrayBuffer[String]() - // The executor takes the row's first String field, not its first column, so a row that - // carries an id ahead of the path still finds the path. Reading column 0 opened the id. - // A row with no string at all makes the executor's `.get` throw, so this raises too - // rather than quietly skipping the row. + // The executor takes the row's first String field, not its first column. buf += "def _texera_file_name(row):" buf += " for _v in row:" buf += " if isinstance(_v, str):" @@ -116,100 +88,47 @@ class FileScanOpDesc buf += "_rows = []" buf += "for _fn in (_texera_file_name(r) for r in in1df.itertuples(index=False)):" - // Every file opens binary and is decoded by `_texera_decode`, which reads the - // bytes the way the engine's reader does (see that helper). A whole file keeps - // its text as decoded, so a CRLF stays a CRLF. Lines go through a StringIO in - // universal-newline mode, which ends a line where `open(..., "r")` does, and so - // where the engine does. - val readWholeOf = if (isBinary) "_f.read()" else s"_texera_decode(_f.read(), $encLit)" + val readWholeOf = + if (TextSourceOpDesc.isBinaryAttribute(attributeType)) "_f.read()" + else s"_texera_decode(_f.read(), $encLit)" val linesOf = s"io.StringIO(_texera_decode(_f.read(), $encLit), newline=None)" - // With extract on the engine reads the files INSIDE the archive, one tuple - // per entry, and skips the entries macOS adds. `zipfile` is the reader that - // matches: the platform casts what it opened to a ZipArchiveInputStream, so - // an archive that is not a zip fails on both sides. A directory entry is - // read, not skipped, there and here alike — it yields nothing. - val (indent, nameExpr, readWhole, lines) = + val (indent, nameExpr) = if (extract) { - buf += " with zipfile.ZipFile(_fn) as _z:" - buf += " for _name in _z.namelist():" - buf += """ if _name.startswith("__MACOSX"):""" - buf += " continue" - buf += " with _z.open(_name) as _f:" - (" " * 16, "_name", readWholeOf, linesOf) + buf ++= TextSourceOpDesc.buildZipExtractLines("_fn").map(" " + _) + (" " * 16, "_name") } else { - buf += " with open(_fn, \"rb\") as _f:" - (" " * 8, "_fn", readWholeOf, linesOf) - } - - // Whatever the flag says, as the platform now reads it: every row carries the - // name of the file its value came from, a line's as much as a whole file's. - // See FileScanUtils.createTuplesFromFile. - val emitFilename = outputFileName - - if (attributeType.isSingle) { - if (emitFilename) buf += s"${indent}_rows.append(($nameExpr, $readWhole))" - else buf += s"${indent}_rows.append($readWhole)" - } else { - val castExpr = attributeType match { - case FileAttributeType.INTEGER => "_texera_parse_int(l, 32)" - case FileAttributeType.LONG => "_texera_parse_int(l, 64)" - case FileAttributeType.DOUBLE => "_texera_java_double(l)" - case FileAttributeType.BOOLEAN => TextSourceOpDesc.BooleanParserCall - case FileAttributeType.TIMESTAMP => "l.rstrip()" - case _ => """l.rstrip("\n")""" + buf += """ with open(_fn, "rb") as _f:""" + (" " * 8, "_fn") } - // The slice applies to the raw lines, as the engine drops and takes - // before parsing: a line outside the window is never converted, so an - // unparseable one there costs nothing. Taking after dropping also keeps - // a large limit from overflowing the end index. The window is per entry, - // as the engine's is: it drops and takes inside the flatMap over entries. - val linesExpr = - if (fileScanOffset.isEmpty && fileScanLimit.isEmpty) lines - else { - val dropped = - fileScanOffset - .filter(_ > 0) - .fold(s"$lines.readlines()")(o => s"$lines.readlines()[$o:]") - fileScanLimit.fold(dropped)(l => s"$dropped[:${l.max(0)}]") - } - val row = if (emitFilename) s"($nameExpr, $castExpr)" else castExpr - buf += s"${indent}_rows.extend($row for l in $linesExpr)" - } - - val colLit = pyStringLiteral(col) - if (emitFilename) { - buf += s"""out1df = pd.DataFrame(_rows, columns=["filename", $colLit])""" - } else { - buf += s"""out1df = pd.DataFrame({$colLit: _rows})""" - } - buf ++= TextSourceOpDesc.parseTimestampColumn(attributeType, colLit) + TextSourceOpDesc.appendFileScanRows( + buf, + indent, + nameExpr, + readWholeOf, + linesOf, + attributeType, + outputFileName, + fileScanOffset, + fileScanLimit + ) + TextSourceOpDesc.appendFileScanDataFrame(buf, attributeType, colLit, outputFileName) buf.mkString("\n") } - override def standaloneHelpers(): Seq[String] = { - val isBinary = - attributeType == FileAttributeType.BINARY || attributeType == FileAttributeType.LARGE_BINARY - TextSourceOpDesc.lineHelpers(attributeType) ++ - (if (isBinary) Seq.empty else Seq(FileScanOpDesc.DecodeHelper)) - } + override def standaloneHelpers(): Seq[String] = + TextSourceOpDesc.fileScanStandaloneHelpers(attributeType, FileScanOpDesc.DecodeHelper) - override def standaloneImports(): Seq[String] = { - val readsLines = - !attributeType.isSingle && attributeType != FileAttributeType.BINARY && - attributeType != FileAttributeType.LARGE_BINARY - (if (readsLines) Seq("import io") else Seq.empty) ++ - (if (extract) Seq("import zipfile") else Seq.empty) - } + override def standaloneImports(): Seq[String] = + TextSourceOpDesc.fileScanStandaloneImports(attributeType, extract) } object FileScanOpDesc { /** The engine decodes through Java's InputStreamReader, which puts U+FFFD in * place of any bytes it cannot decode rather than failing, and reads UTF-16 - * with no byte-order mark as big-endian. Python's codecs refuse both, so a file - * read under the wrong encoding ended the script where the engine read on. + * with no byte-order mark as big-endian. */ val DecodeHelper: String = """def _texera_decode(raw, encoding): diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDesc.scala index 39faab7d3f7..1d00c806a35 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDesc.scala @@ -25,21 +25,15 @@ import com.kjetland.jackson.jsonSchema.annotations.{ JsonSchemaString, JsonSchemaTitle } -import org.apache.texera.amber.core.executor.OpExecWithClassName -import org.apache.texera.amber.core.tuple.{AttributeType, Schema} +import org.apache.texera.amber.core.tuple.Schema import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} -import org.apache.texera.amber.core.workflow.{PhysicalOp, SchemaPropagationFunc} +import org.apache.texera.amber.core.workflow.PhysicalOp import org.apache.texera.amber.operator.StandaloneCodeGenerator import org.apache.texera.amber.operator.StandaloneCodeGenerator.SourceFilePlaceholder import org.apache.texera.amber.operator.metadata.annotations.HideAnnotation import org.apache.texera.amber.operator.source.scan.text.TextSourceOpDesc -import org.apache.texera.amber.operator.source.scan.{ - FileAttributeType, - FileDecodingMethod, - ScanSourceOpDesc -} +import org.apache.texera.amber.operator.source.scan.{FileDecodingMethod, ScanSourceOpDesc} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral -import org.apache.texera.amber.util.JSONUtils.objectMapper @JsonIgnoreProperties(value = Array("limit", "offset", "fileEncoding")) class FileScanSourceOpDesc @@ -55,10 +49,6 @@ class FileScanSourceOpDesc new JsonSchemaString(path = HideAnnotation.hideExpectedValue, value = "binary") ) ) - // Re-declared here rather than inherited so the field can carry the hide - // annotation above; `fileEncoding` from ScanSourceOpDesc is suppressed by the - // @JsonIgnoreProperties on this class, so this is the only charset that - // survives into the executor. var encoding: FileDecodingMethod = FileDecodingMethod.UTF_8 @JsonProperty(defaultValue = "false") @@ -81,131 +71,73 @@ class FileScanSourceOpDesc override def standaloneSourcePath(): Option[String] = fileName override def generateStandaloneCode(): String = { - val col = attributeName - // `encoding` is the charset the panel offers, and the one the executor now - // decodes with. val enc = encoding.toString.replace("_", "-").toLowerCase - val colLit = pyStringLiteral(col) + val colLit = pyStringLiteral(attributeName) val encLit = pyStringLiteral(enc) val buf = scala.collection.mutable.ArrayBuffer[String]() - val isBinary = - attributeType == FileAttributeType.BINARY || attributeType == FileAttributeType.LARGE_BINARY - - val castExpr = attributeType match { - case FileAttributeType.INTEGER => "_texera_parse_int(l, 32)" - case FileAttributeType.LONG => "_texera_parse_int(l, 64)" - case FileAttributeType.DOUBLE => "_texera_java_double(l)" - case FileAttributeType.BOOLEAN => TextSourceOpDesc.BooleanParserCall - case FileAttributeType.TIMESTAMP => "l.rstrip()" - case _ => """l.rstrip("\n")""" - } - - // The slice applies to the raw lines, as the engine drops and takes before - // parsing: a line outside the window is never converted, so an unparseable - // one there costs nothing. Taking after dropping also keeps a large limit - // from overflowing the end index. With extract on, the window is per entry, - // as the engine's is: it drops and takes inside the flatMap over entries. - def windowed(lines: String): String = - if (fileScanOffset.isEmpty && fileScanLimit.isEmpty) lines - else { - val dropped = - fileScanOffset.filter(_ > 0).fold(s"$lines.readlines()")(o => s"$lines.readlines()[$o:]") - fileScanLimit.fold(dropped)(l => s"$dropped[:${l.max(0)}]") - } - - // Every file and entry opens binary and is decoded by the shared - // `_texera_decode`, so UTF-16 without a byte-order mark reads as big-endian, - // as the engine reads it. A whole file keeps its CRLF; lines go through a - // universal-newline StringIO, which ends a line where the engine does. - val readWholeOf = if (isBinary) "_f.read()" else s"_texera_decode(_f.read(), $encLit)" + val readWholeOf = + if (TextSourceOpDesc.isBinaryAttribute(attributeType)) "_f.read()" + else s"_texera_decode(_f.read(), $encLit)" val linesOf = s"io.StringIO(_texera_decode(_f.read(), $encLit), newline=None)" - // Whatever the flag says, as the platform now reads it: every row carries - // the name of the file its value came from, a line's as much as a whole - // file's. See FileScanUtils.createTuplesFromFile. - val emitFilename = outputFileName - if (extract) { - // The engine reads the files INSIDE the archive, one tuple per entry, and - // skips the entries macOS adds. `zipfile` is the reader that matches: the - // platform casts what it opened to a ZipArchiveInputStream, so an archive - // that is not a zip fails on both sides. A directory entry is read, not - // skipped, there and here alike — it yields nothing. The filename column - // carries the ENTRY's name, which is what the engine puts there. buf += "_rows = []" - buf += s"with zipfile.ZipFile($SourceFilePlaceholder) as _z:" - buf += " for _name in _z.namelist():" - buf += """ if _name.startswith("__MACOSX"):""" - buf += " continue" - buf += " with _z.open(_name) as _f:" - if (attributeType.isSingle) { - if (emitFilename) buf += s" _rows.append((_name, $readWholeOf))" - else buf += s" _rows.append($readWholeOf)" - } else { - val linesExpr = windowed(linesOf) - val row = if (emitFilename) s"(_name, $castExpr)" else castExpr - buf += s" _rows.extend($row for l in $linesExpr)" - } - if (emitFilename) buf += s"""out1df = pd.DataFrame(_rows, columns=["filename", $colLit])""" - else buf += s"""out1df = pd.DataFrame({$colLit: _rows})""" + buf ++= TextSourceOpDesc.buildZipExtractLines(SourceFilePlaceholder) + TextSourceOpDesc.appendFileScanRows( + buf, + " " * 12, + "_name", + readWholeOf, + linesOf, + attributeType, + outputFileName, + fileScanOffset, + fileScanLimit + ) + TextSourceOpDesc.appendFileScanDataFrame(buf, attributeType, colLit, outputFileName) } else if (attributeType.isSingle) { val dfCols = - if (emitFilename) s"""{"filename": $SourceFilePlaceholder, $colLit: [$readWholeOf]}""" + if (outputFileName) s"""{"filename": $SourceFilePlaceholder, $colLit: [$readWholeOf]}""" else s"""{$colLit: [$readWholeOf]}""" buf += s"""with open($SourceFilePlaceholder, "rb") as _f:""" buf += s""" out1df = pd.DataFrame($dfCols)""" + buf ++= TextSourceOpDesc.parseTimestampColumn(attributeType, colLit) } else { - val linesExpr = windowed(linesOf) + val castExpr = + TextSourceOpDesc.lineCastExpr(attributeType, "l.rstrip()", """l.rstrip("\n")""") + val linesExpr = + if (fileScanOffset.isEmpty && fileScanLimit.isEmpty) linesOf + else TextSourceOpDesc.sliceLinesExpr(s"$linesOf.readlines()", fileScanOffset, fileScanLimit) val dfCols = - if (emitFilename) + if (outputFileName) s"""{"filename": $SourceFilePlaceholder, $colLit: [$castExpr for l in $linesExpr]}""" else s"""{$colLit: [$castExpr for l in $linesExpr]}""" buf += s"""with open($SourceFilePlaceholder, "rb") as _f:""" buf += s""" out1df = pd.DataFrame($dfCols)""" + buf ++= TextSourceOpDesc.parseTimestampColumn(attributeType, colLit) } - buf ++= TextSourceOpDesc.parseTimestampColumn(attributeType, colLit) buf.mkString("\n") } - override def standaloneHelpers(): Seq[String] = { - val isBinary = - attributeType == FileAttributeType.BINARY || attributeType == FileAttributeType.LARGE_BINARY - TextSourceOpDesc.lineHelpers(attributeType) ++ - (if (isBinary) Seq.empty else Seq(FileScanOpDesc.DecodeHelper)) - } + override def standaloneHelpers(): Seq[String] = + TextSourceOpDesc.fileScanStandaloneHelpers(attributeType, FileScanOpDesc.DecodeHelper) override def standaloneImports(): Seq[String] = - (if (attributeType.isSingle) Seq.empty else Seq("import io")) ++ - (if (extract) Seq("import zipfile") else Seq.empty) + TextSourceOpDesc.fileScanStandaloneImports(attributeType, extract) override def getPhysicalOp( workflowId: WorkflowIdentity, executionId: ExecutionIdentity - ): PhysicalOp = { - PhysicalOp - .sourcePhysicalOp( - workflowId, - executionId, - operatorIdentifier, - OpExecWithClassName( - "org.apache.texera.amber.operator.source.scan.file.FileScanSourceOpExec", - objectMapper.writeValueAsString(this) - ) - ) - .withInputPorts(operatorInfo.inputPorts) - .withOutputPorts(operatorInfo.outputPorts) - .withPropagateSchema( - SchemaPropagationFunc(_ => Map(operatorInfo.outputPorts.head.id -> sourceSchema())) - ) - } + ): PhysicalOp = + TextSourceOpDesc.buildPhysicalOp( + this, + workflowId, + executionId, + "org.apache.texera.amber.operator.source.scan.file.FileScanSourceOpExec" + ) - override def sourceSchema(): Schema = { - var schema = Schema() - if (outputFileName) { - schema = schema.add("filename", AttributeType.STRING) - } - schema.add(attributeName, attributeType.getType) - } + override def sourceSchema(): Schema = + TextSourceOpDesc.buildFileScanSchema(attributeName, attributeType, outputFileName) } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDesc.scala index ea2510dc828..9169d22bf6f 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDesc.scala @@ -21,15 +21,13 @@ package org.apache.texera.amber.operator.source.scan.text import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} -import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.tuple.Schema import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} -import org.apache.texera.amber.core.workflow.{OutputPort, PhysicalOp, SchemaPropagationFunc} +import org.apache.texera.amber.core.workflow.{OutputPort, PhysicalOp} import org.apache.texera.amber.operator.StandaloneCodeGenerator import org.apache.texera.amber.operator.metadata.annotations.UIWidget import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.operator.source.SourceOperatorDescriptor -import org.apache.texera.amber.operator.source.scan.FileAttributeType import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.objectMapper @@ -47,24 +45,14 @@ class TextInputSourceOpDesc workflowId: WorkflowIdentity, executionId: ExecutionIdentity ): PhysicalOp = - PhysicalOp - .sourcePhysicalOp( - workflowId, - executionId, - operatorIdentifier, - OpExecWithClassName( - "org.apache.texera.amber.operator.source.scan.text.TextInputSourceOpExec", - objectMapper.writeValueAsString(this) - ) - ) - .withInputPorts(operatorInfo.inputPorts) - .withOutputPorts(operatorInfo.outputPorts) - .withPropagateSchema( - SchemaPropagationFunc(_ => Map(operatorInfo.outputPorts.head.id -> sourceSchema())) - ) + TextSourceOpDesc.buildPhysicalOp( + this, + workflowId, + executionId, + "org.apache.texera.amber.operator.source.scan.text.TextInputSourceOpExec" + ) - override def sourceSchema(): Schema = - Schema().add(attributeName, attributeType.getType) + override def sourceSchema(): Schema = Schema().add(attributeName, attributeType.getType) override def operatorInfo: OperatorInfo = OperatorInfo( @@ -77,40 +65,22 @@ class TextInputSourceOpDesc override def generateStandaloneCode(): String = { val text = objectMapper.writeValueAsString(textInput) - val col = attributeName - val colLit = pyStringLiteral(col) + val colLit = pyStringLiteral(attributeName) val buf = scala.collection.mutable.ArrayBuffer[String]() buf += s"_text = $text" - val isBinary = - attributeType == FileAttributeType.BINARY || attributeType == FileAttributeType.LARGE_BINARY - if (attributeType.isSingle) { - val valueExpr = if (isBinary) """_text.encode("utf-8")""" else "_text" + val valueExpr = + if (TextSourceOpDesc.isBinaryAttribute(attributeType)) """_text.encode("utf-8")""" + else "_text" buf += s"""out1df = pd.DataFrame({$colLit: [$valueExpr]})""" } else { - val castExpr = attributeType match { - case FileAttributeType.INTEGER => "_texera_parse_int(l, 32)" - case FileAttributeType.LONG => "_texera_parse_int(l, 64)" - case FileAttributeType.DOUBLE => "_texera_java_double(l)" - case FileAttributeType.BOOLEAN => TextSourceOpDesc.BooleanParserCall - case FileAttributeType.TIMESTAMP => "l" - case _ => "l" - } - // The engine splits with linesIterator, which ends a line only at LF, CR - // and CRLF. splitlines also ends one at eight other characters, U+2028 - // and NEL among them, so a row holding one came out as two. A newline at - // the very end closes the last line rather than opening an empty one. + val castExpr = TextSourceOpDesc.lineCastExpr(attributeType, "l", "l") buf += """_lines = _text.replace("\r\n", "\n").replace("\r", "\n").split("\n")""" buf += """if _lines[-1] == "":""" buf += " _lines.pop()" - // The slice applies to the raw lines, as the engine drops and takes - // before parsing: a line outside the window is never converted, so an - // unparseable one there costs nothing. Taking after dropping also keeps - // a large limit from overflowing the end index. - val dropped = fileScanOffset.filter(_ > 0).fold("_lines")(o => s"_lines[$o:]") - val linesExpr = fileScanLimit.fold(dropped)(l => s"$dropped[:${l.max(0)}]") + val linesExpr = TextSourceOpDesc.sliceLinesExpr("_lines", fileScanOffset, fileScanLimit) buf += s"""out1df = pd.DataFrame({$colLit: [$castExpr for l in $linesExpr]})""" buf ++= TextSourceOpDesc.parseTimestampColumn(attributeType, colLit) } @@ -118,5 +88,6 @@ class TextInputSourceOpDesc buf.mkString("\n") } - override def standaloneHelpers(): Seq[String] = TextSourceOpDesc.lineHelpers(attributeType) + override def standaloneHelpers(): Seq[String] = + TextSourceOpDesc.lineHelpers(attributeType) } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/text/TextSourceOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/text/TextSourceOpDesc.scala index b319d0fcb7f..acf7cbaacd9 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/text/TextSourceOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/text/TextSourceOpDesc.scala @@ -26,10 +26,16 @@ import com.kjetland.jackson.jsonSchema.annotations.{ JsonSchemaString, JsonSchemaTitle } +import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.state.NoLoopVariable +import org.apache.texera.amber.core.tuple.{AttributeType, Schema} +import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} +import org.apache.texera.amber.core.workflow.{PhysicalOp, SchemaPropagationFunc} import org.apache.texera.amber.operator.StandaloneHelpers import org.apache.texera.amber.operator.metadata.annotations.HideAnnotation +import org.apache.texera.amber.operator.source.SourceOperatorDescriptor import org.apache.texera.amber.operator.source.scan.FileAttributeType +import org.apache.texera.amber.util.JSONUtils.objectMapper /** * TextSourceOpDesc is a trait holding commonly used properties and functions used for variations of text input processing @@ -51,9 +57,6 @@ trait TextSourceOpDesc { @NoLoopVariable var attributeName: String = "line" - // Named explicitly so reflection can see it. These are the row-window knobs the - // text sources actually read — the ones they inherit are ignored — but they carried - // no @JsonProperty, which leaves them invisible to anything walking the config. @JsonProperty @JsonSchemaTitle("Limit (lines)") @JsonDeserialize(contentAs = classOf[Int]) @@ -117,4 +120,123 @@ object TextSourceOpDesc { Seq(StandaloneHelpers.AttributeCasts) case _ => Seq.empty } + + private[source] def isBinaryAttribute(attributeType: FileAttributeType): Boolean = + attributeType == FileAttributeType.BINARY || attributeType == FileAttributeType.LARGE_BINARY + + private[source] def buildPhysicalOp( + desc: SourceOperatorDescriptor, + workflowId: WorkflowIdentity, + executionId: ExecutionIdentity, + execClassName: String + ): PhysicalOp = + PhysicalOp + .sourcePhysicalOp( + workflowId, + executionId, + desc.operatorIdentifier, + OpExecWithClassName(execClassName, objectMapper.writeValueAsString(desc)) + ) + .withInputPorts(desc.operatorInfo.inputPorts) + .withOutputPorts(desc.operatorInfo.outputPorts) + .withPropagateSchema( + SchemaPropagationFunc(_ => + Map(desc.operatorInfo.outputPorts.head.id -> desc.sourceSchema()) + ) + ) + + private[source] def buildFileScanSchema( + attributeName: String, + attributeType: FileAttributeType, + outputFileName: Boolean + ): Schema = { + var schema = Schema() + if (outputFileName) { + schema = schema.add("filename", AttributeType.STRING) + } + schema.add(attributeName, attributeType.getType) + } + + private[source] def lineCastExpr( + attributeType: FileAttributeType, + timestampExpr: String, + defaultExpr: String + ): String = + attributeType match { + case FileAttributeType.INTEGER => "_texera_parse_int(l, 32)" + case FileAttributeType.LONG => "_texera_parse_int(l, 64)" + case FileAttributeType.DOUBLE => "_texera_java_double(l)" + case FileAttributeType.BOOLEAN => BooleanParserCall + case FileAttributeType.TIMESTAMP => timestampExpr + case _ => defaultExpr + } + + private[source] def sliceLinesExpr( + baseExpr: String, + offset: Option[Int], + limit: Option[Int] + ): String = { + val dropped = offset.filter(_ > 0).fold(baseExpr)(o => s"$baseExpr[$o:]") + limit.fold(dropped)(l => s"$dropped[:${l.max(0)}]") + } + + private[source] def appendFileScanRows( + buf: scala.collection.mutable.ArrayBuffer[String], + indent: String, + nameExpr: String, + readWholeOf: String, + linesOf: String, + attributeType: FileAttributeType, + emitFilename: Boolean, + offset: Option[Int], + limit: Option[Int] + ): Unit = { + if (attributeType.isSingle) { + if (emitFilename) buf += s"${indent}_rows.append(($nameExpr, $readWholeOf))" + else buf += s"${indent}_rows.append($readWholeOf)" + } else { + val castExpr = lineCastExpr(attributeType, "l.rstrip()", """l.rstrip("\n")""") + val linesExpr = + if (offset.isEmpty && limit.isEmpty) linesOf + else sliceLinesExpr(s"$linesOf.readlines()", offset, limit) + val row = if (emitFilename) s"($nameExpr, $castExpr)" else castExpr + buf += s"${indent}_rows.extend($row for l in $linesExpr)" + } + } + + private[source] def buildZipExtractLines(sourceExpr: String): Seq[String] = + Seq( + s"with zipfile.ZipFile($sourceExpr) as _z:", + " for _name in _z.namelist():", + """ if _name.startswith("__MACOSX"):""", + " continue", + " with _z.open(_name) as _f:" + ) + + private[source] def appendFileScanDataFrame( + buf: scala.collection.mutable.ArrayBuffer[String], + attributeType: FileAttributeType, + colLit: String, + emitFilename: Boolean + ): Unit = { + if (emitFilename) buf += s"""out1df = pd.DataFrame(_rows, columns=["filename", $colLit])""" + else buf += s"""out1df = pd.DataFrame({$colLit: _rows})""" + buf ++= parseTimestampColumn(attributeType, colLit) + } + + private[source] def fileScanStandaloneHelpers( + attributeType: FileAttributeType, + decodeHelper: String + ): Seq[String] = + lineHelpers(attributeType) ++ (if (isBinaryAttribute(attributeType)) Seq.empty + else Seq(decodeHelper)) + + private[source] def fileScanStandaloneImports( + attributeType: FileAttributeType, + extract: Boolean + ): Seq[String] = { + val readsLines = !attributeType.isSingle && !isBinaryAttribute(attributeType) + (if (readsLines) Seq("import io") else Seq.empty) ++ + (if (extract) Seq("import zipfile") else Seq.empty) + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/DotPlot/DotPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/DotPlot/DotPlotOpDesc.scala index 46e8e2a782a..0f4c9ab2e4a 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/DotPlot/DotPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/DotPlot/DotPlotOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.DotPlot import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ @@ -74,23 +75,18 @@ class DotPlotOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneCode | |class ProcessTableOperator(UDFTableOperator): | - | def render_error(self, error_msg): - | return '''
Reasons are: {}
- | '''.format(error_msg) + |${runtimeError("DotPlot is not available.", plural = true)} | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - | if table.empty: - | yield {'html-content': self.render_error("Input table is empty.")} - | return + |${emptyTableGuard("Input table is empty.", bodyIndent = 4)} | ${createPlotlyFigure()} - | if table.empty: - | yield {'html-content': self.render_error("No valid rows left (every row has at least 1 missing value).")} - | return + |${emptyTableGuard( + "No valid rows left (every row has at least 1 missing value).", + bodyIndent = 4 + )} | # convert fig to html content - | html = plotly.io.to_html(fig, include_plotlyjs='cdn', auto_play=False) - | yield {'html-content': html} + |${plotlyHtmlYield()} |""" finalCode.encode } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/IcicleChart/IcicleChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/IcicleChart/IcicleChartOpDesc.scala index 557af3a7f29..706b4d1a4ca 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/IcicleChart/IcicleChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/IcicleChart/IcicleChartOpDesc.scala @@ -19,33 +19,25 @@ package org.apache.texera.amber.operator.visualization.IcicleChart -import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ - PythonTemplateBuilderStringContext, - pyStringLiteral -} -import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString -import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode -import org.apache.texera.amber.operator.visualization.PythonVisualizationOpDesc import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} -import org.apache.texera.amber.operator.visualization.hierarchychart.HierarchySection +import org.apache.texera.amber.operator.visualization.hierarchychart.{ + HierarchyChartSupport, + HierarchySection +} +import org.apache.texera.amber.operator.visualization.{ + PlotlyStandaloneCode, + PythonVisualizationOpDesc +} +import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.pybuilder.PythonTemplateBuilder import javax.validation.constraints.{NotEmpty, NotNull} // type constraint: value can only be numeric -@JsonSchemaInject(json = """ -{ - "attributeTypeRules": { - "value": { - "enum": ["integer", "long", "double"] - } - } -} -""") +@JsonSchemaInject(json = PythonVisualizationOpDesc.NumericValueRuleJson) class IcicleChartOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneCode { @JsonProperty(required = true) @JsonSchemaTitle("Hierarchy Path") @@ -69,93 +61,19 @@ class IcicleChartOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneC OperatorGroupConstants.VISUALIZATION_BASIC_GROUP ) - private def getIcicleAttributesInPython: String = - hierarchy.map(c => pyb"${c.attributeName}").mkString(",") - - def manipulateTable(): PythonTemplateBuilder = { - assert(value.nonEmpty, "Value Column cannot be empty") - val attributes = getIcicleAttributesInPython - pyb""" - | table[$value] = table[table[$value] > 0][$value] # remove non-positive numbers from the data - | table.dropna(subset = [$attributes], inplace = True) #remove missing values - |""" - } + def manipulateTable(): PythonTemplateBuilder = + HierarchyChartSupport.filterPositiveAndDropNa(value, hierarchy) - def createPlotlyFigure(): PythonTemplateBuilder = { - assert(hierarchy.nonEmpty, "Hierarchy Path cannot be empty") - val attributes = getIcicleAttributesInPython - pyb""" - | fig = px.icicle(table, path=[$attributes], values=$value, - | color=$value, hover_data=[$attributes], - | color_continuous_scale='RdBu') - |""" - } + def createPlotlyFigure(): PythonTemplateBuilder = + HierarchyChartSupport.createPlotlyFigure("icicle", value, hierarchy) - override def generatePythonCode(): String = { - val finalCode = - pyb""" - |from pytexera import * - | - |import plotly.express as px - |import plotly.graph_objects as go - |import plotly.io - |import numpy as np - | - |class ProcessTableOperator(UDFTableOperator): - | - | # Generate custom error message as html string - | def render_error(self, error_msg) -> str: - | return '''Reason is: {}
- | '''.format(error_msg) - | - | @overrides - | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - | if table.empty: - | yield {'html-content': self.render_error("input table is empty.")} - | return - | ${manipulateTable()} - | if table.empty: - | yield {'html-content': self.render_error("value column contains only non-positive numbers or nulls.")} - | return - | ${createPlotlyFigure()} - | # convert fig to html content - | fig.update_layout(margin=dict(l=0, r=0, b=0, t=0)) - | html = plotly.io.to_html(fig, include_plotlyjs='cdn', auto_play=False) - | yield {'html-content': html} - |""" - finalCode.encode - } - - override def generateStandaloneCode(): String = { - val attributes = hierarchy.map(section => pyStringLiteral(section.attributeName)).mkString(", ") - val valueLit = pyStringLiteral(value) - // The error page is written to the same file a plotted chart lands in, so a - // reason for "no chart" is where the reader looks for the chart — printing it - // to the terminal alone left that file absent. render_error's continuation - // line keeps the runtime path's indentation, since the HTML is triple-quoted - // and those spaces reach the browser. chart_df is a copy: the input can feed another branch. - s"""${standaloneError("Icicle chart is not available.")} - | - |def fail(error_msg): - | with open(outputHtml, "w", encoding="utf-8") as output: - | output.write(render_error(error_msg)) - | print(f"Icicle chart error: {error_msg}") - | - |if in1df.empty: - | fail("input table is empty.") - |else: - | chart_df = in1df.copy() - | chart_df[$valueLit] = chart_df[chart_df[$valueLit] > 0][$valueLit] - | chart_df = chart_df.dropna(subset=[$attributes]) - | if chart_df.empty: - | fail("value column contains only non-positive numbers or nulls.") - | else: - | fig = px.icicle(chart_df, path=[$attributes], values=$valueLit, - | color=$valueLit, hover_data=[$attributes], - | color_continuous_scale='RdBu') - | fig.update_layout(margin=dict(l=0, r=0, b=0, t=0)) - |${standaloneFigure("Icicle chart")}""".stripMargin - } + override def generatePythonCode(): String = + HierarchyChartSupport.runtimePythonCode( + "Icicle chart is not available.", + manipulateTable(), + createPlotlyFigure() + ) + override def generateStandaloneCode(): String = + HierarchyChartSupport.standalonePythonCode("Icicle chart", "icicle", value, hierarchy) } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/PythonCodeFragment.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/PythonCodeFragment.scala index 3ee6a7a1764..884e31773bf 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/PythonCodeFragment.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/PythonCodeFragment.scala @@ -19,6 +19,8 @@ package org.apache.texera.amber.operator.visualization +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral + private[visualization] object PythonCodeFragment { def render(code: String, indent: Int): String = code.stripMargin.linesIterator @@ -26,4 +28,58 @@ private[visualization] object PythonCodeFragment { if (line.isEmpty) line else (" " * indent) + line } .mkString("\n") + + def runtimeError( + title: String, + plural: Boolean = false, + typed: Boolean = false, + colonOnly: Boolean = false, + singleLineClose: Boolean = false, + indent: Int = 4 + ): String = { + val label = + if (colonOnly) "Reason" else if (plural) "Reasons are" else "Reason is" + val sig = s"def render_error(self, error_msg)${if (typed) " -> str" else ""}:" + val pfx = " " * indent + val closeInline = if (singleLineClose) "'''.format(error_msg)" else "" + val tail = + if (singleLineClose) "" else s"\n${" " * (indent + 11)}'''.format(error_msg)" + s"""$pfx$sig + |$pfx return '''$label: {}
$closeInline$tail""".stripMargin + } + + def emptyTableGuard( + message: String, + indent: Int = 8, + bodyIndent: Int = 3, + doubleQuoted: Boolean = false + ): String = { + val q = if (doubleQuoted) "\"" else "'" + val p0 = " " * indent + val pb = " " * (indent + bodyIndent) + val msgLit = pyStringLiteral(message) + s"""${p0}if table.empty: + |${pb}yield {${q}html-content${q}: self.render_error($msgLit)} + |${pb}return""".stripMargin + } + + def plotlyHtmlYield( + indent: Int = 8, + includeComment: Boolean = false, + module: String = "plotly.io", + extraArgs: String = "auto_play=False", + yieldVar: String = "html", + doubleQuoted: Boolean = false, + spacesAroundEq: Boolean = false, + compactDict: Boolean = false + ): String = { + val p = " " * indent + val eq = if (spacesAroundEq) " = " else "=" + val kv = if (compactDict) ":" else ": " + val q = if (doubleQuoted) "\"" else "'" + val comment = if (includeComment) s"$p# convert fig to html content\n" else "" + s"""$comment$p$yieldVar = $module.to_html(fig, include_plotlyjs${eq}'cdn', $extraArgs) + |${p}yield {${q}html-content${q}$kv$yieldVar}""".stripMargin + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/PythonVisualizationOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/PythonVisualizationOpDesc.scala index 1a02de4bf48..4779a0a1c16 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/PythonVisualizationOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/PythonVisualizationOpDesc.scala @@ -41,3 +41,22 @@ trait PythonVisualizationOpDesc extends PythonOperatorDescriptor with Standalone final override def producesDataFrame(): Boolean = false } + +object PythonVisualizationOpDesc { + private[visualization] final val NumericValueRuleJson = """ +{ + "attributeTypeRules": { + "value": { + "enum": ["integer", "long", "double"] + } + } +} +""" + + private[visualization] final val ColorPatternJson = """ +{ + "pattern": "^\\s*$|^\\s*#(?:\\s*[0-9a-fA-F]){3}(?:(?:\\s*[0-9a-fA-F]){3})?\\s*$|^\\s*(?:[rR]\\s*[gG]\\s*[bB]|[hH]\\s*[sS]\\s*[lL]|[hH]\\s*[sS]\\s*[vV])(?:\\s*[aA])?\\s*\\(\\s*(?:\\s*[0-9.])+(?:\\s*%)?(?:\\s*,(?:\\s*[0-9.])+(?:\\s*%)?){2,3}\\s*\\)\\s*$|^\\s*[vV]\\s*[aA]\\s*[rR]\\s*\\(\\s*-\\s*-[^)]*\\)\\s*$|^\\s*[a-zA-Z][a-zA-Z\\s]*$", + "examples": ["red"] +} +""" +} diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/barChart/BarChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/barChart/BarChartOpDesc.scala index ef360bbaa4b..b4648042f07 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/barChart/BarChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/barChart/BarChartOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.barChart import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ @@ -36,15 +37,7 @@ import org.apache.texera.amber.pybuilder.PythonTemplateBuilder import javax.validation.constraints.NotNull //type constraint: value can only be numeric -@JsonSchemaInject(json = """ -{ - "attributeTypeRules": { - "value": { - "enum": ["integer", "long", "double"] - } - } -} -""") +@JsonSchemaInject(json = PythonVisualizationOpDesc.NumericValueRuleJson) class BarChartOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneCode { @JsonProperty(value = "value", required = true) @@ -124,10 +117,7 @@ class BarChartOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneCode |class ProcessTableOperator(UDFTableOperator): | | # Generate custom error message as html string - | def render_error(self, error_msg) -> str: - | return '''Reason is: {}
- | '''.format(error_msg) + |${runtimeError("Bar chart is not available.", typed = true)} | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/boxViolinPlot/BoxViolinPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/boxViolinPlot/BoxViolinPlotOpDesc.scala index 10c5eb5b4dc..5cf4429966e 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/boxViolinPlot/BoxViolinPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/boxViolinPlot/BoxViolinPlotOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.boxViolinPlot import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription, JsonPropertyOrder} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ @@ -36,15 +37,7 @@ import org.apache.texera.amber.pybuilder.PythonTemplateBuilder import javax.validation.constraints.NotNull @JsonPropertyOrder(Array("value", "quartileType", "horizontalOrientation", "violinPlot")) -@JsonSchemaInject(json = """ -{ - "attributeTypeRules": { - "value": { - "enum": ["integer", "long", "double"] - } - } -} -""") +@JsonSchemaInject(json = PythonVisualizationOpDesc.NumericValueRuleJson) class BoxViolinPlotOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneCode { @JsonProperty(value = "value", required = true) @@ -125,24 +118,16 @@ class BoxViolinPlotOpDesc extends PythonVisualizationOpDesc with PlotlyStandalon |class ProcessTableOperator(UDFTableOperator): | | # Generate custom error message as html string - | def render_error(self, error_msg) -> str: - | return '''Reason is: {}
- | '''.format(error_msg) + |${runtimeError("Box/Violin Plot is not available.", typed = true)} | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - | if table.empty: - | yield {'html-content': self.render_error("input table is empty.")} - | return + |${emptyTableGuard("input table is empty.")} | ${manipulateTable()} - | if table.empty: - | yield {'html-content': self.render_error("value column contains only non-positive numbers or nulls.")} - | return + |${emptyTableGuard("value column contains only non-positive numbers or nulls.")} | ${createPlotlyFigure()} | # convert fig to html content - | html = plotly.io.to_html(fig, include_plotlyjs='cdn', auto_play=False) - | yield {'html-content': html} + |${plotlyHtmlYield()} | """ finalCode.encode } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/bubbleChart/BubbleChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/bubbleChart/BubbleChartOpDesc.scala index e13610f5283..244da2717ad 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/bubbleChart/BubbleChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/bubbleChart/BubbleChartOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.bubbleChart import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ @@ -131,24 +132,23 @@ class BubbleChartOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneC | |class ProcessTableOperator(UDFTableOperator): | - | def render_error(self, error_msg): - | return '''Reasons are: {}
- | '''.format(error_msg) + |${runtimeError("TreeMap is not available.", plural = true)} | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - | if table.empty: - | yield {'html-content': self.render_error("Input table is empty.")} - | return + |${emptyTableGuard("Input table is empty.", bodyIndent = 4)} | ${manipulateTable()} | ${createPlotlyFigure()} - | if table.empty: - | yield {'html-content': self.render_error("No valid rows left (every row has at least 1 missing value).")} - | return + |${emptyTableGuard( + "No valid rows left (every row has at least 1 missing value).", + bodyIndent = 4 + )} | fig.update_layout(margin=dict(l=0, r=0, b=0, t=0)) - | html = plotly.io.to_html(fig, include_plotlyjs = 'cdn', auto_play = False) - | yield {'html-content':html} + |${plotlyHtmlYield( + extraArgs = "auto_play = False", + spacesAroundEq = true, + compactDict = true + )} |""" finalCode.encode } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/bulletChart/BulletChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/bulletChart/BulletChartOpDesc.scala index 530ecb79243..cdc26d952b0 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/bulletChart/BulletChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/bulletChart/BulletChartOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.bulletChart import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.fasterxml.jackson.databind.annotation.JsonDeserialize import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} @@ -118,9 +119,12 @@ class BulletChartOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneC |class ProcessTableOperator(UDFTableOperator): | | # Render an error message in HTML format - | def render_error(self, error_msg) -> str: - | return '''Reason: {}
'''.format(error_msg) + |${runtimeError( + "Bullet chart is not available.", + typed = true, + colonOnly = true, + singleLineClose = true + )} | | # Generate a list of grayscale HSL colors with decreasing brightness | def generate_gray_gradient(self, step_count): @@ -146,9 +150,7 @@ class BulletChartOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneC | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - | if table.empty: - | yield {'html-content': self.render_error("Input table is empty.")} - | return + |${emptyTableGuard("Input table is empty.", bodyIndent = 4)} | | try: | value_col = $value @@ -159,9 +161,11 @@ class BulletChartOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneC | return | | table = table.dropna(subset=[value_col]) - | if table.empty: - | yield {'html-content': self.render_error("No valid data rows found after dropping nulls.")} - | return + |${emptyTableGuard( + "No valid data rows found after dropping nulls.", + indent = 12, + bodyIndent = 4 + )} | | threshold_val = $thresholdExpr | diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/choroplethMap/ChoroplethMapOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/choroplethMap/ChoroplethMapOpDesc.scala index 3932e964a3d..26d1cf97b67 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/choroplethMap/ChoroplethMapOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/choroplethMap/ChoroplethMapOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.choroplethMap import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ @@ -104,23 +105,15 @@ class ChoroplethMapOpDesc extends PythonVisualizationOpDesc with PlotlyStandalon |class ProcessTableOperator(UDFTableOperator): | | # Generate custom error message as html string - | def render_error(self, error_msg) -> str: - | return '''Reason is: {}
- | '''.format(error_msg) + |${runtimeError("Choropleth map is not available.", typed = true)} | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - | if table.empty: - | yield {'html-content': self.render_error("Input table is empty.")} - | return + |${emptyTableGuard("Input table is empty.")} | ${manipulateTable()} - | if table.empty: - | yield {'html-content': self.render_error("No valid rows left (every row has at least 1 missing value).")} - | return + |${emptyTableGuard("No valid rows left (every row has at least 1 missing value).")} | ${createPlotlyFigure()} - | html = plotly.io.to_html(fig, include_plotlyjs='cdn', auto_play=False) - | yield {'html-content': html} + |${plotlyHtmlYield()} |""" finalCode.encode } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/continuousErrorBands/BandConfig.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/continuousErrorBands/BandConfig.scala index e0946b810b0..84e1608e879 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/continuousErrorBands/BandConfig.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/continuousErrorBands/BandConfig.scala @@ -25,6 +25,8 @@ import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName import org.apache.texera.amber.operator.visualization.lineChart.LineConfig +import org.apache.texera.amber.operator.visualization.PythonVisualizationOpDesc + import javax.validation.constraints.NotNull class BandConfig extends LineConfig { @@ -47,11 +49,6 @@ class BandConfig extends LineConfig { @JsonProperty(required = false) @JsonSchemaTitle("Fill Color") @JsonPropertyDescription("must be a valid CSS color or hex color string") - @JsonSchemaInject(json = """ -{ - "pattern": "^\\s*$|^\\s*#(?:\\s*[0-9a-fA-F]){3}(?:(?:\\s*[0-9a-fA-F]){3})?\\s*$|^\\s*(?:[rR]\\s*[gG]\\s*[bB]|[hH]\\s*[sS]\\s*[lL]|[hH]\\s*[sS]\\s*[vV])(?:\\s*[aA])?\\s*\\(\\s*(?:\\s*[0-9.])+(?:\\s*%)?(?:\\s*,(?:\\s*[0-9.])+(?:\\s*%)?){2,3}\\s*\\)\\s*$|^\\s*[vV]\\s*[aA]\\s*[rR]\\s*\\(\\s*-\\s*-[^)]*\\)\\s*$|^\\s*[a-zA-Z][a-zA-Z\\s]*$", - "examples": ["red"] -} -""") + @JsonSchemaInject(json = PythonVisualizationOpDesc.ColorPatternJson) var fillColor: EncodableString = "" } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/continuousErrorBands/ContinuousErrorBandsOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/continuousErrorBands/ContinuousErrorBandsOpDesc.scala index 3a61f717dd2..2521744ece7 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/continuousErrorBands/ContinuousErrorBandsOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/continuousErrorBands/ContinuousErrorBandsOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.continuousErrorBands import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ @@ -131,20 +132,14 @@ class ContinuousErrorBandsOpDesc extends PythonVisualizationOpDesc with PlotlySt |class ProcessTableOperator(UDFTableOperator): | | # Generate custom error message as html string - | def render_error(self, error_msg) -> str: - | return '''Reason is: {}
- | '''.format(error_msg) + |${runtimeError("Continuous Error Bands is not available.", typed = true)} | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - | if table.empty: - | yield {'html-content': self.render_error("input table is empty.")} - | return + |${emptyTableGuard("input table is empty.")} | ${createPlotlyFigure()} | # convert fig to html content - | html = plotly.io.to_html(fig, include_plotlyjs='cdn', auto_play=False) - | yield {'html-content': html} + |${plotlyHtmlYield()} |""" finalCode.encode } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/contourPlot/ContourPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/contourPlot/ContourPlotOpDesc.scala index c023b3405fe..c2d214c8b1b 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/contourPlot/ContourPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/contourPlot/ContourPlotOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.contourPlot import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.fasterxml.jackson.databind.annotation.JsonDeserialize import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} @@ -107,19 +108,14 @@ class ContourPlotOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneC |import plotly.io as pio | |class ProcessTableOperator(UDFTableOperator): - | def render_error(self, error_msg) -> str: - | return '''Reason is: {}
- | '''.format(error_msg) + |${runtimeError("Contour plot is not available.", typed = true)} | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: | # A row missing any of the three has no point to contribute, and | # griddata refuses a NaN coordinate outright. | table = table.dropna(subset=[$x, $y, $z]) - | if table.empty: - | yield {'html-content': self.render_error("Table should not have any empty/null values or fields.")} - | return + |${emptyTableGuard("Table should not have any empty/null values or fields.", bodyIndent = 4)} | | x = table[$x].values | y = table[$y].values @@ -148,8 +144,7 @@ class ContourPlotOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneC | colorbar_title=$z | )) | fig.update_layout(title='Contour Plot') - | html = pio.to_html(fig, include_plotlyjs='cdn', full_html=False) - | yield {'html-content': html} + |${plotlyHtmlYield(module = "pio", extraArgs = "full_html=False")} |""".encode } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/dendrogram/DendrogramOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/dendrogram/DendrogramOpDesc.scala index 16ff19e238f..d9c7f20abbb 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/dendrogram/DendrogramOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/dendrogram/DendrogramOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.dendrogram import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.fasterxml.jackson.databind.annotation.JsonDeserialize import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} @@ -114,22 +115,15 @@ class DendrogramOpDesc extends PythonVisualizationOpDesc { |import numpy as np | |class ProcessTableOperator(UDFTableOperator): - | def render_error(self, error_msg): - | return '''Reason is: {}
- | '''.format(error_msg) + |${runtimeError("Dendrogram is not available.")} | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - | if table.empty: - | yield {'html-content': self.render_error("input table is empty.")} - | return + |${emptyTableGuard("input table is empty.")} | # A row missing either coordinate has no position to cluster from, and | # scipy refuses a NaN anywhere in the distance matrix. | table = table.dropna(subset=[$xVal, $yVal]) #remove missing values - | if table.empty: - | yield {'html-content': self.render_error("input table has no rows with all of the configured columns filled in.")} - | return + |${emptyTableGuard("input table has no rows with all of the configured columns filled in.")} | # Clustering starts from the distances between rows, so a single row | # leaves scipy an empty distance matrix and it raises rather than draws. | if len(table) < 2: @@ -137,8 +131,7 @@ class DendrogramOpDesc extends PythonVisualizationOpDesc { | return | ${createDendrogram()} | # convert fig to html content - | html = plotly.io.to_html(fig, include_plotlyjs='cdn', auto_play=False) - | yield {'html-content': html} + |${plotlyHtmlYield()} | |""" finalcode.encode diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/dumbbellPlot/DumbbellPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/dumbbellPlot/DumbbellPlotOpDesc.scala index d9092583eee..57538636d30 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/dumbbellPlot/DumbbellPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/dumbbellPlot/DumbbellPlotOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.dumbbellPlot import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ @@ -168,26 +169,18 @@ class DumbbellPlotOpDesc extends PythonVisualizationOpDesc with PlotlyStandalone |import numpy as np | |class ProcessTableOperator(UDFTableOperator): - | def render_error(self, error_msg): - | return '''Reason is: {}
- | '''.format(error_msg) + |${runtimeError("DumbbellPlot is not available.")} | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - | if table.empty: - | yield {'html-content': self.render_error("input table is empty.")} - | return + |${emptyTableGuard("input table is empty.")} | table = table.dropna(subset=[$comparedColumnName, $categoryColumnName, $measurementColumnName]) #remove missing values - | if table.empty: - | yield {'html-content': self.render_error("input table has no rows with all of the configured columns filled in.")} - | return + |${emptyTableGuard("input table has no rows with all of the configured columns filled in.")} | ${createPlotlyDumbbellLineFigure()} | ${addPlotlyDots()} | # convert fig to html content | fig.update_layout(margin=dict(l=0, r=0, b=60, t=0)) - | html = plotly.io.to_html(fig, include_plotlyjs='cdn', auto_play=False) - | yield {'html-content': html} + |${plotlyHtmlYield()} | |""".encode } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ecdfPlot/ECDFPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ecdfPlot/ECDFPlotOpDesc.scala index 3331ad6f130..10ed3e41c71 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ecdfPlot/ECDFPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ecdfPlot/ECDFPlotOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.ecdfPlot import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode @@ -159,23 +160,18 @@ class ECDFPlotOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneCode |import plotly.io | |class ProcessTableOperator(UDFTableOperator): - | def render_error(self, error_msg): - | return '''Reason is: {}
- | '''.format(error_msg) + |${runtimeError("Empirical cumulative distribution plot is not available.")} | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - | if table.empty: - | yield {'html-content': self.render_error("input table is empty.")} - | return + |${emptyTableGuard("input table is empty.", bodyIndent = 4)} | ${manipulateTable()} - | if table.empty: - | yield {'html-content': self.render_error("no valid rows left after removing missing or non-numeric values.")} - | return + |${emptyTableGuard( + "no valid rows left after removing missing or non-numeric values.", + bodyIndent = 4 + )} | ${createPlotlyFigure()} - | html = plotly.io.to_html(fig, include_plotlyjs='cdn', auto_play=False) - | yield {'html-content': html} + |${plotlyHtmlYield()} |""" finalCode.encode } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/figureFactoryTable/FigureFactoryTableOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/figureFactoryTable/FigureFactoryTableOpDesc.scala index 5ac5c23eda7..127b5481c6a 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/figureFactoryTable/FigureFactoryTableOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/figureFactoryTable/FigureFactoryTableOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.figureFactoryTable import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ @@ -44,12 +45,7 @@ class FigureFactoryTableOpDesc extends PythonVisualizationOpDesc { @JsonProperty(required = false) @JsonSchemaTitle("Font Color (Hex Code)") @JsonPropertyDescription("Font color of the Figure Factory Table") - @JsonSchemaInject(json = """ -{ - "pattern": "^\\s*$|^\\s*#(?:\\s*[0-9a-fA-F]){3}(?:(?:\\s*[0-9a-fA-F]){3})?\\s*$|^\\s*(?:[rR]\\s*[gG]\\s*[bB]|[hH]\\s*[sS]\\s*[lL]|[hH]\\s*[sS]\\s*[vV])(?:\\s*[aA])?\\s*\\(\\s*(?:\\s*[0-9.])+(?:\\s*%)?(?:\\s*,(?:\\s*[0-9.])+(?:\\s*%)?){2,3}\\s*\\)\\s*$|^\\s*[vV]\\s*[aA]\\s*[rR]\\s*\\(\\s*-\\s*-[^)]*\\)\\s*$|^\\s*[a-zA-Z][a-zA-Z\\s]*$", - "examples": ["red"] -} -""") + @JsonSchemaInject(json = PythonVisualizationOpDesc.ColorPatternJson) var fontColor: EncodableString = "#000000" @JsonProperty(required = false) @@ -118,20 +114,15 @@ class FigureFactoryTableOpDesc extends PythonVisualizationOpDesc { | | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: | - | if table.empty: - | yield {'html-content': self.render_error("input table is empty.")} - | return + |${emptyTableGuard("input table is empty.")} | | ${manipulateTable()} | - | if table.empty: - | yield {'html-content': self.render_error("value column contains only non-positive numbers or nulls.")} - | return + |${emptyTableGuard("value column contains only non-positive numbers or nulls.")} | | ${createFigureFactoryTablePlotlyFigure()} | fig.update_layout(margin=dict(l=0, r=0, b=0, t=0)) - | html_content = plotly.io.to_html(fig, include_plotlyjs='cdn', include_mathjax='cdn') - | yield {'html-content': html_content} + |${plotlyHtmlYield(extraArgs = "include_mathjax='cdn'", yieldVar = "html_content")} """.stripMargin } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/funnelPlot/FunnelPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/funnelPlot/FunnelPlotOpDesc.scala index 3a8178a4846..27f5cffa902 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/funnelPlot/FunnelPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/funnelPlot/FunnelPlotOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.funnelPlot import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ @@ -91,20 +92,14 @@ class FunnelPlotOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneCo |import numpy as np | |class ProcessTableOperator(UDFTableOperator): - | def render_error(self, error_msg): - | return '''Reason is: {}
- | '''.format(error_msg) + |${runtimeError("Chart is not available.")} | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - | if table.empty: - | yield {'html-content': self.render_error("input table is empty.")} - | return + |${emptyTableGuard("input table is empty.")} | ${createPlotlyFigure()} | # convert fig to html content - | html = plotly.io.to_html(fig, include_plotlyjs='cdn', auto_play=False) - | yield {'html-content': html} + |${plotlyHtmlYield()} | |""" finalcode.encode diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ganttChart/GanttChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ganttChart/GanttChartOpDesc.scala index 74f82846823..632d2b65988 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ganttChart/GanttChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ganttChart/GanttChartOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.ganttChart import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ @@ -119,24 +120,16 @@ class GanttChartOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneCo |import numpy as np | |class ProcessTableOperator(UDFTableOperator): - | def render_error(self, error_msg): - | return '''Reason: {}
- | '''.format(error_msg) + |${runtimeError("Gantt Chart is not available.", colonOnly = true)} | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - | if table.empty: - | yield {'html-content': self.render_error("Input table is empty.")} - | return + |${emptyTableGuard("Input table is empty.")} | ${manipulateTable()} - | if table.empty: - | yield {'html-content': self.render_error("One or more of your input columns have all missing values")} - | return + |${emptyTableGuard("One or more of your input columns have all missing values")} | ${createPlotlyFigure()} | # convert fig to html content - | html = plotly.io.to_html(fig, include_plotlyjs='cdn', auto_play=False) - | yield {'html-content': html} + |${plotlyHtmlYield()} |""" finalCode.encode } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/gaugeChart/GaugeChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/gaugeChart/GaugeChartOpDesc.scala index 1324d41b8f0..04a491dd1d6 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/gaugeChart/GaugeChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/gaugeChart/GaugeChartOpDesc.scala @@ -19,6 +19,7 @@ package org.apache.texera.amber.operator.visualization.gaugeChart import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.fasterxml.jackson.databind.annotation.JsonDeserialize import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle @@ -94,9 +95,12 @@ class GaugeChartOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneCo | |class ProcessTableOperator(UDFTableOperator): | - | def render_error(self, error_msg) -> str: - | return '''Reason: {}
'''.format(error_msg) + |${runtimeError( + "Gauge chart is not available.", + typed = true, + colonOnly = true, + singleLineClose = true + )} | | def generate_gray_gradient(self, step_count): | colors = [] @@ -107,9 +111,7 @@ class GaugeChartOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneCo | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - | if table.empty: - | yield {'html-content': self.render_error("Input table is empty.")} - | return + |${emptyTableGuard("Input table is empty.", bodyIndent = 4)} | | try: | gauge_value = $value @@ -117,9 +119,11 @@ class GaugeChartOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneCo | threshold_val = $thresholdExpr | | table = table.dropna(subset=[gauge_value]) - | if table.empty: - | yield {'html-content': self.render_error("No non-null rows found for the value column.")} - | return + |${emptyTableGuard( + "No non-null rows found for the value column.", + indent = 12, + bodyIndent = 4 + )} | | valid_steps = $stepsExpr | step_colors = self.generate_gray_gradient(len(valid_steps)) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/heatMap/HeatMapOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/heatMap/HeatMapOpDesc.scala index 085f1adc990..efebf4113e6 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/heatMap/HeatMapOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/heatMap/HeatMapOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.heatMap import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ @@ -87,20 +88,14 @@ class HeatMapOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneCode |import numpy as np | |class ProcessTableOperator(UDFTableOperator): - | def render_error(self, error_msg): - | return '''Reason is: {}
- | '''.format(error_msg) + |${runtimeError("Chart is not available.")} | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - | if table.empty: - | yield {'html-content': self.render_error("input table is empty.")} - | return + |${emptyTableGuard("input table is empty.")} | ${createHeatMap()} | # convert fig to html content - | html = plotly.io.to_html(fig, include_plotlyjs='cdn', auto_play=False) - | yield {'html-content': html} + |${plotlyHtmlYield()} | |""" finalcode.encode diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/hierarchychart/HierarchyChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/hierarchychart/HierarchyChartOpDesc.scala index d216fd34731..7d8e16e735d 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/hierarchychart/HierarchyChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/hierarchychart/HierarchyChartOpDesc.scala @@ -19,32 +19,21 @@ package org.apache.texera.amber.operator.visualization.hierarchychart -import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ - PythonTemplateBuilderStringContext, - pyStringLiteral -} -import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString -import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode -import org.apache.texera.amber.operator.visualization.PythonVisualizationOpDesc import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} +import org.apache.texera.amber.operator.visualization.{ + PlotlyStandaloneCode, + PythonVisualizationOpDesc +} +import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.pybuilder.PythonTemplateBuilder import javax.validation.constraints.{NotEmpty, NotNull} // type constraint: value can only be numeric -@JsonSchemaInject(json = """ -{ - "attributeTypeRules": { - "value": { - "enum": ["integer", "long", "double"] - } - } -} -""") +@JsonSchemaInject(json = PythonVisualizationOpDesc.NumericValueRuleJson) class HierarchyChartOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneCode { @JsonProperty(required = true) @JsonSchemaTitle("Chart Type") @@ -74,93 +63,28 @@ class HierarchyChartOpDesc extends PythonVisualizationOpDesc with PlotlyStandalo OperatorGroupConstants.VISUALIZATION_BASIC_GROUP ) - private def getHierarchyAttributesInPython: String = - hierarchy.map(c => pyb"${c.attributeName}").mkString(",") + def manipulateTable(): PythonTemplateBuilder = + HierarchyChartSupport.filterPositiveAndDropNa(value, hierarchy) - def manipulateTable(): PythonTemplateBuilder = { - assert(value.nonEmpty, "Value Column cannot be empty") - val attributes = getHierarchyAttributesInPython - pyb""" - | table[$value] = table[table[$value] > 0][$value] # remove non-positive numbers from the data - | table.dropna(subset = [$attributes], inplace = True) #remove missing values - |""" - } - - def createPlotlyFigure(): PythonTemplateBuilder = { - assert(hierarchy.nonEmpty, "Hierarchy Path cannot be empty") - val attributes = getHierarchyAttributesInPython - pyb""" - | fig = px.${hierarchyChartType.getPlotlyExpressApiName}(table, path=[$attributes], values=$value, - | color=$value, hover_data=[$attributes], - | color_continuous_scale='RdBu') - |""" - } - - override def generatePythonCode(): String = { - val finalCode = - pyb""" - |from pytexera import * - | - |import plotly.express as px - |import plotly.graph_objects as go - |import plotly.io - |import numpy as np - | - |class ProcessTableOperator(UDFTableOperator): - | - | # Generate custom error message as html string - | def render_error(self, error_msg) -> str: - | return '''Reason is: {}
- | '''.format(error_msg) - | - | @overrides - | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - | if table.empty: - | yield {'html-content': self.render_error("input table is empty.")} - | return - | ${manipulateTable()} - | if table.empty: - | yield {'html-content': self.render_error("value column contains only non-positive numbers or nulls.")} - | return - | ${createPlotlyFigure()} - | # convert fig to html content - | fig.update_layout(margin=dict(l=0, r=0, b=0, t=0)) - | html = plotly.io.to_html(fig, include_plotlyjs='cdn', auto_play=False) - | yield {'html-content': html} - |""" - finalCode.encode - } + def createPlotlyFigure(): PythonTemplateBuilder = + HierarchyChartSupport.createPlotlyFigure( + hierarchyChartType.getPlotlyExpressApiName, + value, + hierarchy + ) - override def generateStandaloneCode(): String = { - val attributes = hierarchy.map(section => pyStringLiteral(section.attributeName)).mkString(", ") - val valueLit = pyStringLiteral(value) - // The error page is written to the same file a plotted chart lands in, so a - // reason for "no chart" is where the reader looks for the chart — printing it - // to the terminal alone left that file absent. render_error's continuation - // line keeps the runtime path's indentation, since the HTML is triple-quoted - // and those spaces reach the browser. chart_df is a copy: the input can feed another branch. - s"""${standaloneError("Hierarchy chart is not available.")} - | - |def fail(error_msg): - | with open(outputHtml, "w", encoding="utf-8") as output: - | output.write(render_error(error_msg)) - | print(f"Hierarchy chart error: {error_msg}") - | - |if in1df.empty: - | fail("input table is empty.") - |else: - | chart_df = in1df.copy() - | chart_df[$valueLit] = chart_df[chart_df[$valueLit] > 0][$valueLit] - | chart_df = chart_df.dropna(subset=[$attributes]) - | if chart_df.empty: - | fail("value column contains only non-positive numbers or nulls.") - | else: - | fig = px.${hierarchyChartType.getPlotlyExpressApiName}(chart_df, path=[$attributes], values=$valueLit, - | color=$valueLit, hover_data=[$attributes], - | color_continuous_scale='RdBu') - | fig.update_layout(margin=dict(l=0, r=0, b=0, t=0)) - |${standaloneFigure("Hierarchy chart")}""".stripMargin - } + override def generatePythonCode(): String = + HierarchyChartSupport.runtimePythonCode( + "Hierarchy chart is not available.", + manipulateTable(), + createPlotlyFigure() + ) + override def generateStandaloneCode(): String = + HierarchyChartSupport.standalonePythonCode( + "Hierarchy chart", + hierarchyChartType.getPlotlyExpressApiName, + value, + hierarchy + ) } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/hierarchychart/HierarchyChartSupport.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/hierarchychart/HierarchyChartSupport.scala new file mode 100644 index 00000000000..de20c583e38 --- /dev/null +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/hierarchychart/HierarchyChartSupport.scala @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.operator.visualization.hierarchychart + +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ +import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} + +/** Shared Python code generation helpers for hierarchy-path Plotly charts (Hierarchy, Icicle). */ +private[visualization] object HierarchyChartSupport { + def attributesInPython(hierarchy: List[HierarchySection]): String = + hierarchy.map(c => pyb"${c.attributeName}").mkString(",") + + def filterPositiveAndDropNa( + value: EncodableString, + hierarchy: List[HierarchySection] + ): PythonTemplateBuilder = { + assert(value.nonEmpty, "Value Column cannot be empty") + val attributes = attributesInPython(hierarchy) + pyb""" + | table[$value] = table[table[$value] > 0][$value] # remove non-positive numbers from the data + | table.dropna(subset = [$attributes], inplace = True) #remove missing values + |""" + } + + def createPlotlyFigure( + plotlyMethod: => String, + value: EncodableString, + hierarchy: List[HierarchySection] + ): PythonTemplateBuilder = { + assert(hierarchy.nonEmpty, "Hierarchy Path cannot be empty") + val attributes = attributesInPython(hierarchy) + pyb""" + | fig = px.$plotlyMethod(table, path=[$attributes], values=$value, + | color=$value, hover_data=[$attributes], + | color_continuous_scale='RdBu') + |""" + } + + def runtimePythonCode( + errorTitle: String, + tableCode: PythonTemplateBuilder, + figureCode: PythonTemplateBuilder + ): String = + pyb""" + |from pytexera import * + | + |import plotly.express as px + |import plotly.graph_objects as go + |import plotly.io + |import numpy as np + | + |class ProcessTableOperator(UDFTableOperator): + | + | # Generate custom error message as html string + |${runtimeError(errorTitle, typed = true)} + | + | @overrides + | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: + |${emptyTableGuard("input table is empty.")} + | $tableCode + |${emptyTableGuard("value column contains only non-positive numbers or nulls.")} + | $figureCode + | # convert fig to html content + | fig.update_layout(margin=dict(l=0, r=0, b=0, t=0)) + |${plotlyHtmlYield()} + |""".encode + + def standalonePythonCode( + chartLabel: String, + plotlyMethod: String, + value: EncodableString, + hierarchy: List[HierarchySection] + ): String = { + val attributes = hierarchy.map(section => pyStringLiteral(section.attributeName)).mkString(", ") + val valueLit = pyStringLiteral(value) + // The error page is written to the same file a plotted chart lands in, so a + // reason for "no chart" is where the reader looks for the chart — printing it + // to the terminal alone left that file absent. render_error's continuation + // line keeps the runtime path's indentation, since the HTML is triple-quoted + // and those spaces reach the browser. chart_df is a copy: the input can feed another branch. + s"""${standaloneError(s"$chartLabel is not available.")} + | + |def fail(error_msg): + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error(error_msg)) + | print(f"$chartLabel error: {error_msg}") + | + |if in1df.empty: + | fail("input table is empty.") + |else: + | chart_df = in1df.copy() + | chart_df[$valueLit] = chart_df[chart_df[$valueLit] > 0][$valueLit] + | chart_df = chart_df.dropna(subset=[$attributes]) + | if chart_df.empty: + | fail("value column contains only non-positive numbers or nulls.") + | else: + | fig = px.$plotlyMethod(chart_df, path=[$attributes], values=$valueLit, + | color=$valueLit, hover_data=[$attributes], + | color_continuous_scale='RdBu') + | fig.update_layout(margin=dict(l=0, r=0, b=0, t=0)) + |${standaloneFigure(chartLabel)}""".stripMargin + } +} diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/histogram/HistogramChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/histogram/HistogramChartOpDesc.scala index 3ccee094f90..2edbeb589d1 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/histogram/HistogramChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/histogram/HistogramChartOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.histogram import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ @@ -106,20 +107,14 @@ class HistogramChartOpDesc extends PythonVisualizationOpDesc with PlotlyStandalo |import numpy as np | |class ProcessTableOperator(UDFTableOperator): - | def render_error(self, error_msg): - | return '''Reason is: {}
- | '''.format(error_msg) + |${runtimeError("Histogram chart is not available.")} | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - | if table.empty: - | yield {'html-content': self.render_error("input table is empty.")} - | return + |${emptyTableGuard("input table is empty.")} | ${createPlotlyFigure()} | # convert fig to html content - | html = plotly.io.to_html(fig, include_plotlyjs='cdn', auto_play=False) - | yield {'html-content': html} + |${plotlyHtmlYield()} | |""" finalCode.encode diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/histogram2d/Histogram2DOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/histogram2d/Histogram2DOpDesc.scala index 012194293e0..99e7419ca24 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/histogram2d/Histogram2DOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/histogram2d/Histogram2DOpDesc.scala @@ -19,6 +19,7 @@ package org.apache.texera.amber.operator.visualization.histogram2d import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ @@ -92,15 +93,11 @@ class Histogram2DOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneC | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: | # Empty-table guard - | if table.empty: - | yield {"html-content": self.render_error("Input table is empty.")} - | return + |${emptyTableGuard("Input table is empty.", bodyIndent = 4, doubleQuoted = true)} | | # Drop rows with missing x/y | table.dropna(subset=[$xColumn, $yColumn], inplace=True) - | if table.empty: - | yield {"html-content": self.render_error("No rows after dropping nulls.")} - | return + |${emptyTableGuard("No rows after dropping nulls.", bodyIndent = 4, doubleQuoted = true)} | | fig = px.density_heatmap( | table, @@ -112,8 +109,7 @@ class Histogram2DOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneC | text_auto=True | ) | - | html = plotly.io.to_html(fig, include_plotlyjs='cdn', auto_play=False) - | yield {"html-content": html} + |${plotlyHtmlYield(doubleQuoted = true)} |""".encode } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/lineChart/LineChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/lineChart/LineChartOpDesc.scala index 23a04d66ad8..6c76c925c8f 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/lineChart/LineChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/lineChart/LineChartOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.lineChart import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ @@ -106,20 +107,14 @@ class LineChartOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneCod |class ProcessTableOperator(UDFTableOperator): | | # Generate custom error message as html string - | def render_error(self, error_msg) -> str: - | return '''Reason is: {}
- | '''.format(error_msg) + |${runtimeError("Line chart is not available.", typed = true)} | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - | if table.empty: - | yield {'html-content': self.render_error("input table is empty.")} - | return + |${emptyTableGuard("input table is empty.", bodyIndent = 4)} | ${createPlotlyFigure()} | # convert fig to html content - | html = plotly.io.to_html(fig, include_plotlyjs='cdn', auto_play=False) - | yield {'html-content': html} + |${plotlyHtmlYield()} |""" finalCode.encode } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/lineChart/LineConfig.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/lineChart/LineConfig.scala index c9c481b1b7e..84821eac58a 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/lineChart/LineConfig.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/lineChart/LineConfig.scala @@ -24,6 +24,8 @@ import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchema import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName +import org.apache.texera.amber.operator.visualization.PythonVisualizationOpDesc + import javax.validation.constraints.NotNull // Type constraint: both axes can only be numeric. The keys are the PROPERTY names -- @@ -78,12 +80,7 @@ class LineConfig { @JsonProperty(value = "color", required = false) @JsonSchemaTitle("Line Color") @JsonPropertyDescription("must be a valid CSS color or hex color string") - @JsonSchemaInject(json = """ -{ - "pattern": "^\\s*$|^\\s*#(?:\\s*[0-9a-fA-F]){3}(?:(?:\\s*[0-9a-fA-F]){3})?\\s*$|^\\s*(?:[rR]\\s*[gG]\\s*[bB]|[hH]\\s*[sS]\\s*[lL]|[hH]\\s*[sS]\\s*[vV])(?:\\s*[aA])?\\s*\\(\\s*(?:\\s*[0-9.])+(?:\\s*%)?(?:\\s*,(?:\\s*[0-9.])+(?:\\s*%)?){2,3}\\s*\\)\\s*$|^\\s*[vV]\\s*[aA]\\s*[rR]\\s*\\(\\s*-\\s*-[^)]*\\)\\s*$|^\\s*[a-zA-Z][a-zA-Z\\s]*$", - "examples": ["red"] -} -""") + @JsonSchemaInject(json = PythonVisualizationOpDesc.ColorPatternJson) var color: EncodableString = "" } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/nestedTable/NestedTableOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/nestedTable/NestedTableOpDesc.scala index c3a66b8490b..94ce50d345b 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/nestedTable/NestedTableOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/nestedTable/NestedTableOpDesc.scala @@ -19,6 +19,7 @@ package org.apache.texera.amber.operator.visualization.nestedTable import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext import org.apache.texera.amber.operator.visualization.PythonVisualizationOpDesc @@ -85,16 +86,11 @@ class NestedTableOpDesc extends PythonVisualizationOpDesc { |import numpy as np | |class ProcessTableOperator(UDFTableOperator): - | def render_error(self, error_msg): - | return '''Reason is: {}
- | '''.format(error_msg) + |${runtimeError("Nested Table is not available.")} | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - | if table.empty: - | yield {'html-content': self.render_error("input table is empty.")} - | return + |${emptyTableGuard("input table is empty.")} | ${createNestedTable()} | # convert table to html content | html = styled_table.to_html() diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/networkGraph/NetworkGraphOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/networkGraph/NetworkGraphOpDesc.scala index ebace5c7a39..ed07a2a797a 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/networkGraph/NetworkGraphOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/networkGraph/NetworkGraphOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.networkGraph import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext @@ -85,10 +86,7 @@ class NetworkGraphOpDesc extends PythonVisualizationOpDesc with PlotlyStandalone |import networkx as nx | |class ProcessTableOperator(UDFTableOperator): - | def render_error(self, error_msg): - | return '''Reason is: {}
- | '''.format(error_msg) + |${runtimeError("Network graph is not available.")} | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/parallelCoordinatesPlot/ParallelCoordinatesPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/parallelCoordinatesPlot/ParallelCoordinatesPlotOpDesc.scala index 3b63d7e9672..982a5a69e39 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/parallelCoordinatesPlot/ParallelCoordinatesPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/parallelCoordinatesPlot/ParallelCoordinatesPlotOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.parallelCoordinatesPlot import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode @@ -109,23 +110,15 @@ class ParallelCoordinatesPlotOpDesc extends PythonVisualizationOpDesc with Plotl | |class ProcessTableOperator(UDFTableOperator): | - | def render_error(self, error_msg): - | return '''Reason is: {}
- | '''.format(error_msg) + |${runtimeError("Parallel coordinates plot is not available.")} | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - | if table.empty: - | yield {'html-content': self.render_error("Input table is empty.")} - | return + |${emptyTableGuard("Input table is empty.", bodyIndent = 4)} | ${manipulateTable()} - | if table.empty: - | yield {'html-content': self.render_error("No valid rows after filtering.")} - | return + |${emptyTableGuard("No valid rows after filtering.", bodyIndent = 4)} | ${createPlotlyFigure()} - | html = plotly.io.to_html(fig, include_plotlyjs='cdn', auto_play=False) - | yield {'html-content': html} + |${plotlyHtmlYield()} |""" finalcode.encode } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/pieChart/PieChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/pieChart/PieChartOpDesc.scala index ac4197db9b4..6cde06161a6 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/pieChart/PieChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/pieChart/PieChartOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.pieChart import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ @@ -36,15 +37,7 @@ import org.apache.texera.amber.pybuilder.PythonTemplateBuilder import javax.validation.constraints.NotNull // type constraint: value can only be numeric -@JsonSchemaInject(json = """ -{ - "attributeTypeRules": { - "value": { - "enum": ["integer", "long", "double"] - } - } -} -""") +@JsonSchemaInject(json = PythonVisualizationOpDesc.NumericValueRuleJson) class PieChartOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneCode { @JsonProperty(value = "value", required = true) @@ -96,29 +89,21 @@ class PieChartOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneCode |import numpy as np | |class ProcessTableOperator(UDFTableOperator): - | def render_error(self, error_msg): - | return '''Reason is: {}
- | '''.format(error_msg) + |${runtimeError("PieChart is not available.")} | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: | original_table = table - | if table.empty: - | yield {'html-content': self.render_error("input table is empty.")} - | return + |${emptyTableGuard("input table is empty.")} | ${manipulateTable()} - | if table.empty: - | yield {'html-content': self.render_error("value column contains only non-positive numbers.")} - | return + |${emptyTableGuard("value column contains only non-positive numbers.")} | duplicates = table.duplicated(subset=[$name]) | if duplicates.any(): | yield {'html-content': self.render_error("duplicates in name column, need to aggregate")} | return | ${createPlotlyFigure()} | # convert fig to html content - | html = plotly.io.to_html(fig, include_plotlyjs='cdn', auto_play=False) - | yield {'html-content': html} + |${plotlyHtmlYield()} | |""" finalcode.encode diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/quiverPlot/QuiverPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/quiverPlot/QuiverPlotOpDesc.scala index b2380f389b5..f76dbc8ec0e 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/quiverPlot/QuiverPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/quiverPlot/QuiverPlotOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.quiverPlot import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext @@ -110,16 +111,11 @@ class QuiverPlotOpDesc extends PythonVisualizationOpDesc { | |class ProcessTableOperator(UDFTableOperator): | - | def render_error(self, error_msg): - | return '''Reasons are: {}
- | '''.format(error_msg) + |${runtimeError("Quiver Plot is not available.", plural = true)} | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - | if table.empty: - | yield {'html-content': self.render_error("Input table is empty.")} - | return + |${emptyTableGuard("Input table is empty.", bodyIndent = 4)} | | required_columns = {$x, $y, $u, $v} | if not required_columns.issubset(table.columns): @@ -147,8 +143,7 @@ class QuiverPlotOpDesc extends PythonVisualizationOpDesc { | yield {'html-content': self.render_error(f"Plotly error: {str(e)}")} | return | - | html = plotly.io.to_html(fig, include_plotlyjs='cdn', auto_play=False) - | yield {'html-content': html} + |${plotlyHtmlYield()} |""" finalCode.encode } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/radarChart/RadarChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/radarChart/RadarChartOpDesc.scala index 6e81b332551..8158de76a96 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/radarChart/RadarChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/radarChart/RadarChartOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.radarChart import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode @@ -136,23 +137,15 @@ class RadarChartOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneCo |import pandas as pd | |class ProcessTableOperator(UDFTableOperator): - | def render_error(self, error_msg): - | return '''Reason is: {}
- | '''.format(error_msg) + |${runtimeError("RadarChart is not available.")} | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - | if table.empty: - | yield {'html-content': self.render_error("input table is empty.")} - | return + |${emptyTableGuard("input table is empty.")} | ${manipulateTable()} - | if table.empty: - | yield {'html-content': self.render_error("input table is empty after removing missing values.")} - | return + |${emptyTableGuard("input table is empty after removing missing values.")} | ${createPlotlyFigure()} - | html = plotly.io.to_html(fig, include_plotlyjs='cdn', auto_play=False) - | yield {'html-content': html} + |${plotlyHtmlYield()} | |""" finalcode.encode diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/radarPlot/RadarPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/radarPlot/RadarPlotOpDesc.scala index 3ff89e666bb..f9b248b9722 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/radarPlot/RadarPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/radarPlot/RadarPlotOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.radarPlot import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.operator.metadata.annotations.{ @@ -152,21 +153,15 @@ class RadarPlotOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneCod | |class ProcessTableOperator(UDFTableOperator): | - | def render_error(self, error_msg): - | return '''Reason is: {}
- | '''.format(error_msg) + |${runtimeError("Radar Plot is not available.")} | | @overrides | def process_table(self, table: Table, port: int): - | if table.empty: - | yield {'html-content': self.render_error("Input table is empty.")} - | return + |${emptyTableGuard("Input table is empty.", bodyIndent = 4)} | | ${generateRadarPlotCode()} | - | html = plotly.io.to_html(fig, include_plotlyjs='cdn', auto_play=False, config={'responsive': True}) - | yield {'html-content': html} + |${plotlyHtmlYield(extraArgs = "auto_play=False, config={'responsive': True}")} |""" finalCode.encode } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/rangeSlider/RangeSliderOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/rangeSlider/RangeSliderOpDesc.scala index 7deac358d4a..0745cfd6256 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/rangeSlider/RangeSliderOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/rangeSlider/RangeSliderOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.rangeSlider import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext @@ -113,25 +114,19 @@ class RangeSliderOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneC |import numpy as np | |class ProcessTableOperator(UDFTableOperator): - | def render_error(self, error_msg): - | return '''Reason is: {}
- | '''.format(error_msg) + |${runtimeError("RangeChart is not available.")} | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: | original_table = table - | if table.empty: - | yield {'html-content': self.render_error("input table is empty.")} - | return + |${emptyTableGuard("input table is empty.")} | if $yAxis.strip() == "" or $xAxis.strip() == "": | yield {'html-content': self.render_error("Y-axis or X-axis is empty")} | return | ${manipulateTable()} | ${createPlotlyFigure()} | # convert fig to html content - | html = plotly.io.to_html(fig, include_plotlyjs='cdn', auto_play=False) - | yield {'html-content': html} + |${plotlyHtmlYield()} |""" finalcode.encode } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/sankeyDiagram/SankeyDiagramOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/sankeyDiagram/SankeyDiagramOpDesc.scala index 65798707fd1..9cb29a7a22c 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/sankeyDiagram/SankeyDiagramOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/sankeyDiagram/SankeyDiagramOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.sankeyDiagram import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext @@ -119,23 +120,18 @@ class SankeyDiagramOpDesc extends PythonVisualizationOpDesc with PlotlyStandalon | |class ProcessTableOperator(UDFTableOperator): | - | def render_error(self, error_msg): - | return '''Reasons are: {}
- | '''.format(error_msg) + |${runtimeError("Sankey Diagram is not available.", plural = true)} | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - | if table.empty: - | yield {'html-content': self.render_error("Input table is empty.")} - | return + |${emptyTableGuard("Input table is empty.", bodyIndent = 4)} | ${createPlotlyFigure()} - | if table.empty: - | yield {'html-content': self.render_error("No valid rows left (every row has at least 1 missing value).")} - | return + |${emptyTableGuard( + "No valid rows left (every row has at least 1 missing value).", + bodyIndent = 4 + )} | # convert fig to html content - | html = plotly.io.to_html(fig, include_plotlyjs='cdn', auto_play=False) - | yield {'html-content': html} + |${plotlyHtmlYield()} |""" finalCode.encode } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/scatter3DChart/Scatter3dChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/scatter3DChart/Scatter3dChartOpDesc.scala index c07ca12755b..9a510aa4e8f 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/scatter3DChart/Scatter3dChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/scatter3DChart/Scatter3dChartOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.scatter3DChart import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext @@ -101,20 +102,14 @@ class Scatter3dChartOpDesc extends PythonVisualizationOpDesc with PlotlyStandalo |import numpy as np | |class ProcessTableOperator(UDFTableOperator): - | def render_error(self, error_msg): - | return '''Reason is: {}
- | '''.format(error_msg) + |${runtimeError("Chart is not available.")} | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - | if table.empty: - | yield {'html-content': self.render_error("input table is empty.")} - | return + |${emptyTableGuard("input table is empty.")} | ${createPlotlyFigure()} | # convert fig to html content - | html = plotly.io.to_html(fig, include_plotlyjs='cdn', auto_play=False) - | yield {'html-content': html} + |${plotlyHtmlYield()} | |""" finalcode.encode diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/scatterplot/ScatterplotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/scatterplot/ScatterplotOpDesc.scala index cd07ae535a1..48753c2d0b0 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/scatterplot/ScatterplotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/scatterplot/ScatterplotOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.scatterplot import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext @@ -148,23 +149,22 @@ class ScatterplotOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneC | |class ProcessTableOperator(UDFTableOperator): | - | def render_error(self, error_msg): - | return '''Reasons are: {}
- | '''.format(error_msg) + |${runtimeError("Scatter Plot is not available.", plural = true)} | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - | if table.empty: - | yield {'html-content': self.render_error("Input table is empty.")} - | return + |${emptyTableGuard("Input table is empty.", bodyIndent = 4)} | ${manipulateTable()} | ${createPlotlyFigure()} - | if table.empty: - | yield {'html-content': self.render_error("No valid rows left (every row has at least 1 missing value).")} - | return - | html = plotly.io.to_html(fig, include_plotlyjs = 'cdn', auto_play = False) - | yield {'html-content':html} + |${emptyTableGuard( + "No valid rows left (every row has at least 1 missing value).", + bodyIndent = 4 + )} + |${plotlyHtmlYield( + extraArgs = "auto_play = False", + spacesAroundEq = true, + compactDict = true + )} |""" finalCode.encode } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/tablesChart/TablesPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/tablesChart/TablesPlotOpDesc.scala index 2d36d8b6cb7..d687f2b50a3 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/tablesChart/TablesPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/tablesChart/TablesPlotOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.tablesChart import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode @@ -82,15 +83,11 @@ class TablesPlotOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneCo | | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: | - | if table.empty: - | yield {'html-content': self.render_error("input table is empty.")} - | return + |${emptyTableGuard("input table is empty.")} | | ${manipulateTable()} | - | if table.empty: - | yield {'html-content': self.render_error("value column contains only non-positive numbers or nulls.")} - | return + |${emptyTableGuard("value column contains only non-positive numbers or nulls.")} | | ${createPlotlyFigure()} diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ternaryContour/TernaryContourOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ternaryContour/TernaryContourOpDesc.scala index d9da28438f0..53a59536463 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ternaryContour/TernaryContourOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ternaryContour/TernaryContourOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.ternaryContour import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.core.workflow.OutputPort.OutputMode @@ -147,24 +148,23 @@ class TernaryContourOpDesc extends PythonVisualizationOpDesc { |class ProcessTableOperator(UDFTableOperator): | | # Generate custom error message as html string - | def render_error(self, error_msg): - | return '''Reasons are: {}
- | '''.format(error_msg) + |${runtimeError("TernaryContour is not available.", plural = true)} | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - | if table.empty: - | yield {'html-content': self.render_error("Input table is empty.")} - | return + |${emptyTableGuard("Input table is empty.", bodyIndent = 4)} | ${manipulateTable()} - | if table.empty: - | yield {'html-content': self.render_error("No valid rows left (every row has at least 1 missing value).")} - | return + |${emptyTableGuard( + "No valid rows left (every row has at least 1 missing value).", + bodyIndent = 4 + )} | ${createPlotlyFigure()} | # Convert fig to html content - | html = plotly.io.to_html(fig, include_plotlyjs = 'cdn', auto_play = False) - | yield {'html-content':html} + |${plotlyHtmlYield( + extraArgs = "auto_play = False", + spacesAroundEq = true, + compactDict = true + )} |""" finalCode.encode } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ternaryPlot/TernaryPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ternaryPlot/TernaryPlotOpDesc.scala index f3347147f22..d497b48526f 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ternaryPlot/TernaryPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ternaryPlot/TernaryPlotOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.ternaryPlot import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext @@ -134,24 +135,23 @@ class TernaryPlotOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneC |class ProcessTableOperator(UDFTableOperator): | | # Generate custom error message as html string - | def render_error(self, error_msg): - | return '''Reasons are: {}
- | '''.format(error_msg) + |${runtimeError("TernaryPlot is not available.", plural = true)} | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - | if table.empty: - | yield {'html-content': self.render_error("Input table is empty.")} - | return + |${emptyTableGuard("Input table is empty.", bodyIndent = 4)} | ${manipulateTable()} - | if table.empty: - | yield {'html-content': self.render_error("No valid rows left (every row has at least 1 missing value).")} - | return + |${emptyTableGuard( + "No valid rows left (every row has at least 1 missing value).", + bodyIndent = 4 + )} | ${createPlotlyFigure()} | # Convert fig to html content - | html = plotly.io.to_html(fig, include_plotlyjs = 'cdn', auto_play = False) - | yield {'html-content':html} + |${plotlyHtmlYield( + extraArgs = "auto_play = False", + spacesAroundEq = true, + compactDict = true + )} |""" finalCode.encode } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/timeSeriesplot/TimeSeriesOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/timeSeriesplot/TimeSeriesOpDesc.scala index b350c9544b8..8b178395fce 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/timeSeriesplot/TimeSeriesOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/timeSeriesplot/TimeSeriesOpDesc.scala @@ -19,6 +19,7 @@ package org.apache.texera.amber.operator.visualization.timeSeriesplot import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext @@ -124,17 +125,13 @@ class TimeSeriesOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneCo | | @overrides | def process_table(self, table: Table, port: int): - | if table.empty: - | yield {'html-content': self.render_error("Input table is empty.")} - | return + |${emptyTableGuard("Input table is empty.", bodyIndent = 4)} | | try: | table[$timeColumn] = pd.to_datetime(table[$timeColumn], errors='coerce') | table = table.dropna(subset=$dropnaStr).sort_values(by=$timeColumn) | - | if table.empty: - | yield {'html-content': self.render_error("Table became empty after filtering.")} - | return + |${emptyTableGuard("Table became empty after filtering.", indent = 12, bodyIndent = 4)} | | fig = $plotFunc(table, x=$timeColumn, y=$valueColumn$colorArg$facetArg) | @@ -149,8 +146,7 @@ class TimeSeriesOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneCo | template="plotly_white" | ) | - | html = plotly.io.to_html(fig, include_plotlyjs='cdn', full_html=False) - | yield {'html-content': html} + |${plotlyHtmlYield(indent = 12, extraArgs = "full_html=False")} | | except Exception as e: | yield {'html-content': self.render_error(str(e))} diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/treeplot/TreePlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/treeplot/TreePlotOpDesc.scala index 7d66be03825..f5397cea54a 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/treeplot/TreePlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/treeplot/TreePlotOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.treeplot import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext @@ -104,9 +105,7 @@ class TreePlotOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneCode | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - | if table.empty: - | yield {'html-content': self.render_error("Input table is empty.")} - | return + |${emptyTableGuard("Input table is empty.", bodyIndent = 4)} | | edges = [] | for item in table[$edgeListColumn].dropna(): @@ -174,8 +173,7 @@ class TreePlotOpDesc extends PythonVisualizationOpDesc with PlotlyStandaloneCode | hovermode='closest', | plot_bgcolor='rgb(248,248,248)') | - | html = plotly.io.to_html(fig, include_plotlyjs='cdn', auto_play=False) - | yield {'html-content': html} + |${plotlyHtmlYield()} | |""".encode } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/waterfallChart/WaterfallChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/waterfallChart/WaterfallChartOpDesc.scala index c2e6921dec5..719e0e9a741 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/waterfallChart/WaterfallChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/waterfallChart/WaterfallChartOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.waterfallChart import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext @@ -101,19 +102,13 @@ class WaterfallChartOpDesc extends PythonVisualizationOpDesc with PlotlyStandalo |class ProcessTableOperator(UDFTableOperator): | | # Generate custom error message as html string - | def render_error(self, error_msg) -> str: - | return '''Reason is: {}
- | '''.format(error_msg) + |${runtimeError("Waterfall chart is not available.", typed = true)} | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - | if table.empty: - | yield {'html-content': self.render_error("input table is empty.")} - | return + |${emptyTableGuard("input table is empty.", bodyIndent = 4)} | ${createPlotlyFigure()} - | html = plotly.io.to_html(fig, include_plotlyjs='cdn', auto_play=False) - | yield {'html-content': html} + |${plotlyHtmlYield()} |""" finalCode.encode } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/windRoseChart/WindRoseChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/windRoseChart/WindRoseChartOpDesc.scala index f7ed97ef2d3..fc774fafdc3 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/windRoseChart/WindRoseChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/windRoseChart/WindRoseChartOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.windRoseChart import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.core.workflow.OutputPort.OutputMode @@ -110,24 +111,18 @@ class WindRoseChartOpDesc extends PythonVisualizationOpDesc with PlotlyStandalon |class ProcessTableOperator(UDFTableOperator): | | # Generate custom error message as html string - | def render_error(self, error_msg) -> str: - | return '''Reason is: {}
- | '''.format(error_msg) + |${runtimeError("Wind Rose chart is not available.", typed = true)} | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - | if table.empty: - | yield {'html-content': self.render_error("input table is empty.")} - | return + |${emptyTableGuard("input table is empty.", bodyIndent = 4)} | if table[$rColumn].dtype.kind not in ["i", "u", "f"]: | yield {'html-content': self.render_error( | "Radial column must be numeric (int, float, or double)." | )} | return | ${createPlotlyFigure()} - | html = plotly.io.to_html(fig, include_plotlyjs='cdn', auto_play=False) - | yield {'html-content': html} + |${plotlyHtmlYield()} |""" finalCode.encode } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/wordCloud/WordCloudOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/wordCloud/WordCloudOpDesc.scala index 7ff9c392807..01b80d1fa3d 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/wordCloud/WordCloudOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/wordCloud/WordCloudOpDesc.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.operator.visualization.wordCloud import org.apache.texera.amber.operator.visualization.StandaloneChartCode._ +import org.apache.texera.amber.operator.visualization.PythonCodeFragment._ import com.fasterxml.jackson.annotation.JsonProperty import com.kjetland.jackson.jsonSchema.annotations.{ JsonSchemaInject, @@ -112,21 +113,14 @@ class WordCloudOpDesc extends PythonVisualizationOpDesc { |class ProcessTableOperator(UDFTableOperator): | | # Generate custom error message as html string - | def render_error(self, error_msg) -> str: - | return '''Reason is: {}
- | '''.format(error_msg) + |${runtimeError("Wordcloud is not available.", typed = true)} | | @overrides | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: - | if table.empty: - | yield {'html-content': self.render_error("input table is empty.")} - | return + |${emptyTableGuard("input table is empty.")} | ${guardNotText()} | ${manipulateTable()} - | if table.empty: - | yield {'html-content': self.render_error("text column does not contain words or contains only nulls.")} - | return + |${emptyTableGuard("text column does not contain words or contains only nulls.")} | ${createWordCloudFigure()} | ${ImageUtility.encodeImageToHTML()} | yield {'html-content': html} diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpSpec.scala index 9bd4e476e27..34d29127b29 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregateOpSpec.scala @@ -19,354 +19,216 @@ package org.apache.texera.amber.operator.aggregate -import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema, Tuple} -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.core.tuple.AttributeType import org.scalatest.funsuite.AnyFunSuite import java.sql.Timestamp class AggregateOpSpec extends AnyFunSuite { - - /** Helpers */ - - private def makeAggregationOp( - fn: AggregationFunction, - attributeName: String, - resultName: String - ): AggregationOperation = { - val operation = new AggregationOperation() - operation.aggFunction = fn - operation.attribute = attributeName - operation.resultAttribute = resultName - operation - } - - private def makeSchema(fields: (String, AttributeType)*): Schema = - Schema(fields.map { case (n, t) => new Attribute(n, t) }.toList) - - private def makeTuple(schema: Schema, values: Any*): Tuple = - Tuple(schema, values.toArray) + import AggregationTestSupport._ test("getAggregationAttribute keeps original type for SUM") { - val operation = makeAggregationOp(AggregationFunction.SUM, "amount", "total_amount") - val attr = operation.getAggregationAttribute(AttributeType.DOUBLE) - - assert(attr.getName == "total_amount") - assert(attr.getType == AttributeType.DOUBLE) + assertAggregationAttribute( + AggregationFunction.SUM, + "amount", + "total_amount", + AttributeType.DOUBLE, + AttributeType.DOUBLE + ) } test("getAggregationAttribute maps COUNT result to INTEGER regardless of input type") { - val operation = makeAggregationOp(AggregationFunction.COUNT, "quantity", "row_count") - val attr = operation.getAggregationAttribute(AttributeType.LONG) - - assert(attr.getName == "row_count") - assert(attr.getType == AttributeType.INTEGER) + assertAggregationAttribute( + AggregationFunction.COUNT, + "quantity", + "row_count", + AttributeType.LONG, + AttributeType.INTEGER + ) } test("getAggregationAttribute maps COUNT result to INTEGER even with a null input type") { // COUNT(*) (empty attribute) has no input column, so schema propagation passes a // null attrType; it must still resolve to INTEGER without dereferencing it. - val operation = makeAggregationOp(AggregationFunction.COUNT, "", "row_count") - val attr = operation.getAggregationAttribute(null) - - assert(attr.getName == "row_count") - assert(attr.getType == AttributeType.INTEGER) + assertAggregationAttribute( + AggregationFunction.COUNT, + "", + "row_count", + null, + AttributeType.INTEGER + ) } test("getAggregationAttribute maps CONCAT result type to STRING") { - val operation = makeAggregationOp(AggregationFunction.CONCAT, "tag", "all_tags") - val attr = operation.getAggregationAttribute(AttributeType.INTEGER) - - assert(attr.getName == "all_tags") - assert(attr.getType == AttributeType.STRING) + assertAggregationAttribute( + AggregationFunction.CONCAT, + "tag", + "all_tags", + AttributeType.INTEGER, + AttributeType.STRING + ) } test("getAggregationAttribute maps AVERAGE result type to DOUBLE regardless of input") { - val operation = makeAggregationOp(AggregationFunction.AVERAGE, "price", "avg_price") - val attr = operation.getAggregationAttribute(AttributeType.LONG) - - assert(attr.getName == "avg_price") - assert(attr.getType == AttributeType.DOUBLE) + assertAggregationAttribute( + AggregationFunction.AVERAGE, + "price", + "avg_price", + AttributeType.LONG, + AttributeType.DOUBLE + ) } test("getAggregationAttribute keeps original type for MIN") { - val operation = makeAggregationOp(AggregationFunction.MIN, "ts", "min_ts") - val attr = operation.getAggregationAttribute(AttributeType.TIMESTAMP) - - assert(attr.getName == "min_ts") - assert(attr.getType == AttributeType.TIMESTAMP) + assertAggregationAttribute( + AggregationFunction.MIN, + "ts", + "min_ts", + AttributeType.TIMESTAMP, + AttributeType.TIMESTAMP + ) } test("getAggregationAttribute keeps original type for MAX") { - val operation = makeAggregationOp(AggregationFunction.MAX, "score", "max_score") - val attr = operation.getAggregationAttribute(AttributeType.DOUBLE) - - assert(attr.getName == "max_score") - assert(attr.getType == AttributeType.DOUBLE) + assertAggregationAttribute( + AggregationFunction.MAX, + "score", + "max_score", + AttributeType.DOUBLE, + AttributeType.DOUBLE + ) } test("getAggregationAttribute throws RuntimeException when aggFunction is null") { - val operation = new AggregationOperation() - operation.attribute = "src" - operation.resultAttribute = "out" - // aggFunction left null on purpose - + val operation = makeAggregationOp(null, "src", "out") assertThrows[RuntimeException] { operation.getAggregationAttribute(AttributeType.INTEGER) } } - // --------------------------------------------------------------------------- - // Basic DistributedAggregation behaviour via AggregationOperation.getAggFunc - // --------------------------------------------------------------------------- - test("SUM aggregation over INTEGER column adds values correctly") { - val schema = makeSchema("amount" -> AttributeType.INTEGER) - val tuple1 = makeTuple(schema, 5) - val tuple2 = makeTuple(schema, 7) - val tuple3 = makeTuple(schema, 3) - - val operation = makeAggregationOp(AggregationFunction.SUM, "amount", "total_amount") - val agg = operation.getAggFunc(AttributeType.INTEGER) - - var partial = agg.init() - partial = agg.iterate(partial, tuple1) - partial = agg.iterate(partial, tuple2) - partial = agg.iterate(partial, tuple3) - - val result = agg.finalAgg(partial).asInstanceOf[Number].intValue() - assert(result == 15) + val result = + runSingleAttrAgg(AggregationFunction.SUM, "amount", AttributeType.INTEGER, Seq(5, 7, 3)) + assert(result.asInstanceOf[Number].intValue() == 15) } test("SUM aggregation over DOUBLE column keeps fractional part") { - val schema = makeSchema("score" -> AttributeType.DOUBLE) - val tuple1 = makeTuple(schema, 1.25) - val tuple2 = makeTuple(schema, 2.75) - - val operation = makeAggregationOp(AggregationFunction.SUM, "score", "total_score") - val agg = operation.getAggFunc(AttributeType.DOUBLE) - - var partial = agg.init() - partial = agg.iterate(partial, tuple1) - partial = agg.iterate(partial, tuple2) - - val result = agg.finalAgg(partial).asInstanceOf[java.lang.Double].doubleValue() - assert(math.abs(result - 4.0) < 1e-6) + val result = + runSingleAttrAgg(AggregationFunction.SUM, "score", AttributeType.DOUBLE, Seq(1.25, 2.75)) + assert(math.abs(result.asInstanceOf[java.lang.Double].doubleValue() - 4.0) < 1e-6) } test("COUNT with an empty attribute (COUNT(*)) counts all rows regardless of nulls") { - // An empty attribute means COUNT(*); the GUI sends "" when no column is selected. - val schema = makeSchema("points" -> AttributeType.INTEGER) - val tuple1 = makeTuple(schema, 10) - val tuple2 = makeTuple(schema, null) - val tuple3 = makeTuple(schema, 20) - - val operation = makeAggregationOp(AggregationFunction.COUNT, "", "row_count") - val agg = operation.getAggFunc(AttributeType.INTEGER) - - var partial = agg.init() - partial = agg.iterate(partial, tuple1) - partial = agg.iterate(partial, tuple2) - partial = agg.iterate(partial, tuple3) - - val result = agg.finalAgg(partial).asInstanceOf[Number].intValue() - assert(result == 3) + val result = + runSingleAttrAgg(AggregationFunction.COUNT, "", AttributeType.INTEGER, Seq(10, null, 20)) + assert(result.asInstanceOf[Number].intValue() == 3) } test("COUNT with a null attribute also counts all rows") { - // A null attribute is treated the same as empty (COUNT(*)). - val schema = makeSchema("points" -> AttributeType.INTEGER) - val tuple1 = makeTuple(schema, 10) - val tuple2 = makeTuple(schema, null) - val tuple3 = makeTuple(schema, 20) - - val operation = makeAggregationOp(AggregationFunction.COUNT, null, "row_count") - val agg = operation.getAggFunc(AttributeType.INTEGER) - - var partial = agg.init() - partial = agg.iterate(partial, tuple1) - partial = agg.iterate(partial, tuple2) - partial = agg.iterate(partial, tuple3) - - val result = agg.finalAgg(partial).asInstanceOf[Number].intValue() - assert(result == 3) + val result = + runSingleAttrAgg(AggregationFunction.COUNT, null, AttributeType.INTEGER, Seq(10, null, 20)) + assert(result.asInstanceOf[Number].intValue() == 3) } test("COUNT aggregation with attribute set only counts non-null values") { - val schema = makeSchema("points" -> AttributeType.INTEGER) - val tuple1 = makeTuple(schema, 10) - val tuple2 = makeTuple(schema, null) - val tuple3 = makeTuple(schema, 5) - - val operation = makeAggregationOp(AggregationFunction.COUNT, "points", "non_null_points") - val agg = operation.getAggFunc(AttributeType.INTEGER) - - var partial = agg.init() - partial = agg.iterate(partial, tuple1) - partial = agg.iterate(partial, tuple2) - partial = agg.iterate(partial, tuple3) - - val result = agg.finalAgg(partial).asInstanceOf[Number].intValue() - assert(result == 2) + val result = + runSingleAttrAgg(AggregationFunction.COUNT, "points", AttributeType.INTEGER, Seq(10, null, 5)) + assert(result.asInstanceOf[Number].intValue() == 2) } test("CONCAT aggregation concatenates string representations with commas") { - val schema = makeSchema("tag" -> AttributeType.STRING) - val tuple1 = makeTuple(schema, "red") - val tuple2 = makeTuple(schema, null) - val tuple3 = makeTuple(schema, "blue") - - val operation = makeAggregationOp(AggregationFunction.CONCAT, "tag", "all_tags") - val agg = operation.getAggFunc(AttributeType.STRING) - - var partial = agg.init() - partial = agg.iterate(partial, tuple1) - partial = agg.iterate(partial, tuple2) - partial = agg.iterate(partial, tuple3) - - val result = agg.finalAgg(partial).asInstanceOf[String] - assert(result == "red,,blue") + val result = runSingleAttrAgg( + AggregationFunction.CONCAT, + "tag", + AttributeType.STRING, + Seq("red", null, "blue") + ) + assert(result.asInstanceOf[String] == "red,,blue") } test("MIN aggregation finds smallest INTEGER and returns null when given no values") { - val schema = makeSchema("temperature" -> AttributeType.INTEGER) - val tuple1 = makeTuple(schema, 10) - val tuple2 = makeTuple(schema, -2) - val tuple3 = makeTuple(schema, 5) - - val operation = makeAggregationOp(AggregationFunction.MIN, "temperature", "min_temp") - val agg = operation.getAggFunc(AttributeType.INTEGER) - - // Empty case: never iterate, just finalize init - val emptyPartial = agg.init() - val emptyResult = agg.finalAgg(emptyPartial) - assert(emptyResult == null) - - // Non-empty case - var partial = agg.init() - partial = agg.iterate(partial, tuple1) - partial = agg.iterate(partial, tuple2) - partial = agg.iterate(partial, tuple3) - - val result = agg.finalAgg(partial).asInstanceOf[Number].intValue() - assert(result == -2) + assert( + runSingleAttrAgg( + AggregationFunction.MIN, + "temperature", + AttributeType.INTEGER, + Seq.empty + ) == null + ) + val result = runSingleAttrAgg( + AggregationFunction.MIN, + "temperature", + AttributeType.INTEGER, + Seq(10, -2, 5) + ) + assert(result.asInstanceOf[Number].intValue() == -2) } test("MAX aggregation finds largest LONG value") { - val schema = makeSchema("latency" -> AttributeType.LONG) - val tuple1 = makeTuple(schema, 100L) - val tuple2 = makeTuple(schema, 50L) - val tuple3 = makeTuple(schema, 250L) - - val operation = makeAggregationOp(AggregationFunction.MAX, "latency", "max_latency") - val agg = operation.getAggFunc(AttributeType.LONG) - - var partial = agg.init() - partial = agg.iterate(partial, tuple1) - partial = agg.iterate(partial, tuple2) - partial = agg.iterate(partial, tuple3) - - val result = agg.finalAgg(partial).asInstanceOf[java.lang.Long].longValue() - assert(result == 250L) + val result = + runSingleAttrAgg(AggregationFunction.MAX, "latency", AttributeType.LONG, Seq(100L, 50L, 250L)) + assert(result.asInstanceOf[java.lang.Long].longValue() == 250L) } test( "MIN aggregation (DOUBLE) is solid: empty/null/NaN, infinities, signed zero, and many values" ) { val schema = makeSchema("temperature" -> AttributeType.DOUBLE) - - val operation = makeAggregationOp(AggregationFunction.MIN, "temperature", "min_temp") - val agg = operation.getAggFunc(AttributeType.DOUBLE) - - // ----------------------- - // 0) Empty input => null - // ----------------------- - val emptyPartial = agg.init() - val emptyResult = agg.finalAgg(emptyPartial) - assert(emptyResult == null) - - // --------------------------------------------------------- - // 1) Only nulls / only NaNs => null - // --------------------------------------------------------- - val onlyNulls = Seq(makeTuple(schema, null), makeTuple(schema, null), makeTuple(schema, null)) - var partialNulls = agg.init() - onlyNulls.foreach(tp => partialNulls = agg.iterate(partialNulls, tp)) - assert(agg.finalAgg(partialNulls) == null) - - val onlyNaNs = Seq(makeTuple(schema, Double.NaN), makeTuple(schema, Double.NaN)) - var partialNans = agg.init() - onlyNaNs.foreach(tp => partialNans = agg.iterate(partialNans, tp)) - assert(agg.finalAgg(partialNans) == null) - - // --------------------------------------------------------- - // 2) Basic decimals + negatives: should find the true minimum - // --------------------------------------------------------- - val basics = Seq( - makeTuple(schema, 10.25), - makeTuple(schema, -2.5), - makeTuple(schema, 5.0), - makeTuple(schema, -2.5000000001), // slightly smaller than -2.5 - makeTuple(schema, 1.0e-12) + val agg = makeAggregationOp(AggregationFunction.MIN, "temperature", "min_temp") + .getAggFunc(AttributeType.DOUBLE) + + assert(agg.finalAgg(agg.init()) == null) + assert( + runSingleAttrAgg( + AggregationFunction.MIN, + "temperature", + AttributeType.DOUBLE, + Seq(null, null, null) + ) == null ) - - var partial = agg.init() - basics.foreach(tp => partial = agg.iterate(partial, tp)) - - val basicResult = agg.finalAgg(partial).asInstanceOf[Number].doubleValue() - assert(basicResult == -2.5000000001) - - // --------------------------------------------------------- - // 3) NaN + null interleaving must not poison the result - // (especially if NaN appears first) - // --------------------------------------------------------- - val mixed = Seq( - makeTuple(schema, Double.NaN), - makeTuple(schema, null), - makeTuple(schema, 3.14159), - makeTuple(schema, -0.125), - makeTuple(schema, Double.NaN), - makeTuple(schema, -9999.0), - makeTuple(schema, null) + assert( + runSingleAttrAgg( + AggregationFunction.MIN, + "temperature", + AttributeType.DOUBLE, + Seq(Double.NaN, Double.NaN) + ) == null ) - var partialMixed = agg.init() - mixed.foreach(tp => partialMixed = agg.iterate(partialMixed, tp)) + val basicResult = runSingleAttrAgg( + AggregationFunction.MIN, + "temperature", + AttributeType.DOUBLE, + Seq(10.25, -2.5, 5.0, -2.5000000001, 1.0e-12) + ).asInstanceOf[Number].doubleValue() + assert(basicResult == -2.5000000001) - val mixedResult = agg.finalAgg(partialMixed).asInstanceOf[Number].doubleValue() + val mixedResult = runSingleAttrAgg( + AggregationFunction.MIN, + "temperature", + AttributeType.DOUBLE, + Seq(Double.NaN, null, 3.14159, -0.125, Double.NaN, -9999.0, null) + ).asInstanceOf[Number].doubleValue() assert(mixedResult == -9999.0) - // --------------------------------------------------------- - // 4) Infinities: min should be -Infinity if present - // --------------------------------------------------------- - val infinities = Seq( - makeTuple(schema, Double.PositiveInfinity), - makeTuple(schema, 42.0), - makeTuple(schema, Double.NegativeInfinity), - makeTuple(schema, -1.0) - ) - - var partialInf = agg.init() - infinities.foreach(tp => partialInf = agg.iterate(partialInf, tp)) - - val infResult = agg.finalAgg(partialInf).asInstanceOf[Number].doubleValue() + val infResult = runSingleAttrAgg( + AggregationFunction.MIN, + "temperature", + AttributeType.DOUBLE, + Seq(Double.PositiveInfinity, 42.0, Double.NegativeInfinity, -1.0) + ).asInstanceOf[Number].doubleValue() assert(infResult.isNegInfinity) - // --------------------------------------------------------- - // 5) Signed zero: MIN(-0.0, +0.0) should be -0.0 - // --------------------------------------------------------- - val signedZero = Seq(makeTuple(schema, 0.0), makeTuple(schema, -0.0), makeTuple(schema, 0.0)) - var partialZero = agg.init() - signedZero.foreach(tp => partialZero = agg.iterate(partialZero, tp)) - - val zeroValue = agg.finalAgg(partialZero).asInstanceOf[Number].doubleValue() - val zeroBits = java.lang.Double.doubleToRawLongBits(zeroValue) - val negZeroBits = java.lang.Double.doubleToRawLongBits(-0.0) - assert(zeroBits == negZeroBits) - - // --------------------------------------------------------- - // 6) Stress test: many values, compare against a reference min - // Reference rule here: ignore null and NaN; return null if none left. - // --------------------------------------------------------- + val zeroValue = runSingleAttrAgg( + AggregationFunction.MIN, + "temperature", + AttributeType.DOUBLE, + Seq(0.0, -0.0, 0.0) + ).asInstanceOf[Number].doubleValue() + assert( + java.lang.Double.doubleToRawLongBits(zeroValue) == java.lang.Double.doubleToRawLongBits(-0.0) + ) + val rng = new scala.util.Random(1337) val values: Seq[java.lang.Double] = (1 to 10000).map { index => @@ -377,7 +239,6 @@ class AggregateOpSpec extends AnyFunSuite { case 3 => Double.NegativeInfinity case 4 => -0.0 case _ => - // wide-ish range with some tiny magnitudes too val sign = if (rng.nextBoolean()) 1.0 else -1.0 sign * (rng.nextDouble() * 1.0e6) / (if (rng.nextInt(20) == 0) 1.0e12 else 1.0) } @@ -404,12 +265,10 @@ class AggregateOpSpec extends AnyFunSuite { assert(gotAny == null) } else { val got = gotAny.asInstanceOf[Number].doubleValue() - // exact match: should be one of the seen inputs, no tolerance needed if ( expected == 0.0 && java.lang.Double.doubleToRawLongBits(expected) != java.lang.Double .doubleToRawLongBits(got) ) { - // If expected is -0.0, enforce it assert( java.lang.Double.doubleToRawLongBits(got) == java.lang.Double.doubleToRawLongBits( expected @@ -423,61 +282,45 @@ class AggregateOpSpec extends AnyFunSuite { test("MAX aggregation finds largest DOUBLE value") { val maxValue = 99.144 - val schema = makeSchema("debt" -> AttributeType.DOUBLE) - val tuple1 = makeTuple(schema, -100.123) - val tuple2 = makeTuple(schema, 50.12) - val tuple3 = makeTuple(schema, maxValue) - - val operation = makeAggregationOp(AggregationFunction.MAX, "debt", "nax_debt") - val agg = operation.getAggFunc(AttributeType.DOUBLE) - - var partial = agg.init() - partial = agg.iterate(partial, tuple1) - partial = agg.iterate(partial, tuple2) - partial = agg.iterate(partial, tuple3) - - val result = agg.finalAgg(partial).asInstanceOf[java.lang.Double].doubleValue() - assert(result == maxValue) + val result = runSingleAttrAgg( + AggregationFunction.MAX, + "debt", + AttributeType.DOUBLE, + Seq(-100.123, 50.12, maxValue) + ) + assert(result.asInstanceOf[java.lang.Double].doubleValue() == maxValue) } test("MAX aggregation finds largest INTEGER and returns null when given no values") { - val schema = makeSchema("temperature" -> AttributeType.INTEGER) - val tuple1 = makeTuple(schema, 10) - val tuple2 = makeTuple(schema, -2) - val tuple3 = makeTuple(schema, 5) - - val operation = makeAggregationOp(AggregationFunction.MAX, "temperature", "max_temp") - val agg = operation.getAggFunc(AttributeType.INTEGER) - - // Empty case: never iterate, just finalize init - val emptyPartial = agg.init() - val emptyResult = agg.finalAgg(emptyPartial) - assert(emptyResult == null) - - // Non-empty case - var partial = agg.init() - partial = agg.iterate(partial, tuple1) - partial = agg.iterate(partial, tuple2) - partial = agg.iterate(partial, tuple3) - - val result = agg.finalAgg(partial).asInstanceOf[Number].intValue() - assert(result == 10) + assert( + runSingleAttrAgg( + AggregationFunction.MAX, + "temperature", + AttributeType.INTEGER, + Seq.empty + ) == null + ) + val result = runSingleAttrAgg( + AggregationFunction.MAX, + "temperature", + AttributeType.INTEGER, + Seq(10, -2, 5) + ) + assert(result.asInstanceOf[Number].intValue() == 10) } test("MAX aggregation returns null when all values are null") { - val schema = makeSchema("temperature" -> AttributeType.INTEGER) - - val operation = makeAggregationOp(AggregationFunction.MAX, "temperature", "max_temp") - val agg = operation.getAggFunc(AttributeType.INTEGER) - - var partial = agg.init() - Seq.fill(3)(makeTuple(schema, null)).foreach(tp => partial = agg.iterate(partial, tp)) - assert(agg.finalAgg(partial) == null) + assert( + runSingleAttrAgg( + AggregationFunction.MAX, + "temperature", + AttributeType.INTEGER, + Seq.fill(3)(null) + ) == null + ) } test("MAX aggregation keeps the type's maximum value when it is the true maximum") { - // Regression: the finalizer used to mistake a partial equal to - // maxValue(attributeType) for the "no value seen" sentinel and emit null. val cases: Seq[(AttributeType, Seq[Any], Any)] = Seq( (AttributeType.INTEGER, Seq(1, 5, Int.MaxValue), Int.MaxValue), (AttributeType.LONG, Seq(1L, 5L, Long.MaxValue), Long.MaxValue), @@ -488,226 +331,123 @@ class AggregateOpSpec extends AnyFunSuite { new Timestamp(Long.MaxValue) ) ) - for ((attrType, values, expected) <- cases) { - val schema = makeSchema("v" -> attrType) - val operation = makeAggregationOp(AggregationFunction.MAX, "v", "max_v") - val agg = operation.getAggFunc(attrType) - - var partial = agg.init() - values.foreach(v => partial = agg.iterate(partial, makeTuple(schema, v))) - - assert(agg.finalAgg(partial) == expected, s"MAX over $attrType must keep $expected") + assert( + runSingleAttrAgg(AggregationFunction.MAX, "v", attrType, values) == expected, + s"MAX over $attrType must keep $expected" + ) } } test("AVERAGE aggregation ignores nulls and returns null when all values are null") { - val schema = makeSchema("price" -> AttributeType.DOUBLE) - val tuple1 = makeTuple(schema, 10.0) - val tuple2 = makeTuple(schema, null) - val tuple3 = makeTuple(schema, 20.0) - - val operation = makeAggregationOp(AggregationFunction.AVERAGE, "price", "avg_price") - val agg = operation.getAggFunc(AttributeType.DOUBLE) - - // Mixed null and non-null - var partial = agg.init() - partial = agg.iterate(partial, tuple1) - partial = agg.iterate(partial, tuple2) - partial = agg.iterate(partial, tuple3) - - val avg = agg.finalAgg(partial).asInstanceOf[java.lang.Double].doubleValue() - assert(math.abs(avg - 15.0) < 1e-6) - - // All nulls - val allNull = makeTuple(schema, null) - var partialAllNull = agg.init() - partialAllNull = agg.iterate(partialAllNull, allNull) - val allNullResult = agg.finalAgg(partialAllNull) - assert(allNullResult == null) + val avg = runSingleAttrAgg( + AggregationFunction.AVERAGE, + "price", + AttributeType.DOUBLE, + Seq(10.0, null, 20.0) + ) + assert(math.abs(avg.asInstanceOf[java.lang.Double].doubleValue() - 15.0) < 1e-6) + assert( + runSingleAttrAgg( + AggregationFunction.AVERAGE, + "price", + AttributeType.DOUBLE, + Seq(null) + ) == null + ) } - // --------------------------------------------------------------------------- - // getFinal behaviour - // --------------------------------------------------------------------------- - test("getFinal rewrites COUNT into SUM over the intermediate result attribute") { - val operation = makeAggregationOp(AggregationFunction.COUNT, "price", "price_count") - val finalOp = operation.getFinal - + val finalOp = makeAggregationOp(AggregationFunction.COUNT, "price", "price_count").getFinal assert(finalOp.aggFunction == AggregationFunction.SUM) - assert(finalOp.attribute == "price_count") - assert(finalOp.resultAttribute == "price_count") + assert(finalOp.attribute == "price_count" && finalOp.resultAttribute == "price_count") } test("getFinal keeps non-COUNT aggregation function and rewires attribute to resultAttribute") { - val operation = makeAggregationOp(AggregationFunction.SUM, "amount", "total_amount") - val finalOp = operation.getFinal - + val finalOp = makeAggregationOp(AggregationFunction.SUM, "amount", "total_amount").getFinal assert(finalOp.aggFunction == AggregationFunction.SUM) - assert(finalOp.attribute == "total_amount") - assert(finalOp.resultAttribute == "total_amount") + assert(finalOp.attribute == "total_amount" && finalOp.resultAttribute == "total_amount") } - // --------------------------------------------------------------------------- - // AggregateOpExec: integration-style tests with groupBy - // --------------------------------------------------------------------------- - test("AggregateOpExec groups by a single key and computes SUM per group") { - // schema: city (group key), sales - val schema = makeSchema( - "city" -> AttributeType.STRING, - "sales" -> AttributeType.INTEGER + val schema = makeSchema("city" -> AttributeType.STRING, "sales" -> AttributeType.INTEGER) + val exec = openAggregateExec( + List(makeAggregationOp(AggregationFunction.SUM, "sales", "total_sales")), + List("city") + ) + val results = runAggregateExec( + exec, + makeTuple(schema, "NY", 10), + makeTuple(schema, "SF", 20), + makeTuple(schema, "NY", 5) ) - - val tuple1 = makeTuple(schema, "NY", 10) - val tuple2 = makeTuple(schema, "SF", 20) - val tuple3 = makeTuple(schema, "NY", 5) - - val desc = new AggregateOpDesc() - val sumAgg = makeAggregationOp(AggregationFunction.SUM, "sales", "total_sales") - desc.aggregations = List(sumAgg) - desc.groupByKeys = List("city") - - val descJson = objectMapper.writeValueAsString(desc) - - val exec = new AggregateOpExec(descJson) - exec.open() - exec.processTuple(tuple1, 0) - exec.processTuple(tuple2, 0) - exec.processTuple(tuple3, 0) - - val results = exec.onFinish(0).toList - - // Expect two output rows: (NY, 15) and (SF, 20) val resultMap = results.map { tupleLike => val fields = tupleLike.getFields - val city = fields(0).asInstanceOf[String] - val total = fields(1).asInstanceOf[Number].intValue() - city -> total + fields(0).asInstanceOf[String] -> fields(1).asInstanceOf[Number].intValue() }.toMap - - assert(resultMap.size == 2) - assert(resultMap("NY") == 15) - assert(resultMap("SF") == 20) + assert(resultMap == Map("NY" -> 15, "SF" -> 20)) } test("AggregateOpExec performs global SUM and COUNT when there are no groupBy keys") { - // schema: region (ignored for aggregation), revenue - val schema = makeSchema( - "region" -> AttributeType.STRING, - "revenue" -> AttributeType.INTEGER + val schema = makeSchema("region" -> AttributeType.STRING, "revenue" -> AttributeType.INTEGER) + val exec = openAggregateExec( + List( + makeAggregationOp(AggregationFunction.SUM, "revenue", "total_revenue"), + makeAggregationOp(AggregationFunction.COUNT, "revenue", "row_count") + ), + List() + ) + val results = runAggregateExec( + exec, + makeTuple(schema, "west", 100), + makeTuple(schema, "east", 200), + makeTuple(schema, "west", 50) ) - - val tuple1 = makeTuple(schema, "west", 100) - val tuple2 = makeTuple(schema, "east", 200) - val tuple3 = makeTuple(schema, "west", 50) - - val desc = new AggregateOpDesc() - val sumAgg = makeAggregationOp(AggregationFunction.SUM, "revenue", "total_revenue") - val countAgg = makeAggregationOp(AggregationFunction.COUNT, "revenue", "row_count") - desc.aggregations = List(sumAgg, countAgg) - desc.groupByKeys = List() // global aggregation - - val descJson = objectMapper.writeValueAsString(desc) - - val exec = new AggregateOpExec(descJson) - exec.open() - exec.processTuple(tuple1, 0) - exec.processTuple(tuple2, 0) - exec.processTuple(tuple3, 0) - - val results = exec.onFinish(0).toList assert(results.size == 1) - val fields = results.head.getFields - // No group keys, so fields(0) is SUM(revenue), fields(1) is COUNT(revenue) - val totalRevenue = fields(0).asInstanceOf[Number].intValue() - val rowCount = fields(1).asInstanceOf[Number].intValue() - - assert(totalRevenue == 350) - assert(rowCount == 3) + assert(fields(0).asInstanceOf[Number].intValue() == 350) + assert(fields(1).asInstanceOf[Number].intValue() == 3) } test("AggregateOpExec computes COUNT(*) over every row (including nulls) end-to-end") { - // region (ignored), revenue (one null). An empty attribute means COUNT(*), so the - // executor must skip the input-column lookup and still count all 3 rows. - val schema = makeSchema( - "region" -> AttributeType.STRING, - "revenue" -> AttributeType.INTEGER + val schema = makeSchema("region" -> AttributeType.STRING, "revenue" -> AttributeType.INTEGER) + val exec = openAggregateExec( + List(makeAggregationOp(AggregationFunction.COUNT, "", "row_count")), + List() ) - - val tuple1 = makeTuple(schema, "west", 100) - val tuple2 = makeTuple(schema, "east", null) - val tuple3 = makeTuple(schema, "west", 50) - - val desc = new AggregateOpDesc() - desc.aggregations = List(makeAggregationOp(AggregationFunction.COUNT, "", "row_count")) - desc.groupByKeys = List() // global aggregation - - val exec = new AggregateOpExec(objectMapper.writeValueAsString(desc)) - exec.open() - exec.processTuple(tuple1, 0) - exec.processTuple(tuple2, 0) - exec.processTuple(tuple3, 0) - - val results = exec.onFinish(0).toList - assert(results.size == 1) - assert(results.head.getFields(0).asInstanceOf[Number].intValue() == 3) + val results = runAggregateExec( + exec, + makeTuple(schema, "west", 100), + makeTuple(schema, "east", null), + makeTuple(schema, "west", 50) + ) + assert(results.size == 1 && results.head.getFields(0).asInstanceOf[Number].intValue() == 3) } - // `close()` also resets `distributedAggregations`, which is deliberately not - // asserted: `onFinish` reads that list once per accumulated group and `close()` - // leaves none, while a reopened executor resets it again — so the reset is only - // observable by calling `processTuple` after `close()` without an intervening - // `open()`, which the executor lifecycle never does. Pinning it would cement an - // illegal call order. test("AggregateOpExec close() discards the accumulated groups") { - val schema = makeSchema( - "city" -> AttributeType.STRING, - "sales" -> AttributeType.INTEGER + val schema = makeSchema("city" -> AttributeType.STRING, "sales" -> AttributeType.INTEGER) + val exec = openAggregateExec( + List(makeAggregationOp(AggregationFunction.SUM, "sales", "total_sales")), + List("city") + ) + assert( + runAggregateExec(exec, makeTuple(schema, "NY", 10), makeTuple(schema, "SF", 20)).size == 2 ) - - val desc = new AggregateOpDesc() - desc.aggregations = List(makeAggregationOp(AggregationFunction.SUM, "sales", "total_sales")) - desc.groupByKeys = List("city") - - val exec = new AggregateOpExec(objectMapper.writeValueAsString(desc)) - exec.open() - exec.processTuple(makeTuple(schema, "NY", 10), 0) - exec.processTuple(makeTuple(schema, "SF", 20), 0) - - // Guard the fixture: the two groups really are accumulated, so the empty - // result asserted after close() cannot pass vacuously. - assert(exec.onFinish(0).toList.size == 2) - exec.close() assert(exec.onFinish(0).toList.isEmpty) } test("AggregateOpExec open() after close() accumulates from an empty state") { - // The executor object outlives a single run, so a reopened executor must not - // fold the previous run's partial aggregates into the new result. - val schema = makeSchema( - "city" -> AttributeType.STRING, - "sales" -> AttributeType.INTEGER + val schema = makeSchema("city" -> AttributeType.STRING, "sales" -> AttributeType.INTEGER) + val exec = openAggregateExec( + List(makeAggregationOp(AggregationFunction.SUM, "sales", "total_sales")), + List("city") ) - - val desc = new AggregateOpDesc() - desc.aggregations = List(makeAggregationOp(AggregationFunction.SUM, "sales", "total_sales")) - desc.groupByKeys = List("city") - - val exec = new AggregateOpExec(objectMapper.writeValueAsString(desc)) - exec.open() exec.processTuple(makeTuple(schema, "NY", 10), 0) exec.close() exec.open() - exec.processTuple(makeTuple(schema, "NY", 3), 0) - - val results = exec.onFinish(0).toList - assert(results.size == 1) - // 3, not 13: the first run's partial sum for "NY" must have been dropped. - assert(results.head.getFields(1).asInstanceOf[Number].intValue() == 3) + val results = runAggregateExec(exec, makeTuple(schema, "NY", 3)) + assert(results.size == 1 && results.head.getFields(1).asInstanceOf[Number].intValue() == 3) } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregationOperationSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregationOperationSpec.scala index 444d8247c8e..7fbba202e3b 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregationOperationSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/aggregate/AggregationOperationSpec.scala @@ -20,54 +20,116 @@ package org.apache.texera.amber.operator.aggregate import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema, Tuple} +import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.scalatest.Assertions import org.scalatest.flatspec.AnyFlatSpec import java.sql.Timestamp +private[aggregate] object AggregationTestSupport extends Assertions { + def makeAggregationOp( + fn: AggregationFunction, + attributeName: String, + resultName: String + ): AggregationOperation = { + val operation = new AggregationOperation() + operation.aggFunction = fn + operation.attribute = attributeName + operation.resultAttribute = resultName + operation + } + + def makeSchema(fields: (String, AttributeType)*): Schema = + Schema(fields.map { case (n, t) => new Attribute(n, t) }.toList) + + def makeTuple(schema: Schema, values: Any*): Tuple = + Tuple(schema, values.toArray) + + def singleTuple(name: String, t: AttributeType, value: AnyRef): Tuple = + Tuple.builder(makeSchema(name -> t)).add(new Attribute(name, t), value).build() + + def partialOf( + agg: DistributedAggregation[Object], + attr: String, + t: AttributeType, + values: Seq[Any] + ): Object = { + val schema = makeSchema(Option(attr).filter(_.nonEmpty).getOrElse("points") -> t) + values.map(v => makeTuple(schema, v)).foldLeft(agg.init())(agg.iterate) + } + + def runSingleAttrAgg( + fn: AggregationFunction, + attr: String, + t: AttributeType, + values: Seq[Any] + ): Object = { + val agg = makeAggregationOp(fn, attr, "result").getAggFunc(t) + agg.finalAgg(partialOf(agg, attr, t, values)) + } + + def openAggregateExec( + aggregations: List[AggregationOperation], + groupByKeys: List[String] + ): AggregateOpExec = { + val desc = new AggregateOpDesc() + desc.aggregations = aggregations + desc.groupByKeys = groupByKeys + val exec = new AggregateOpExec(objectMapper.writeValueAsString(desc)) + exec.open() + exec + } + + def runAggregateExec( + exec: AggregateOpExec, + tuples: Tuple* + ): List[org.apache.texera.amber.core.tuple.TupleLike] = { + tuples.foreach(exec.processTuple(_, 0)) + exec.onFinish(0).toList + } + + def assertAggregationAttribute( + fn: AggregationFunction, + attribute: String, + resultAttribute: String, + inputType: AttributeType, + expectedType: AttributeType + ): Unit = { + val attr = makeAggregationOp(fn, attribute, resultAttribute).getAggregationAttribute(inputType) + assert(attr.getName == resultAttribute) + assert(attr.getType == expectedType) + } + + def assertNeutralMergeWithEmptyPartial( + fn: AggregationFunction, + sample: AnyRef, + t: AttributeType = AttributeType.INTEGER + ): Unit = { + val agg = makeAggregationOp(fn, "v", "r").getAggFunc(t) + val seen = agg.iterate(agg.init(), singleTuple("v", t, sample)) + assert(agg.finalAgg(agg.merge(seen, agg.init())) == sample) + assert(agg.finalAgg(agg.merge(agg.init(), seen)) == sample) + assert(agg.finalAgg(agg.merge(agg.init(), agg.init())) == null) + } +} + /** * Coverage notes: * `AggregateOpSpec` (in this same package) already exercises the happy paths for - * `getAggregationAttribute`, the per-kind `init` / `iterate` / `finalAgg` semantics - * (SUM/COUNT/AVERAGE/MIN/MAX/CONCAT, including null-handling and AVERAGE-of-empty), - * and the `getFinal` rewrite. This spec deliberately does NOT duplicate those. - * - * What this spec adds: - * - `getAggFunc` validation errors (unsupported attribute types on SUM/MIN/MAX, - * null aggFunction). Note: SUM/MIN/MAX do accept TIMESTAMP alongside numerics. - * - The CONCAT-specific `merge` partial-combination behavior — `AggregateOpSpec` - * exercises iterate/finalAgg but never calls `merge` directly. - * - A two-stage worker→final pipeline that runs a real partial aggregation - * on each "worker", emits a partial tuple, then applies `getFinal` and - * re-aggregates the partials end-to-end. - * - `AveragePartialObj` (a plain `case class`, not a value class) field - * exposure and case-class value equality / hashCode. + * `getAggregationAttribute`, the per-kind `init` / `iterate` / `finalAgg` semantics, + * and the `getFinal` rewrite. This spec exercises validation errors, merge semantics, + * multi-stage `getFinal` worker->final pipelines, and timestamp averaging. */ class AggregationOperationSpec extends AnyFlatSpec { - - // --- helpers --------------------------------------------------------------- - - private def schemaWith(name: String, t: AttributeType): Schema = - new Schema(new Attribute(name, t)) - - private def tupleOf(name: String, t: AttributeType, value: AnyRef): Tuple = - Tuple.builder(schemaWith(name, t)).add(new Attribute(name, t), value).build() + import AggregationTestSupport._ private def op( func: AggregationFunction, attribute: String = "v", resultAttribute: String = "r" - ): AggregationOperation = { - val o = new AggregationOperation() - o.aggFunction = func - o.attribute = attribute - o.resultAttribute = resultAttribute - o - } - - // --- getAggFunc: type validation (not covered in AggregateOpSpec) ---------- + ): AggregationOperation = makeAggregationOp(func, attribute, resultAttribute) "AggregationOperation.getAggFunc" should "throw UnsupportedOperationException for unsupported attribute types on SUM" in { - // SUM accepts INTEGER/LONG/DOUBLE/TIMESTAMP; STRING is rejected. val ex = intercept[UnsupportedOperationException] { op(AggregationFunction.SUM).getAggFunc(AttributeType.STRING) } @@ -75,7 +137,6 @@ class AggregationOperationSpec extends AnyFlatSpec { } it should "throw UnsupportedOperationException for unsupported attribute types on MIN and MAX" in { - // MIN/MAX accept INTEGER/LONG/DOUBLE/TIMESTAMP; STRING and BOOLEAN are rejected. intercept[UnsupportedOperationException] { op(AggregationFunction.MIN).getAggFunc(AttributeType.STRING) } @@ -91,8 +152,6 @@ class AggregationOperationSpec extends AnyFlatSpec { assert(ex.getMessage.contains("Unknown aggregation function")) } - // --- CONCAT partial merge (iterate is covered in AggregateOpSpec) ---------- - "CONCAT aggregation merge" should "join two non-empty partials with a comma and short-circuit when either is empty" in { val agg = op(AggregationFunction.CONCAT).getAggFunc(AttributeType.STRING) @@ -102,125 +161,94 @@ class AggregationOperationSpec extends AnyFlatSpec { assert(agg.merge("", "") == "") } - // --- partial + final pipeline ---------------------------------------------- - "Worker → final aggregation pipeline" should "give the same total as a single-pass COUNT when partials are re-aggregated via getFinal" in { - // Two "workers" each run a COUNT over their slice of the data. Each - // worker emits a partial output (an Integer count). The "final" stage - // re-aggregates those partial outputs as a SUM over the result column, - // which getFinal is supposed to produce. val workerOp = op(AggregationFunction.COUNT, attribute = "v", resultAttribute = "row_count") - val workerAgg = workerOp.getAggFunc(AttributeType.INTEGER) - - val w1Tuples = Seq( - tupleOf("v", AttributeType.INTEGER, Int.box(10)), - tupleOf("v", AttributeType.INTEGER, null), - tupleOf("v", AttributeType.INTEGER, Int.box(20)) - ) - val w1State = w1Tuples.foldLeft(workerAgg.init())(workerAgg.iterate) - val w1Out = workerAgg.finalAgg(w1State).asInstanceOf[Integer] + val w1Out = runSingleAttrAgg( + AggregationFunction.COUNT, + "v", + AttributeType.INTEGER, + Seq(Int.box(10), null, Int.box(20)) + ).asInstanceOf[Integer] assert(w1Out == 2, "worker 1 saw two non-null values") - val w2Tuples = Seq( - tupleOf("v", AttributeType.INTEGER, Int.box(30)), - tupleOf("v", AttributeType.INTEGER, Int.box(40)), - tupleOf("v", AttributeType.INTEGER, Int.box(50)) - ) - val w2State = w2Tuples.foldLeft(workerAgg.init())(workerAgg.iterate) - val w2Out = workerAgg.finalAgg(w2State).asInstanceOf[Integer] + val w2Out = runSingleAttrAgg( + AggregationFunction.COUNT, + "v", + AttributeType.INTEGER, + Seq(Int.box(30), Int.box(40), Int.box(50)) + ).asInstanceOf[Integer] assert(w2Out == 3) - // Final stage: re-aggregate the partial counts via getFinal. val finalOp = workerOp.getFinal assert(finalOp.aggFunction == AggregationFunction.SUM) assert(finalOp.attribute == "row_count") val finalAgg = finalOp.getAggFunc(AttributeType.INTEGER) - val partial1 = tupleOf("row_count", AttributeType.INTEGER, w1Out) - val partial2 = tupleOf("row_count", AttributeType.INTEGER, w2Out) - val finalState = - finalAgg.iterate(finalAgg.iterate(finalAgg.init(), partial1), partial2) - val finalCount = finalAgg.finalAgg(finalState).asInstanceOf[Integer] - assert(finalCount == 5, "summing partial counts must match a single-pass COUNT") + assert( + finalAgg.finalAgg( + partialOf(finalAgg, "row_count", AttributeType.INTEGER, Seq(w1Out, w2Out)) + ) == 5 + ) } it should "give the same total as a single-pass SUM when partials are re-aggregated via getFinal" in { - // For SUM, getFinal keeps aggFunction = SUM and rebinds attribute to the - // result column. The pipeline must produce the same total as a single-pass - // SUM over all the input tuples. val workerOp = op(AggregationFunction.SUM, attribute = "v", resultAttribute = "total") val workerAgg = workerOp.getAggFunc(AttributeType.INTEGER) - - val groups = Seq( + val partials = Seq( Seq(Int.box(1), Int.box(2), Int.box(3)), Seq(Int.box(10), Int.box(20)) + ).map(vs => + workerAgg.finalAgg(partialOf(workerAgg, "v", AttributeType.INTEGER, vs)).asInstanceOf[Integer] ) - val partials: Seq[Integer] = groups.map { values => - val state = values - .map(v => tupleOf("v", AttributeType.INTEGER, v)) - .foldLeft(workerAgg.init())(workerAgg.iterate) - workerAgg.finalAgg(state).asInstanceOf[Integer] - } assert(partials == Seq(6: Integer, 30: Integer)) val finalOp = workerOp.getFinal assert(finalOp.aggFunction == AggregationFunction.SUM) assert(finalOp.attribute == "total") val finalAgg = finalOp.getAggFunc(AttributeType.INTEGER) - val finalState = partials - .map(p => tupleOf("total", AttributeType.INTEGER, p)) - .foldLeft(finalAgg.init())(finalAgg.iterate) - val finalSum = finalAgg.finalAgg(finalState).asInstanceOf[Integer] - assert(finalSum == 36, "single-pass SUM(1+2+3+10+20) == 36") + assert( + finalAgg.finalAgg(partialOf(finalAgg, "total", AttributeType.INTEGER, partials)) == 36, + "single-pass SUM(1+2+3+10+20) == 36" + ) } it should "keep the type's maximum value when partial MAX results are re-aggregated via getFinal" in { - // Regression: the local stage used to finalize a partial equal to - // maxValue(attributeType) to null, so the final stage reported the largest - // value from the other worker instead of the true maximum. val workerOp = op(AggregationFunction.MAX, attribute = "v", resultAttribute = "max_v") - val workerAgg = workerOp.getAggFunc(AttributeType.INTEGER) - - val w1State = Seq[AnyRef](Int.box(1), Int.box(Int.MaxValue)) - .map(v => tupleOf("v", AttributeType.INTEGER, v)) - .foldLeft(workerAgg.init())(workerAgg.iterate) - val w1Out = workerAgg.finalAgg(w1State) + val w1Out = runSingleAttrAgg( + AggregationFunction.MAX, + "v", + AttributeType.INTEGER, + Seq(Int.box(1), Int.box(Int.MaxValue)) + ) assert(w1Out == Int.box(Int.MaxValue), "worker 1's true maximum is Int.MaxValue") - val w2State = Seq[AnyRef](Int.box(5), Int.box(2)) - .map(v => tupleOf("v", AttributeType.INTEGER, v)) - .foldLeft(workerAgg.init())(workerAgg.iterate) - val w2Out = workerAgg.finalAgg(w2State) + val w2Out = runSingleAttrAgg( + AggregationFunction.MAX, + "v", + AttributeType.INTEGER, + Seq(Int.box(5), Int.box(2)) + ) assert(w2Out == Int.box(5)) val finalOp = workerOp.getFinal assert(finalOp.aggFunction == AggregationFunction.MAX) val finalAgg = finalOp.getAggFunc(AttributeType.INTEGER) - val finalState = Seq(w1Out, w2Out) - .map(p => tupleOf("max_v", AttributeType.INTEGER, p)) - .foldLeft(finalAgg.init())(finalAgg.iterate) - assert(finalAgg.finalAgg(finalState) == Int.box(Int.MaxValue)) + assert( + finalAgg.finalAgg( + partialOf(finalAgg, "max_v", AttributeType.INTEGER, Seq(w1Out, w2Out)) + ) == Int + .box(Int.MaxValue) + ) } - // --- AveragePartialObj ----------------------------------------------------- - "AveragePartialObj" should "expose its sum and count fields and support value equality" in { val a = AveragePartialObj(10.0, 4) val b = AveragePartialObj(10.0, 4) - assert(a.sum == 10.0) - assert(a.count == 4) - assert(a == b) - assert(a.hashCode == b.hashCode) + assert(a.sum == 10.0 && a.count == 4 && a == b && a.hashCode == b.hashCode) } - // --- getAggFunc: the accepting side of the four-clause type guard ---------- - - // SUM/MIN/MAX each guard on - // `!= INTEGER && != DOUBLE && != LONG && != TIMESTAMP`. The existing tests - // only drive the rejecting side plus INTEGER/DOUBLE, so the chain is never - // walked to its later clauses; TIMESTAMP in particular has to pass all four. private val supportedAggTypes = Seq( AttributeType.INTEGER, AttributeType.DOUBLE, @@ -240,94 +268,78 @@ class AggregationOperationSpec extends AnyFlatSpec { } it should "reject unsupported attribute types on SUM, MIN and MAX, naming the aggregation and the type" in { - val unsupported = - Seq(AttributeType.STRING, AttributeType.BOOLEAN, AttributeType.BINARY) + val unsupported = Seq(AttributeType.STRING, AttributeType.BOOLEAN, AttributeType.BINARY) for ((func, name) <- guardedAggregations; t <- unsupported) { val ex = intercept[UnsupportedOperationException](op(func).getAggFunc(t)) assert(ex.getMessage == s"Unsupported attribute type for $name aggregation: $t") } } - // --- AVERAGE over TIMESTAMP: the timestamp branch of getNumericalValue ----- - - // Everywhere else AVERAGE is driven over DOUBLE, which takes the - // `value.toString.toDouble` path. A TIMESTAMP column is the only route into - // the `parseTimestamp(...).getTime` branch. private val earlier = Timestamp.valueOf("2020-03-05 10:00:00") private val later = Timestamp.valueOf("2020-03-05 11:00:00") private val midpointMillis = (earlier.getTime + later.getTime) / 2.0 "AVERAGE over a TIMESTAMP column" should "average the values' epoch milliseconds" in { - val agg = op(AggregationFunction.AVERAGE).getAggFunc(AttributeType.TIMESTAMP) - val state = Seq(earlier, later) - .map(ts => tupleOf("v", AttributeType.TIMESTAMP, ts)) - .foldLeft(agg.init())(agg.iterate) - - assert(agg.finalAgg(state).asInstanceOf[java.lang.Double] == midpointMillis) + assert( + runSingleAttrAgg( + AggregationFunction.AVERAGE, + "v", + AttributeType.TIMESTAMP, + Seq(earlier, later) + ).asInstanceOf[java.lang.Double] == midpointMillis + ) } it should "combine per-worker partials through merge" in { val agg = op(AggregationFunction.AVERAGE).getAggFunc(AttributeType.TIMESTAMP) - val p1 = agg.iterate(agg.init(), tupleOf("v", AttributeType.TIMESTAMP, earlier)) - val p2 = agg.iterate(agg.init(), tupleOf("v", AttributeType.TIMESTAMP, later)) - - val merged = agg.merge(p1, p2) - - assert(agg.finalAgg(merged).asInstanceOf[java.lang.Double] == midpointMillis) + val p1 = partialOf(agg, "v", AttributeType.TIMESTAMP, Seq(earlier)) + val p2 = partialOf(agg, "v", AttributeType.TIMESTAMP, Seq(later)) + assert(agg.finalAgg(agg.merge(p1, p2)).asInstanceOf[java.lang.Double] == midpointMillis) } it should "ignore null timestamps and return null when every value is null" in { - val agg = op(AggregationFunction.AVERAGE).getAggFunc(AttributeType.TIMESTAMP) - - val mixed = Seq[AnyRef](earlier, null, later) - .map(ts => tupleOf("v", AttributeType.TIMESTAMP, ts)) - .foldLeft(agg.init())(agg.iterate) - assert(agg.finalAgg(mixed).asInstanceOf[java.lang.Double] == midpointMillis) - - val allNull = - agg.iterate(agg.init(), tupleOf("v", AttributeType.TIMESTAMP, null)) - assert(agg.finalAgg(allNull) == null) + assert( + runSingleAttrAgg( + AggregationFunction.AVERAGE, + "v", + AttributeType.TIMESTAMP, + Seq(earlier, null, later) + ).asInstanceOf[java.lang.Double] == midpointMillis + ) + assert( + runSingleAttrAgg( + AggregationFunction.AVERAGE, + "v", + AttributeType.TIMESTAMP, + Seq(null) + ) == null + ) } - // --- merge: the partial-combination lambda of each aggregation -------------- - - // `AggregateOpSpec` drives init/iterate/finalAgg for every aggregation but only - // ever merges AVERAGE/CONCAT partials. The `merge` lambdas of SUM, COUNT, MIN - // and MAX are what the global stage calls when several workers report in, so - // each one is exercised here directly and cross-checked against the equivalent - // single-pass aggregation. - - private def aggregateAll( - agg: DistributedAggregation[Object], - t: AttributeType, - values: Seq[AnyRef] - ): Object = - agg.finalAgg(values.map(v => tupleOf("v", t, v)).foldLeft(agg.init())(agg.iterate)) - "SUM aggregation merge" should "add two partials, matching a single-pass SUM" in { val agg = op(AggregationFunction.SUM).getAggFunc(AttributeType.LONG) val left = Seq[AnyRef](Long.box(1L), Long.box(2L)) val right = Seq[AnyRef](Long.box(10L), Long.box(20L)) - - val p1 = left.map(v => tupleOf("v", AttributeType.LONG, v)).foldLeft(agg.init())(agg.iterate) - val p2 = right.map(v => tupleOf("v", AttributeType.LONG, v)).foldLeft(agg.init())(agg.iterate) + val p1 = partialOf(agg, "v", AttributeType.LONG, left) + val p2 = partialOf(agg, "v", AttributeType.LONG, right) assert(agg.finalAgg(agg.merge(p1, p2)) == Long.box(33L)) - assert(agg.finalAgg(agg.merge(p1, p2)) == aggregateAll(agg, AttributeType.LONG, left ++ right)) - // merging with an untouched (zero) partial must be a no-op + assert( + agg.finalAgg(agg.merge(p1, p2)) == runSingleAttrAgg( + AggregationFunction.SUM, + "v", + AttributeType.LONG, + left ++ right + ) + ) assert(agg.finalAgg(agg.merge(p1, agg.init())) == Long.box(3L)) } "COUNT aggregation merge" should "add the per-worker counts" in { val agg = op(AggregationFunction.COUNT).getAggFunc(AttributeType.INTEGER) - val p1 = Seq[AnyRef](Int.box(1), null, Int.box(3)) - .map(v => tupleOf("v", AttributeType.INTEGER, v)) - .foldLeft(agg.init())(agg.iterate) - val p2 = Seq[AnyRef](Int.box(4)) - .map(v => tupleOf("v", AttributeType.INTEGER, v)) - .foldLeft(agg.init())(agg.iterate) - - // COUNT(v) skips the null, so 2 + 1 == 3 + val p1 = partialOf(agg, "v", AttributeType.INTEGER, Seq(Int.box(1), null, Int.box(3))) + val p2 = partialOf(agg, "v", AttributeType.INTEGER, Seq(Int.box(4))) + assert(agg.finalAgg(agg.merge(p1, p2)) == Int.box(3)) assert(agg.finalAgg(agg.merge(agg.init(), agg.init())) == Int.box(0)) } @@ -338,85 +350,53 @@ class AggregationOperationSpec extends AnyFlatSpec { val left = Seq[AnyRef](Double.box(4.0), Double.box(9.0)) val right = Seq[AnyRef](Double.box(-1.5), Double.box(2.0)) - def partialOf(agg: DistributedAggregation[Object], values: Seq[AnyRef]): Object = - values.map(v => tupleOf("v", AttributeType.DOUBLE, v)).foldLeft(agg.init())(agg.iterate) + val minL = partialOf(minAgg, "v", AttributeType.DOUBLE, left) + val minR = partialOf(minAgg, "v", AttributeType.DOUBLE, right) + assert(minAgg.finalAgg(minAgg.merge(minL, minR)) == Double.box(-1.5)) + assert(minAgg.finalAgg(minAgg.merge(minR, minL)) == Double.box(-1.5)) - assert( - minAgg.finalAgg(minAgg.merge(partialOf(minAgg, left), partialOf(minAgg, right))) - == Double.box(-1.5) - ) - // merge must be symmetric - assert( - minAgg.finalAgg(minAgg.merge(partialOf(minAgg, right), partialOf(minAgg, left))) - == Double.box(-1.5) - ) - assert( - maxAgg.finalAgg(maxAgg.merge(partialOf(maxAgg, left), partialOf(maxAgg, right))) - == Double.box(9.0) - ) - assert( - maxAgg.finalAgg(maxAgg.merge(partialOf(maxAgg, right), partialOf(maxAgg, left))) - == Double.box(9.0) - ) + val maxL = partialOf(maxAgg, "v", AttributeType.DOUBLE, left) + val maxR = partialOf(maxAgg, "v", AttributeType.DOUBLE, right) + assert(maxAgg.finalAgg(maxAgg.merge(maxL, maxR)) == Double.box(9.0)) + assert(maxAgg.finalAgg(maxAgg.merge(maxR, maxL)) == Double.box(9.0)) } "MIN aggregation merge" should "stay neutral when one side saw no values" in { - val agg = op(AggregationFunction.MIN).getAggFunc(AttributeType.INTEGER) - val seen = agg.iterate(agg.init(), tupleOf("v", AttributeType.INTEGER, Int.box(7))) - - // An empty partial is the sentinel maxValue, so it must lose the comparison. - assert(agg.finalAgg(agg.merge(seen, agg.init())) == Int.box(7)) - assert(agg.finalAgg(agg.merge(agg.init(), seen)) == Int.box(7)) - // Two empty partials still finalize to null (no rows anywhere). - assert(agg.finalAgg(agg.merge(agg.init(), agg.init())) == null) + assertNeutralMergeWithEmptyPartial(AggregationFunction.MIN, Int.box(7)) } "MAX aggregation merge" should "stay neutral when one side saw no values" in { - val agg = op(AggregationFunction.MAX).getAggFunc(AttributeType.INTEGER) - val seen = agg.iterate(agg.init(), tupleOf("v", AttributeType.INTEGER, Int.box(7))) - - // An empty partial is the sentinel minValue, so it must lose the comparison. - assert(agg.finalAgg(agg.merge(seen, agg.init())) == Int.box(7)) - assert(agg.finalAgg(agg.merge(agg.init(), seen)) == Int.box(7)) - // Two empty partials still finalize to null (no rows anywhere). - assert(agg.finalAgg(agg.merge(agg.init(), agg.init())) == null) + assertNeutralMergeWithEmptyPartial(AggregationFunction.MAX, Int.box(7)) } - // --- CONCAT: a null first value seeds the partial with an empty string ------ - "CONCAT aggregation" should "swallow leading nulls but keep interior ones as empty slots" in { - // AggregateOpSpec only ever concatenates a null in the middle of the stream. - // A null on the very first tuple takes the `partial == ""` side of the branch - // and leaves the partial empty, so — unlike an interior null — it does not - // occupy a slot in the comma-joined output. - val agg = op(AggregationFunction.CONCAT).getAggFunc(AttributeType.STRING) - val result = Seq[AnyRef](null, "red", null, "blue") - .map(v => tupleOf("v", AttributeType.STRING, v)) - .foldLeft(agg.init())(agg.iterate) - - assert(agg.finalAgg(result) == "red,,blue") + assert( + runSingleAttrAgg( + AggregationFunction.CONCAT, + "v", + AttributeType.STRING, + Seq(null, "red", null, "blue") + ) == "red,,blue" + ) } it should "return an empty string when every value is null" in { - val agg = op(AggregationFunction.CONCAT).getAggFunc(AttributeType.STRING) - val onlyNull = agg.iterate(agg.init(), tupleOf("v", AttributeType.STRING, null)) - - assert(agg.finalAgg(onlyNull) == "") + assert( + runSingleAttrAgg(AggregationFunction.CONCAT, "v", AttributeType.STRING, Seq(null)) == "" + ) } it should "stringify non-string values it is pointed at" in { - // CONCAT is schema-restricted to STRING columns in the UI, but the executor - // only ever calls `.toString`, so an INTEGER column still concatenates. - val agg = op(AggregationFunction.CONCAT).getAggFunc(AttributeType.STRING) - val result = Seq[AnyRef](Int.box(1), Int.box(2)) - .map(v => tupleOf("v", AttributeType.INTEGER, v)) - .foldLeft(agg.init())(agg.iterate) - - assert(agg.finalAgg(result) == "1,2") + assert( + runSingleAttrAgg( + AggregationFunction.CONCAT, + "v", + AttributeType.INTEGER, + Seq(Int.box(1), Int.box(2)) + ) == "1,2" + ) } - // --- getAggregationAttribute / getFinal: message and identity guarantees ---- - "getAggregationAttribute" should "name the unknown aggregation function in its error" in { val ex = intercept[RuntimeException](op(null).getAggregationAttribute(AttributeType.INTEGER)) assert(ex.getMessage == "Unknown aggregation function: null") @@ -428,10 +408,7 @@ class AggregationOperationSpec extends AnyFlatSpec { assert(copy ne original) assert(copy.aggFunction == AggregationFunction.MAX) - // the final stage reads and writes the same (result) column - assert(copy.attribute == "dst") - assert(copy.resultAttribute == "dst") - // the original is left untouched + assert(copy.attribute == "dst" && copy.resultAttribute == "dst") assert(original.attribute == "src") } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/dictionary/DictionaryMatcherOpExecSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/dictionary/DictionaryMatcherOpExecSpec.scala index 9eb2cdf627a..b6ab3aa4cbe 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/dictionary/DictionaryMatcherOpExecSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/dictionary/DictionaryMatcherOpExecSpec.scala @@ -35,10 +35,7 @@ class DictionaryMatcherOpExecSpec extends AnyFlatSpec with BeforeAndAfter { .builder(tupleSchema) .add(new Attribute("field1", AttributeType.STRING), "nice a a person") .add(new Attribute("field2", AttributeType.INTEGER), 1) - .add( - new Attribute("field3", AttributeType.BOOLEAN), - true - ) + .add(new Attribute("field3", AttributeType.BOOLEAN), true) .build() var opExec: DictionaryMatcherOpExec = _ @@ -56,6 +53,44 @@ class DictionaryMatcherOpExecSpec extends AnyFlatSpec with BeforeAndAfter { outputSchema = opDesc.getExternalOutputSchemas(Map(PortIdentity() -> tupleSchema)).values.head } + private def withExecutor( + matchingType: MatchingType, + dictionary: String = dictionaryScan + )(body: => Unit): Unit = { + opDesc.dictionary = dictionary + opDesc.matchingType = matchingType + opExec = new DictionaryMatcherOpExec(objectMapper.writeValueAsString(opDesc)) + opExec.open() + try body + finally opExec.close() + } + + private def tupleWith(field1: String): Tuple = + Tuple + .builder(tupleSchema) + .add(new Attribute("field1", AttributeType.STRING), field1) + .add(new Attribute("field2", AttributeType.INTEGER), 1) + .add(new Attribute("field3", AttributeType.BOOLEAN), true) + .build() + + private def isMatched(inputTuple: Tuple): Boolean = + opExec + .processTuple(inputTuple, 0) + .next() + .asInstanceOf[SchemaEnforceable] + .enforceSchema(outputSchema) + .getField[Boolean]("matched") + + private def assertMatch( + matchingType: MatchingType, + dictionary: String = dictionaryScan, + inputTuple: Tuple = tuple, + expected: Boolean = true + ): Unit = + withExecutor(matchingType, dictionary) { + assert(isMatched(inputTuple) == expected) + } + it should "prepare the dictionary at the first tuple, not at open()" in { opExec = new DictionaryMatcherOpExec(objectMapper.writeValueAsString(opDesc)) opExec.open() @@ -64,136 +99,43 @@ class DictionaryMatcherOpExecSpec extends AnyFlatSpec with BeforeAndAfter { assert(opExec.dictionaryEntries == List(dictionaryScan)) } - /** - * Test cases that all Matching Types should match the query - */ + /** Test cases that all Matching Types should match the query */ it should "match a tuple if present in the given dictionary entry when matching type is SCANBASED" in { - opDesc.matchingType = MatchingType.SCANBASED - opExec = new DictionaryMatcherOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() - val processedTuple = opExec.processTuple(tuple, 0).next() - assert( - processedTuple.asInstanceOf[SchemaEnforceable].enforceSchema(outputSchema).getField("matched") - ) - opExec.close() + assertMatch(MatchingType.SCANBASED) } it should "match a tuple if present in the given dictionary entry when matching type is SUBSTRING" in { - opDesc.matchingType = MatchingType.SUBSTRING - opExec = new DictionaryMatcherOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() - val processedTuple = opExec.processTuple(tuple, 0).next() - assert( - processedTuple.asInstanceOf[SchemaEnforceable].enforceSchema(outputSchema).getField("matched") - ) - opExec.close() + assertMatch(MatchingType.SUBSTRING) } it should "match a tuple if present in the given dictionary entry when matching type is CONJUNCTION_INDEXBASED" in { - opDesc.matchingType = MatchingType.CONJUNCTION_INDEXBASED - opExec = new DictionaryMatcherOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() - val processedTuple = opExec.processTuple(tuple, 0).next() - assert( - processedTuple.asInstanceOf[SchemaEnforceable].enforceSchema(outputSchema).getField("matched") - ) - opExec.close() + assertMatch(MatchingType.CONJUNCTION_INDEXBASED) } - /** - * Test cases that SCANBASED and SUBSTRING Matching Types should fail to match a query - */ + /** Test cases that SCANBASED and SUBSTRING Matching Types should fail to match a query */ it should "not match a tuple if not present in the given dictionary entry when matching type is SCANBASED and not exact match" in { - opDesc.dictionary = dictionaryConjunction - opDesc.matchingType = MatchingType.SCANBASED - opExec = new DictionaryMatcherOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() - val processedTuple = opExec.processTuple(tuple, 0).next() - assert( - !processedTuple - .asInstanceOf[SchemaEnforceable] - .enforceSchema(outputSchema) - .getField[Boolean]("matched") - ) - opExec.close() + assertMatch(MatchingType.SCANBASED, dictionaryConjunction, expected = false) } it should "not match a tuple if the given dictionary entry doesn't contain all the tuple when the matching type is SUBSTRING" in { - opDesc.dictionary = dictionaryConjunction - opDesc.matchingType = MatchingType.SUBSTRING - opExec = new DictionaryMatcherOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() - val processedTuple = opExec.processTuple(tuple, 0).next() - assert( - !processedTuple - .asInstanceOf[SchemaEnforceable] - .enforceSchema(outputSchema) - .getField[Boolean]("matched") - ) - opExec.close() + assertMatch(MatchingType.SUBSTRING, dictionaryConjunction, expected = false) } it should "match a tuple if present in the given dictionary entry when matching type is CONJUNCTION_INDEXBASED even with different order" in { - opDesc.dictionary = dictionaryConjunction - opDesc.matchingType = MatchingType.CONJUNCTION_INDEXBASED - opExec = new DictionaryMatcherOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() - val processedTuple = opExec.processTuple(tuple, 0).next() - assert( - processedTuple - .asInstanceOf[SchemaEnforceable] - .enforceSchema(outputSchema) - .getField[Boolean]("matched") - ) - opExec.close() + assertMatch(MatchingType.CONJUNCTION_INDEXBASED, dictionaryConjunction, expected = true) } - /** - * Test cases that only SUBSTRING Matching Type should match the query - */ + /** Test cases that only SUBSTRING Matching Type should match the query */ it should "not match a tuple if not present in the given dictionary entry when matching type is SCANBASED when the entry contains more text" in { - opDesc.dictionary = dictionarySubstring - opDesc.matchingType = MatchingType.SCANBASED - opExec = new DictionaryMatcherOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() - val processedTuple = opExec.processTuple(tuple, 0).next() - assert( - !processedTuple - .asInstanceOf[SchemaEnforceable] - .enforceSchema(outputSchema) - .getField[Boolean]("matched") - ) - opExec.close() + assertMatch(MatchingType.SCANBASED, dictionarySubstring, expected = false) } it should "not match a tuple if not present in the given dictionary entry when matching type is CONJUNCTION_INDEXBASED when the entry contains more text" in { - opDesc.dictionary = dictionarySubstring - opDesc.matchingType = MatchingType.CONJUNCTION_INDEXBASED - opExec = new DictionaryMatcherOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() - val processedTuple = opExec.processTuple(tuple, 0).next() - assert( - !processedTuple - .asInstanceOf[SchemaEnforceable] - .enforceSchema(outputSchema) - .getField[Boolean]("matched") - ) - opExec.close() + assertMatch(MatchingType.CONJUNCTION_INDEXBASED, dictionarySubstring, expected = false) } it should "match a tuple if not present in the given dictionary entry when matching type is SUBSTRING when the entry contains more text" in { - opDesc.dictionary = dictionarySubstring - opDesc.matchingType = MatchingType.SUBSTRING - opExec = new DictionaryMatcherOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() - val processedTuple = opExec.processTuple(tuple, 0).next() - assert( - processedTuple - .asInstanceOf[SchemaEnforceable] - .enforceSchema(outputSchema) - .getField[Boolean]("matched") - ) - opExec.close() + assertMatch(MatchingType.SUBSTRING, dictionarySubstring, expected = true) } it should "close properly" in { @@ -214,57 +156,31 @@ class DictionaryMatcherOpExecSpec extends AnyFlatSpec with BeforeAndAfter { assert(opExec.dictionaryEntries == null) } - private def tupleWith(field1: String): Tuple = - Tuple - .builder(tupleSchema) - .add(new Attribute("field1", AttributeType.STRING), field1) - .add(new Attribute("field2", AttributeType.INTEGER), 1) - .add(new Attribute("field3", AttributeType.BOOLEAN), true) - .build() - - private def isMatched(inputTuple: Tuple): Boolean = - opExec - .processTuple(inputTuple, 0) - .next() - .asInstanceOf[SchemaEnforceable] - .enforceSchema(outputSchema) - .getField[Boolean]("matched") - it should "not label an empty field as matched even though every entry contains the empty string" in { - opDesc.dictionary = dictionaryScan - opDesc.matchingType = MatchingType.SUBSTRING - opExec = new DictionaryMatcherOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() // Without the empty-text guard, SUBSTRING would ask whether "nice a a person" // contains "" -- which every string does -- and report a match. - assert(!isMatched(tupleWith(""))) - opExec.close() + assertMatch(MatchingType.SUBSTRING, dictionaryScan, tupleWith(""), expected = false) } it should "not match under CONJUNCTION_INDEXBASED when the field tokenizes to nothing" in { - opDesc.dictionary = "the" - opDesc.matchingType = MatchingType.CONJUNCTION_INDEXBASED - opExec = new DictionaryMatcherOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() // "the" is an English stop word, so both the entry and the field tokenize to // the empty set, and the empty set is trivially a subset of itself. Only the // explicit non-emptiness check keeps this from being reported as a match. - assert(!isMatched(tupleWith("the"))) - opExec.close() + assertMatch(MatchingType.CONJUNCTION_INDEXBASED, "the", tupleWith("the"), expected = false) } it should "ignore dictionary tokens that stem into an English stop word" in { - opDesc.dictionary = "nice willing person" - opDesc.matchingType = MatchingType.CONJUNCTION_INDEXBASED - opExec = new DictionaryMatcherOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() // The analyzer stems "willing" to "will", which is an English stop word, so the // entry reduces to {nice, person} and is still a subset of the field's // {person, nice}. The field deliberately reverses the two words so that plain // substring containment ("nice willing person" contains "person nice") is // false: only the tokenized conjunction path can satisfy this assertion. - assert(isMatched(tupleWith("person nice"))) - opExec.close() + assertMatch( + MatchingType.CONJUNCTION_INDEXBASED, + "nice willing person", + tupleWith("person nice"), + expected = true + ) } it should "ignore dictionary tokens that are URL stop words" in { @@ -274,46 +190,40 @@ class DictionaryMatcherOpExecSpec extends AnyFlatSpec with BeforeAndAfter { // entry can never fire (recorded as dead data, not pinned). As above, the // field reverses the word order so substring containment cannot satisfy it. for (urlWord <- List("http", "org", "net", "com", "store", "www", "html")) { - opDesc.dictionary = s"nice $urlWord person" - opDesc.matchingType = MatchingType.CONJUNCTION_INDEXBASED - opExec = new DictionaryMatcherOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() - withClue(s"URL stop word $urlWord: ")(assert(isMatched(tupleWith("person nice")))) - opExec.close() + withClue(s"URL stop word $urlWord: ") { + assertMatch( + MatchingType.CONJUNCTION_INDEXBASED, + s"nice $urlWord person", + tupleWith("person nice"), + expected = true + ) + } } } it should "split the dictionary on commas" in { - opDesc.dictionary = "cat,dog" - opDesc.matchingType = MatchingType.SCANBASED - opExec = new DictionaryMatcherOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() - // Two entries, not one: "dog" is an entry on its own and the raw text is not. - assert(isMatched(tupleWith("dog"))) - assert(!isMatched(tupleWith("cat,dog"))) - opExec.close() + withExecutor(MatchingType.SCANBASED, "cat,dog") { + // Two entries, not one: "dog" is an entry on its own and the raw text is not. + assert(isMatched(tupleWith("dog"))) + assert(!isMatched(tupleWith("cat,dog"))) + } } it should "lower-case the tuple field before comparing it to the dictionary" in { - opDesc.dictionary = dictionaryScan - opDesc.matchingType = MatchingType.SCANBASED - opExec = new DictionaryMatcherOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() // Dictionary entries are lower-cased when they are split, so the field has to // be folded too or an upper-case field could never match a lower-case entry. - assert(isMatched(tupleWith(dictionaryScan.toUpperCase))) - opExec.close() + assertMatch( + MatchingType.SCANBASED, + dictionaryScan, + tupleWith(dictionaryScan.toUpperCase), + expected = true + ) } it should "report a null field as unmatched instead of failing" in { - opDesc.dictionary = dictionaryScan - opDesc.matchingType = MatchingType.SCANBASED - opExec = new DictionaryMatcherOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() // The dictionary lookup casts the field to String and lower-cases it, so a null // field has to be filtered out before the lookup is reached. - assert(!isMatched(tupleWith(null))) - opExec.close() + assertMatch(MatchingType.SCANBASED, dictionaryScan, tupleWith(null), expected = false) } it should "empty the tokenized dictionary on close" in { @@ -330,10 +240,6 @@ class DictionaryMatcherOpExecSpec extends AnyFlatSpec with BeforeAndAfter { } it should "fail loudly rather than silently report no match when no matching type is configured" in { - opDesc.dictionary = dictionaryScan - opDesc.matchingType = null - opExec = new DictionaryMatcherOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() // A descriptor whose "Matching type" is absent from the JSON deserializes to a // null matchingType -- Jackson does not enforce `required` on read -- so this is // a reachable runtime state. What is pinned is only that the operator refuses @@ -341,9 +247,10 @@ class DictionaryMatcherOpExecSpec extends AnyFlatSpec with BeforeAndAfter { // is scalac's business (today a MatchError from the non-exhaustive match) and // is deliberately not named, so replacing it with an explicit, clearer throw // would not break this test. - intercept[RuntimeException] { - opExec.processTuple(tuple, 0).next() + withExecutor(null, dictionaryScan) { + intercept[RuntimeException] { + opExec.processTuple(tuple, 0).next() + } } - opExec.close() } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceInferenceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceInferenceOpDescSpec.scala index 5592944018b..44e2885a267 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceInferenceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/HuggingFaceInferenceOpDescSpec.scala @@ -249,7 +249,7 @@ class HuggingFaceInferenceOpDescSpec extends AnyFlatSpec with Matchers { code should include("self._read_image_input()") code should include("self._read_binary_value") code should include("self._compress_image_bytes") - code should include("""if content_type.startswith("image/"):""") + code should include("""if content_type.startswith(("image/", "audio/", "video/")):""") } it should @@ -320,8 +320,8 @@ class HuggingFaceInferenceOpDescSpec extends AnyFlatSpec with Matchers { code should include("MAX_REMOTE_FETCH_BYTES") code should include("Remote file exceeds the") // all remote fetch sites route through the helper (no raw requests.get on these URLs) - code should include("_, data = self._fetch_remote_url(image_input)") - code should include("_, data = self._fetch_remote_url(audio_input)") + code should include("def _read_media_upload(self, value, error_message):") + code should include("_, data = self._fetch_remote_url(source)") code should include("_, data = self._fetch_remote_url(val)") code should include("raw_content_type, data = self._fetch_remote_url(url)") code should not include "def _audio_url_to_data_url" @@ -511,7 +511,7 @@ class HuggingFaceInferenceOpDescSpec extends AnyFlatSpec with Matchers { ) code should not include "data:audio/wav;base64" code should include( - """if content_type.startswith("audio/") or content_type.startswith("video/"):""" + """if content_type.startswith(("image/", "audio/", "video/")):""" ) } @@ -560,6 +560,7 @@ class HuggingFaceInferenceOpDescSpec extends AnyFlatSpec with Matchers { MediaGenCodegen.tasks.foreach { t => val code = makeDesc(task = t).generatePythonCode() code should include("""payload = {"inputs": prompt_value}""") + code should include("""if "video" in body:""") } } @@ -713,8 +714,9 @@ class HuggingFaceInferenceOpDescSpec extends AnyFlatSpec with Matchers { it should "report a clear error when a configured image/audio column is missing from the input table" in { val code = makeDesc().generatePythonCode() - code should include("Input Image Column '") - code should include("Input Audio Column '") + code should include("""media_kind = "image" if task in image_tasks""") + code should include(""""audio" if task in audio_only_tasks""") + code should include("Input {media_kind.title()} Column") code should include("not found in the input table") } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/codegen/HuggingFaceCodegenBaseSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/codegen/HuggingFaceCodegenBaseSpec.scala index 96fcaba4478..e8e428dd09e 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/codegen/HuggingFaceCodegenBaseSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/codegen/HuggingFaceCodegenBaseSpec.scala @@ -139,12 +139,13 @@ class HuggingFaceCodegenBaseSpec extends AnyFlatSpec with Matchers { } it should "route every image chat text part through the task-aware helper" in { - // Every image chat branch builds a two-part content list — the three in - // _call_provider (zai-org, OpenAI-compatible, unknown-provider fallback) and - // the model-author one in _post_with_fallback. The text part must be the - // reformulated prompt, not the bare prompt_value. + // Three routes share the provider wrapper and the model-author route builds parts directly. val out = HuggingFaceCodegenBase.render(textContext, StubCodegen) - out.split("""\{"type": "text", "text": self\._chat_content_for_task\(""").length - 1 shouldBe 4 + out.split("""messages = self\._provider_chat_messages\(""").length - 1 shouldBe 3 + out.split("""self\._image_chat_parts\(""").length - 1 shouldBe 2 + out + .split("""self\._chat_content_for_task\(pipeline_payload, prompt_value\) or""") + .length - 1 shouldBe 2 out should not include """{"type": "text", "text": prompt_value if prompt_value else""" } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/codegen/TaskCodegenSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/codegen/TaskCodegenSpec.scala index 7ee15f935d8..94193444a11 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/codegen/TaskCodegenSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/codegen/TaskCodegenSpec.scala @@ -26,13 +26,8 @@ import org.scalatest.matchers.should.Matchers * Covers the one piece of behavior the TaskCodegen trait implements itself: * the default `tasks` set. * - * Reachability, stated plainly: no production path executes this default today. - * The only four `.tasks` call sites in main are HuggingFaceInferenceOpDesc's - * dispatcher registrations (lines 175-178), and all four are on codegens that - * declare `override val tasks`; TextGenCodegen, the sole single-task codegen, is - * registered by its `task` string and never asks for `tasks`. This suite - * therefore documents an extension point for future single-task codegens rather - * than guarding a shipped path. + * HuggingFaceInferenceOpDesc uses this default when registering a codegen + * that handles one task. */ class TaskCodegenSpec extends AnyFlatSpec with Matchers { diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intervalJoin/IntervalOpExecSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intervalJoin/IntervalOpExecSpec.scala index 5dbf840ff6e..f9f8aca0709 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intervalJoin/IntervalOpExecSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intervalJoin/IntervalOpExecSpec.scala @@ -48,13 +48,8 @@ class IntervalOpExecSpec extends AnyFlatSpec { .build() } - def doubleTuple(name: String, n: Int = 1, i: Double): Tuple = { - Tuple - .builder(schema(name, AttributeType.DOUBLE, n)) - .add(new Attribute(name, AttributeType.DOUBLE), i) - .add(new Attribute(name + "_" + 1, AttributeType.DOUBLE), i) - .build() - } + def doubleTuple(name: String, n: Int = 1, i: Double): Tuple = + newTuple[Double](name, n, i, AttributeType.DOUBLE) def schema(name: String, attributeType: AttributeType, n: Int = 1): Schema = { Schema() @@ -63,13 +58,8 @@ class IntervalOpExecSpec extends AnyFlatSpec { } - def timeStampTuple(name: String, n: Int = 1, i: Timestamp): Tuple = { - Tuple - .builder(schema(name, AttributeType.TIMESTAMP, n)) - .add(new Attribute(name, AttributeType.TIMESTAMP), i) - .add(new Attribute(name + "_" + 1, AttributeType.TIMESTAMP), i) - .build() - } + def timeStampTuple(name: String, n: Int = 1, i: Timestamp): Tuple = + newTuple[Timestamp](name, n, i, AttributeType.TIMESTAMP) def bruteForceJoin[T]( leftInput: Array[T], @@ -276,87 +266,29 @@ class IntervalOpExecSpec extends AnyFlatSpec { rangeList ) } - it should "work with Integer value int [] interval, simple test" in { - val pointList: Array[Int] = (1 to 10).toArray[Int] - val rangeList: Array[Int] = Array(1, 5, 8) - testJoin[Int]( - "point", - "range", - includeLeftBound = true, - includeRightBound = true, - AttributeType.INTEGER, - TimeIntervalType.DAY, - 3, - pointList, - rangeList - ) - } - - it should "work with Integer value int [] interval, same key" in { - - val pointList: Array[Int] = (1 to 10).toArray[Int] - val rangeList: Array[Int] = Array(1, 5, 8) - testJoin[Int]( - "same", - "same", - includeLeftBound = true, - includeRightBound = true, - AttributeType.INTEGER, - TimeIntervalType.DAY, - 3, - pointList, - rangeList - ) - } - - it should "work with Integer value int [) interval" in { - - val pointList: Array[Int] = (1 to 10).toArray[Int] - val rangeList: Array[Int] = Array(1, 5, 8) - testJoin[Int]( - "point", - "range", - includeLeftBound = true, - includeRightBound = false, - AttributeType.INTEGER, - TimeIntervalType.DAY, - 3, - pointList, - rangeList - ) - } - - it should "work with Integer value int (] interval" in { - - val pointList: Array[Int] = (1 to 10).toArray[Int] - val rangeList: Array[Int] = Array(1, 5, 8) - testJoin[Int]( - "point", - "range", - includeLeftBound = false, - includeRightBound = true, - AttributeType.INTEGER, - TimeIntervalType.DAY, - 3, - pointList, - rangeList - ) - } - it should "work with Integer value int () interval" in { - - val pointList: Array[Int] = (1 to 10).toArray[Int] - val rangeList: Array[Int] = Array(1, 5, 8) - testJoin[Int]( - "point", - "range", - includeLeftBound = false, - includeRightBound = false, - AttributeType.INTEGER, - TimeIntervalType.DAY, - 3, - pointList, - rangeList - ) + Seq( + ("work with Integer value int [] interval, simple test", "point", "range", true, true), + ("work with Integer value int [] interval, same key", "same", "same", true, true), + ("work with Integer value int [) interval", "point", "range", true, false), + ("work with Integer value int (] interval", "point", "range", false, true), + ("work with Integer value int () interval", "point", "range", false, false) + ).foreach { + case (title, leftKey, rightKey, includeLeft, includeRight) => + it should title in { + val pointList: Array[Int] = (1 to 10).toArray[Int] + val rangeList: Array[Int] = Array(1, 5, 8) + testJoin[Int]( + leftKey, + rightKey, + includeLeftBound = includeLeft, + includeRightBound = includeRight, + AttributeType.INTEGER, + TimeIntervalType.DAY, + 3, + pointList, + rangeList + ) + } } it should "work with Timestamp value int [] interval" in { val pointList: Array[Timestamp] = (1L to 10L) @@ -406,54 +338,27 @@ class IntervalOpExecSpec extends AnyFlatSpec { opExec.close() } - it should "work with Long value int [] interval" in { - - val pointList: Array[Long] = (1L to 10L).toArray - val rangeList: Array[Long] = Array(1L, 5L, 8L) - testJoin[Long]( - "point", - "range", - includeLeftBound = true, - includeRightBound = true, - AttributeType.LONG, - TimeIntervalType.DAY, - 3, - pointList, - rangeList - ) - } - - it should "work with basic two input streams with left empty table" in { - - val pointList: Array[Long] = Array() - val rangeList: Array[Long] = Array(1L, 5L, 8L) - testJoin[Long]( - "point", - "range", - includeLeftBound = true, - includeRightBound = true, - AttributeType.LONG, - TimeIntervalType.DAY, - 3, - pointList, - rangeList - ) - } - - it should "work with basic two input streams with right empty table" in { - val pointList: Array[Long] = (1L to 10L).toArray - val rangeList: Array[Long] = Array() - testJoin[Long]( - "point", - "range", - includeLeftBound = true, - includeRightBound = true, - AttributeType.LONG, - TimeIntervalType.DAY, - 3, - pointList, - rangeList - ) + Seq( + ("work with Long value int [] interval", false, false), + ("work with basic two input streams with left empty table", true, false), + ("work with basic two input streams with right empty table", false, true) + ).foreach { + case (title, emptyLeft, emptyRight) => + it should title in { + val pointList: Array[Long] = if (emptyLeft) Array() else (1L to 10L).toArray + val rangeList: Array[Long] = if (emptyRight) Array() else Array(1L, 5L, 8L) + testJoin[Long]( + "point", + "range", + includeLeftBound = true, + includeRightBound = true, + AttributeType.LONG, + TimeIntervalType.DAY, + 3, + pointList, + rangeList + ) + } } it should "test larger dataset(1k)" in { diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/keywordSearch/KeywordSearchOpExecSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/keywordSearch/KeywordSearchOpExecSpec.scala index 542ccbf4b3a..d035a3da7b0 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/keywordSearch/KeywordSearchOpExecSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/keywordSearch/KeywordSearchOpExecSpec.scala @@ -54,204 +54,95 @@ class KeywordSearchOpExecSpec extends AnyFlatSpec with BeforeAndAfter { createTuple("everything was absolutely perfect.") ) - it should "find exact match with single number" in { + private def searchMatches(keyword: String, isCaseSensitive: Boolean = false): List[String] = { opDesc.attribute = "text" - opDesc.keyword = "3" + opDesc.keyword = keyword + opDesc.isCaseSensitive = isCaseSensitive val opExec = new KeywordSearchOpExec(objectMapper.writeValueAsString(opDesc)) opExec.open() - val results = testData.filter(t => opExec.processTuple(t, inputPort).nonEmpty) - assert(results.length == 1) - assert(results.head.getField[String]("text") == "3 stars") - opExec.close() + try { + testData + .filter(t => opExec.processTuple(t, inputPort).hasNext) + .map(_.getField[String]("text")) + } finally { + opExec.close() + opDesc.isCaseSensitive = false + } + } + + it should "find exact match with single number" in { + assert(searchMatches("3") == List("3 stars")) } it should "find exact phrase match" in { - opDesc.attribute = "text" - opDesc.keyword = "\"3 stars\"" - val opExec = new KeywordSearchOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() - val results = testData.filter(t => opExec.processTuple(t, inputPort).nonEmpty) - assert(results.length == 1) - assert(results.head.getField[String]("text") == "3 stars") - opExec.close() + assert(searchMatches("\"3 stars\"") == List("3 stars")) } it should "find all occurrences of Trump" in { - opDesc.attribute = "text" - opDesc.keyword = "Trump" - val opExec = new KeywordSearchOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() - val results = testData.filter(t => opExec.processTuple(t, inputPort).nonEmpty) - assert(results.length == 2) - assert(results.map(_.getField[String]("text")).toSet == Set("Trump Biden", "Trump")) - opExec.close() + assert(searchMatches("Trump").toSet == Set("Trump Biden", "Trump")) } it should "find all occurrences of Biden" in { - opDesc.attribute = "text" - opDesc.keyword = "Biden" - val opExec = new KeywordSearchOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() - val results = testData.filter(t => opExec.processTuple(t, inputPort).nonEmpty) - assert(results.length == 1) - assert(results.head.getField[String]("text") == "Trump Biden") - opExec.close() + assert(searchMatches("Biden") == List("Trump Biden")) } it should "find records containing both Trump AND Biden" in { - opDesc.attribute = "text" - opDesc.keyword = "Trump AND Biden" - val opExec = new KeywordSearchOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() - val results = testData.filter(t => opExec.processTuple(t, inputPort).nonEmpty) - assert(results.length == 1) - assert(results.head.getField[String]("text") == "Trump Biden") - opExec.close() + assert(searchMatches("Trump AND Biden") == List("Trump Biden")) } it should "find no matches for exact phrase 'Trump AND Biden'" in { - opDesc.attribute = "text" - opDesc.keyword = "\"Trump AND Biden\"" - val opExec = new KeywordSearchOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() - val results = testData.filter(t => opExec.processTuple(t, inputPort).hasNext) - assert(results.isEmpty) - opExec.close() + assert(searchMatches("\"Trump AND Biden\"").isEmpty) } it should "find no matches for partial word 'ell'" in { - opDesc.attribute = "text" - opDesc.keyword = "ell" - val opExec = new KeywordSearchOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() - val results = testData.filter(t => opExec.processTuple(t, inputPort).hasNext) - assert(results.isEmpty) - opExec.close() + assert(searchMatches("ell").isEmpty) } it should "find exact match for word 'the'" in { - opDesc.attribute = "text" - opDesc.keyword = "the" - val opExec = new KeywordSearchOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() - val results = testData.filter(t => opExec.processTuple(t, inputPort).hasNext) - assert(results.length == 1) - assert(results.head.getField[String]("text") == "the name") - opExec.close() + assert(searchMatches("the") == List("the name")) } it should "find exact match for word 'an'" in { - opDesc.attribute = "text" - opDesc.keyword = "an" - val opExec = new KeywordSearchOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() - val results = testData.filter(t => opExec.processTuple(t, inputPort).hasNext) - assert(results.length == 1) - assert(results.head.getField[String]("text") == "an eye") - opExec.close() + assert(searchMatches("an") == List("an eye")) } it should "find exact match for word 'to'" in { - opDesc.attribute = "text" - opDesc.keyword = "to" - val opExec = new KeywordSearchOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() - val results = testData.filter(t => opExec.processTuple(t, inputPort).hasNext) - assert(results.length == 1) - assert(results.head.getField[String]("text") == "to you") - opExec.close() + assert(searchMatches("to") == List("to you")) } it should "find case-insensitive match for 'twitter'" in { - opDesc.attribute = "text" - opDesc.keyword = "twitter" - val opExec = new KeywordSearchOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() - val results = testData.filter(t => opExec.processTuple(t, inputPort).hasNext) - assert(results.length == 1) - assert(results.head.getField[String]("text") == "Twitter") - opExec.close() + assert(searchMatches("twitter") == List("Twitter")) } it should "find exact match for Korean text '안녕하세요'" in { - opDesc.attribute = "text" - opDesc.keyword = "안녕하세요" - val opExec = new KeywordSearchOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() - val results = testData.filter(t => opExec.processTuple(t, inputPort).hasNext) - assert(results.length == 1) - assert(results.head.getField[String]("text") == "안녕하세요") - opExec.close() + assert(searchMatches("안녕하세요") == List("안녕하세요")) } it should "find exact match for Chinese text '你好'" in { - opDesc.attribute = "text" - opDesc.keyword = "你好" - val opExec = new KeywordSearchOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() - val results = testData.filter(t => opExec.processTuple(t, inputPort).hasNext) - assert(results.length == 1) - assert(results.head.getField[String]("text") == "你好") - opExec.close() + assert(searchMatches("你好") == List("你好")) } it should "find no matches for special character '@'" in { - opDesc.attribute = "text" - opDesc.keyword = "@" - val opExec = new KeywordSearchOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() - val results = testData.filter(t => opExec.processTuple(t, inputPort).hasNext) - assert(results.isEmpty) - opExec.close() + assert(searchMatches("@").isEmpty) } it should "find exact match for special characters '_!@,-'" in { - opDesc.attribute = "text" - opDesc.keyword = "_!@,-" - val opExec = new KeywordSearchOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() - val results = testData.filter(t => opExec.processTuple(t, inputPort).hasNext) - assert(results.isEmpty) - opExec.close() + assert(searchMatches("_!@,-").isEmpty) } it should "not match 'twitter' against 'Twitter' when case-sensitive" in { - opDesc.attribute = "text" - opDesc.keyword = "twitter" - opDesc.isCaseSensitive = true - val opExec = new KeywordSearchOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() - val results = testData.filter(t => opExec.processTuple(t, inputPort).hasNext) - assert(results.isEmpty) - opExec.close() - opDesc.isCaseSensitive = false + assert(searchMatches("twitter", isCaseSensitive = true).isEmpty) } it should "match 'Twitter' against 'Twitter' when case-sensitive" in { - opDesc.attribute = "text" - opDesc.keyword = "Twitter" - opDesc.isCaseSensitive = true - val opExec = new KeywordSearchOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() - val results = testData.filter(t => opExec.processTuple(t, inputPort).hasNext) - assert(results.length == 1) - assert(results.head.getField[String]("text") == "Twitter") - opExec.close() - opDesc.isCaseSensitive = false + assert(searchMatches("Twitter", isCaseSensitive = true) == List("Twitter")) } it should "match a word adjacent to punctuation when case-sensitive" in { // Regression test: case sensitivity must change only case handling, not // punctuation -- "perfect" should still match "...perfect." (word-boundary split). - opDesc.attribute = "text" - opDesc.keyword = "perfect" - opDesc.isCaseSensitive = true - val opExec = new KeywordSearchOpExec(objectMapper.writeValueAsString(opDesc)) - opExec.open() - val results = testData.filter(t => opExec.processTuple(t, inputPort).hasNext) - assert(results.length == 1) - assert(results.head.getField[String]("text") == "everything was absolutely perfect.") - opExec.close() - opDesc.isCaseSensitive = false + assert( + searchMatches("perfect", isCaseSensitive = true) == List("everything was absolutely perfect.") + ) } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/KNNTrainer/SklearnAdvancedKNNClassifierTrainerOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/KNNTrainer/SklearnAdvancedKNNClassifierTrainerOpDescSpec.scala index 7d3acae13e1..e7fd4ba3e54 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/KNNTrainer/SklearnAdvancedKNNClassifierTrainerOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/KNNTrainer/SklearnAdvancedKNNClassifierTrainerOpDescSpec.scala @@ -21,11 +21,13 @@ package org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.KNNTrai import org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.base.{ HyperParameters, - SklearnMLOperatorDescriptor + SklearnTrainerOpDescSpecBehaviors } import org.scalatest.flatspec.AnyFlatSpec -class SklearnAdvancedKNNClassifierTrainerOpDescSpec extends AnyFlatSpec { +class SklearnAdvancedKNNClassifierTrainerOpDescSpec + extends AnyFlatSpec + with SklearnTrainerOpDescSpecBehaviors { private def hyperParam( parameter: SklearnAdvancedKNNParameters, @@ -46,45 +48,17 @@ class SklearnAdvancedKNNClassifierTrainerOpDescSpec extends AnyFlatSpec { d.generatePythonCode() } - "SklearnAdvancedKNNClassifierTrainerOpDesc.getImportStatements" should - "return the canonical KNeighborsClassifier import" in { - val d = new SklearnAdvancedKNNClassifierTrainerOpDesc - assert(d.getImportStatements == "from sklearn.neighbors import KNeighborsClassifier") - } - - "SklearnAdvancedKNNClassifierTrainerOpDesc.getOperatorInfo" should - "return 'KNN Classifier'" in { - val d = new SklearnAdvancedKNNClassifierTrainerOpDesc - assert(d.getOperatorInfo == "KNN Classifier") - } - - it should "be stable across two instances (no instance-state interaction)" in { - val a = new SklearnAdvancedKNNClassifierTrainerOpDesc - val b = new SklearnAdvancedKNNClassifierTrainerOpDesc - assert(a.getImportStatements == b.getImportStatements) - assert(a.getOperatorInfo == b.getOperatorInfo) - } - - "SklearnAdvancedKNNClassifierTrainerOpDesc" should - "extend SklearnMLOperatorDescriptor (compile-time enforced)" in { - val d: SklearnMLOperatorDescriptor[SklearnAdvancedKNNParameters] = - new SklearnAdvancedKNNClassifierTrainerOpDesc - assert(d.getImportStatements.contains("KNeighborsClassifier")) - } - - it should "be matchable via the SklearnMLOperatorDescriptor type-pattern" in { - val any: AnyRef = new SklearnAdvancedKNNClassifierTrainerOpDesc - val matched = any match { - case _: SklearnMLOperatorDescriptor[_] => true - case _ => false - } - assert(matched) - } + trainerDescriptorBehaviors[ + SklearnAdvancedKNNParameters, + SklearnAdvancedKNNClassifierTrainerOpDesc + ]( + "SklearnAdvancedKNNClassifierTrainerOpDesc", + () => new SklearnAdvancedKNNClassifierTrainerOpDesc, + "from sklearn.neighbors import KNeighborsClassifier", + "KNN Classifier", + expectedImportToken = "KNeighborsClassifier" + ) - // The declared type is emitted as the callable that converts what the user - // typed, so a hyperparameter is only usable when that callable can return - // something scikit-learn accepts. These two could not: `metric` names a word - // from a fixed set, and `metric_params` a mapping. "SklearnAdvancedKNNParameters.metric" should "convert with str, the accepted metrics being words" in { assert( @@ -105,8 +79,6 @@ class SklearnAdvancedKNNClassifierTrainerOpDescSpec extends AnyFlatSpec { hyperParam(SklearnAdvancedKNNParameters.metric, "minkowski"), hyperParam(SklearnAdvancedKNNParameters.metric_params, """{"p": 2}""") ) - // The values themselves travel base64-encoded, so what is checked here is - // which callable each one is handed to. val modelCall = code.linesIterator.find(_.contains("model = KNeighborsClassifier(")).get assert(modelCall.contains("metric = str (")) assert(modelCall.contains("metric_params = json.loads (")) diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/KNNTrainer/SklearnAdvancedKNNRegressorTrainerOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/KNNTrainer/SklearnAdvancedKNNRegressorTrainerOpDescSpec.scala index 0715f120c27..01e0e46f9b4 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/KNNTrainer/SklearnAdvancedKNNRegressorTrainerOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/KNNTrainer/SklearnAdvancedKNNRegressorTrainerOpDescSpec.scala @@ -19,48 +19,25 @@ package org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.KNNTrainer -import org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.base.SklearnMLOperatorDescriptor +import org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.base.SklearnTrainerOpDescSpecBehaviors import org.scalatest.flatspec.AnyFlatSpec -class SklearnAdvancedKNNRegressorTrainerOpDescSpec extends AnyFlatSpec { - - "SklearnAdvancedKNNRegressorTrainerOpDesc.getImportStatements" should - "return the canonical KNeighborsRegressor import" in { - val d = new SklearnAdvancedKNNRegressorTrainerOpDesc - assert(d.getImportStatements == "from sklearn.neighbors import KNeighborsRegressor") - } - - "SklearnAdvancedKNNRegressorTrainerOpDesc.getOperatorInfo" should "return 'KNN Regressor'" in { - val d = new SklearnAdvancedKNNRegressorTrainerOpDesc - assert(d.getOperatorInfo == "KNN Regressor") - } - - it should "be stable across two instances (no instance-state interaction)" in { - val a = new SklearnAdvancedKNNRegressorTrainerOpDesc - val b = new SklearnAdvancedKNNRegressorTrainerOpDesc - assert(a.getImportStatements == b.getImportStatements) - assert(a.getOperatorInfo == b.getOperatorInfo) - } - - "SklearnAdvancedKNNRegressorTrainerOpDesc" should - "extend SklearnMLOperatorDescriptor (compile-time enforced)" in { - val d: SklearnMLOperatorDescriptor[SklearnAdvancedKNNParameters] = - new SklearnAdvancedKNNRegressorTrainerOpDesc - assert(d.getImportStatements.contains("KNeighborsRegressor")) - } - - it should "be matchable via the SklearnMLOperatorDescriptor type-pattern" in { - val any: AnyRef = new SklearnAdvancedKNNRegressorTrainerOpDesc - val matched = any match { - case _: SklearnMLOperatorDescriptor[_] => true - case _ => false - } - assert(matched) - } +class SklearnAdvancedKNNRegressorTrainerOpDescSpec + extends AnyFlatSpec + with SklearnTrainerOpDescSpecBehaviors { + + trainerDescriptorBehaviors[ + SklearnAdvancedKNNParameters, + SklearnAdvancedKNNRegressorTrainerOpDesc + ]( + "SklearnAdvancedKNNRegressorTrainerOpDesc", + () => new SklearnAdvancedKNNRegressorTrainerOpDesc, + "from sklearn.neighbors import KNeighborsRegressor", + "KNN Regressor", + expectedImportToken = "KNeighborsRegressor" + ) it should "differ from the Classifier sibling on both methods" in { - // Catches a copy-paste regression where the Regressor accidentally - // returned the Classifier's strings (or vice-versa). val regressor = new SklearnAdvancedKNNRegressorTrainerOpDesc val classifier = new SklearnAdvancedKNNClassifierTrainerOpDesc assert(regressor.getImportStatements != classifier.getImportStatements) diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/SVCTrainer/SklearnAdvancedSVCTrainerOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/SVCTrainer/SklearnAdvancedSVCTrainerOpDescSpec.scala index c554dae6db4..9d86bece600 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/SVCTrainer/SklearnAdvancedSVCTrainerOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/SVCTrainer/SklearnAdvancedSVCTrainerOpDescSpec.scala @@ -19,42 +19,19 @@ package org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.SVCTrainer -import org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.base.SklearnMLOperatorDescriptor +import org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.base.SklearnTrainerOpDescSpecBehaviors import org.scalatest.flatspec.AnyFlatSpec -class SklearnAdvancedSVCTrainerOpDescSpec extends AnyFlatSpec { - - "SklearnAdvancedSVCTrainerOpDesc.getImportStatements" should - "return the canonical SVC import (from sklearn.svm)" in { - val d = new SklearnAdvancedSVCTrainerOpDesc - assert(d.getImportStatements == "from sklearn.svm import SVC") - } - - "SklearnAdvancedSVCTrainerOpDesc.getOperatorInfo" should "return 'SVM Classifier'" in { - val d = new SklearnAdvancedSVCTrainerOpDesc - assert(d.getOperatorInfo == "SVM Classifier") - } - - it should "be stable across two instances (no instance-state interaction)" in { - val a = new SklearnAdvancedSVCTrainerOpDesc - val b = new SklearnAdvancedSVCTrainerOpDesc - assert(a.getImportStatements == b.getImportStatements) - assert(a.getOperatorInfo == b.getOperatorInfo) - } - - "SklearnAdvancedSVCTrainerOpDesc" should - "extend SklearnMLOperatorDescriptor (compile-time enforced)" in { - val d: SklearnMLOperatorDescriptor[SklearnAdvancedSVCParameters] = - new SklearnAdvancedSVCTrainerOpDesc - assert(d.getImportStatements.contains("SVC")) - } - - it should "be matchable via the SklearnMLOperatorDescriptor type-pattern" in { - val any: AnyRef = new SklearnAdvancedSVCTrainerOpDesc - val matched = any match { - case _: SklearnMLOperatorDescriptor[_] => true - case _ => false - } - assert(matched) - } +class SklearnAdvancedSVCTrainerOpDescSpec + extends AnyFlatSpec + with SklearnTrainerOpDescSpecBehaviors { + + trainerDescriptorBehaviors[SklearnAdvancedSVCParameters, SklearnAdvancedSVCTrainerOpDesc]( + "SklearnAdvancedSVCTrainerOpDesc", + () => new SklearnAdvancedSVCTrainerOpDesc, + "from sklearn.svm import SVC", + "SVM Classifier", + importCaseSuffix = "from sklearn.svm", + expectedImportToken = "SVC" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/SVRTrainer/SklearnAdvancedSVRTrainerOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/SVRTrainer/SklearnAdvancedSVRTrainerOpDescSpec.scala index 80e7c87cc0b..b9167417b8a 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/SVRTrainer/SklearnAdvancedSVRTrainerOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/SVRTrainer/SklearnAdvancedSVRTrainerOpDescSpec.scala @@ -20,44 +20,21 @@ package org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.SVRTrainer import org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.SVCTrainer.SklearnAdvancedSVCTrainerOpDesc -import org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.base.SklearnMLOperatorDescriptor +import org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.base.SklearnTrainerOpDescSpecBehaviors import org.scalatest.flatspec.AnyFlatSpec -class SklearnAdvancedSVRTrainerOpDescSpec extends AnyFlatSpec { - - "SklearnAdvancedSVRTrainerOpDesc.getImportStatements" should - "return the canonical SVR import (from sklearn.svm)" in { - val d = new SklearnAdvancedSVRTrainerOpDesc - assert(d.getImportStatements == "from sklearn.svm import SVR") - } - - "SklearnAdvancedSVRTrainerOpDesc.getOperatorInfo" should "return 'SVM Regressor'" in { - val d = new SklearnAdvancedSVRTrainerOpDesc - assert(d.getOperatorInfo == "SVM Regressor") - } - - it should "be stable across two instances (no instance-state interaction)" in { - val a = new SklearnAdvancedSVRTrainerOpDesc - val b = new SklearnAdvancedSVRTrainerOpDesc - assert(a.getImportStatements == b.getImportStatements) - assert(a.getOperatorInfo == b.getOperatorInfo) - } - - "SklearnAdvancedSVRTrainerOpDesc" should - "extend SklearnMLOperatorDescriptor (compile-time enforced)" in { - val d: SklearnMLOperatorDescriptor[SklearnAdvancedSVRParameters] = - new SklearnAdvancedSVRTrainerOpDesc - assert(d.getImportStatements.contains("SVR")) - } - - it should "be matchable via the SklearnMLOperatorDescriptor type-pattern" in { - val any: AnyRef = new SklearnAdvancedSVRTrainerOpDesc - val matched = any match { - case _: SklearnMLOperatorDescriptor[_] => true - case _ => false - } - assert(matched) - } +class SklearnAdvancedSVRTrainerOpDescSpec + extends AnyFlatSpec + with SklearnTrainerOpDescSpecBehaviors { + + trainerDescriptorBehaviors[SklearnAdvancedSVRParameters, SklearnAdvancedSVRTrainerOpDesc]( + "SklearnAdvancedSVRTrainerOpDesc", + () => new SklearnAdvancedSVRTrainerOpDesc, + "from sklearn.svm import SVR", + "SVM Regressor", + importCaseSuffix = "from sklearn.svm", + expectedImportToken = "SVR" + ) it should "differ from the SVC sibling on both methods" in { val regressor = new SklearnAdvancedSVRTrainerOpDesc diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/base/SklearnMLOperatorDescriptorSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/base/SklearnMLOperatorDescriptorSpec.scala index a4411b56970..ee63b074b80 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/base/SklearnMLOperatorDescriptorSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/base/SklearnMLOperatorDescriptorSpec.scala @@ -87,3 +87,50 @@ class SklearnMLOperatorDescriptorSpec extends AnyFlatSpec with Matchers { op.selectedFeatures shouldBe null } } + +private[sklearnAdvanced] trait SklearnTrainerOpDescSpecBehaviors { + self: org.scalatest.flatspec.AnyFlatSpec => + + protected def trainerDescriptorBehaviors[P <: ParamClass, D <: SklearnMLOperatorDescriptor[P]]( + descriptorName: String, + newOp: () => D, + expectedImport: String, + expectedOperatorInfo: String, + importCaseSuffix: String = "", + operatorInfoQuoteSingle: Boolean = true, + expectedImportToken: String + ): Unit = { + val importSuffix = if (importCaseSuffix.nonEmpty) s" ($importCaseSuffix)" else "" + val infoQuote = + if (operatorInfoQuoteSingle) s"'$expectedOperatorInfo'" else expectedOperatorInfo + + s"$descriptorName.getImportStatements" should s"return the canonical $expectedImportToken import$importSuffix" in { + assert(newOp().getImportStatements == expectedImport) + } + + s"$descriptorName.getOperatorInfo" should s"return $infoQuote" in { + assert(newOp().getOperatorInfo == expectedOperatorInfo) + } + + it should "be stable across two instances (no instance-state interaction)" in { + val a = newOp() + val b = newOp() + assert(a.getImportStatements == b.getImportStatements) + assert(a.getOperatorInfo == b.getOperatorInfo) + } + + descriptorName should "extend SklearnMLOperatorDescriptor (compile-time enforced)" in { + val d: SklearnMLOperatorDescriptor[P] = newOp() + assert(d.getImportStatements.contains(expectedImportToken)) + } + + it should "be matchable via the SklearnMLOperatorDescriptor type-pattern" in { + val any: AnyRef = newOp() + val matched = any match { + case _: SklearnMLOperatorDescriptor[_] => true + case _ => false + } + assert(matched) + } + } +} diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/projection/ProjectionOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/projection/ProjectionOpDescSpec.scala index 74ce2c759c4..a54ee9fd280 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/projection/ProjectionOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/projection/ProjectionOpDescSpec.scala @@ -29,6 +29,7 @@ import org.apache.texera.amber.core.workflow.{ } import org.scalatest.BeforeAndAfter import org.scalatest.flatspec.AnyFlatSpec + class ProjectionOpDescSpec extends AnyFlatSpec with BeforeAndAfter { val schema = new Schema( new Attribute("field1", AttributeType.STRING), @@ -41,88 +42,57 @@ class ProjectionOpDescSpec extends AnyFlatSpec with BeforeAndAfter { projectionOpDesc = new ProjectionOpDesc() } - it should "take in attribute names" in { - projectionOpDesc.attributes ++= List( - new AttributeUnit("field1", "f1"), - new AttributeUnit("fields2", "f2") - ) + private def units(pairs: (String, String)*): List[AttributeUnit] = + pairs.map { case (orig, alias) => new AttributeUnit(orig, alias) }.toList - assert(projectionOpDesc.attributes.length == 2) + private def deriveSchema(isDrop: Boolean = false)(pairs: (String, String)*): Schema = { + projectionOpDesc.isDrop = isDrop + projectionOpDesc.attributes ++= units(pairs: _*) + projectionOpDesc.getExternalOutputSchemas(Map(PortIdentity() -> schema)).values.head + } + it should "take in attribute names" in { + projectionOpDesc.attributes ++= units("field1" -> "f1", "fields2" -> "f2") + assert(projectionOpDesc.attributes.length == 2) } it should "filter schema correctly" in { - projectionOpDesc.attributes ++= List( - new AttributeUnit("field1", "f1"), - new AttributeUnit("field2", "f2") - ) - val outputSchema = - projectionOpDesc.getExternalOutputSchemas(Map(PortIdentity() -> schema)).values.head + val outputSchema = deriveSchema()("field1" -> "f1", "field2" -> "f2") assert(outputSchema.getAttributes.length == 2) - } it should "reorder schema" in { - projectionOpDesc.attributes ++= List( - new AttributeUnit("field2", "f2"), - new AttributeUnit("field1", "f1") - ) - val outputSchema = - projectionOpDesc.getExternalOutputSchemas(Map(PortIdentity() -> schema)).values.head + val outputSchema = deriveSchema()("field2" -> "f2", "field1" -> "f1") assert(outputSchema.getAttributes.length == 2) assert(outputSchema.getIndex("f2") == 0) assert(outputSchema.getIndex("f1") == 1) - } it should "raise RuntimeException on non-existing fields" in { - projectionOpDesc.attributes ++= List( - new AttributeUnit("field---5", "f5"), - new AttributeUnit("field---6", "f6") - ) assertThrows[RuntimeException] { - projectionOpDesc.getExternalOutputSchemas(Map(PortIdentity() -> schema)).values.head + deriveSchema()("field---5" -> "f5", "field---6" -> "f6") } - } it should "raise IllegalArgumentException on empty attributes" in { - assertThrows[IllegalArgumentException] { - projectionOpDesc.getExternalOutputSchemas(Map(PortIdentity() -> schema)).values.head + deriveSchema()() } - } it should "raise RuntimeException on duplicate alias" in { - - projectionOpDesc.attributes ++= List( - new AttributeUnit("field2", "f"), - new AttributeUnit("field1", "f") - ) assertThrows[RuntimeException] { - projectionOpDesc.getExternalOutputSchemas(Map(PortIdentity() -> schema)).values.head + deriveSchema()("field2" -> "f", "field1" -> "f") } } it should "allow alias to be optional" in { - projectionOpDesc.attributes ++= List( - new AttributeUnit("field1", "f1"), - new AttributeUnit("field2", "") - ) - val outputSchema = - projectionOpDesc.getExternalOutputSchemas(Map(PortIdentity() -> schema)).values.head + val outputSchema = deriveSchema()("field1" -> "f1", "field2" -> "") assert(outputSchema.getAttributes.length == 2) - } it should "derive the drop-mode schema with original names, types and order" in { - projectionOpDesc.isDrop = true - projectionOpDesc.attributes ++= List( - new AttributeUnit("field2", "") - ) - val outputSchema = - projectionOpDesc.getExternalOutputSchemas(Map(PortIdentity() -> schema)).values.head + val outputSchema = deriveSchema(isDrop = true)("field2" -> "") assert(outputSchema.getAttributes.length == 2) assert(outputSchema.getIndex("field1") == 0) assert(outputSchema.getIndex("field3") == 1) @@ -131,47 +101,27 @@ class ProjectionOpDescSpec extends AnyFlatSpec with BeforeAndAfter { } it should "ignore aliases when deriving the drop-mode schema" in { - projectionOpDesc.isDrop = true - projectionOpDesc.attributes ++= List( - new AttributeUnit("field1", "renamed") - ) - val outputSchema = - projectionOpDesc.getExternalOutputSchemas(Map(PortIdentity() -> schema)).values.head + val outputSchema = deriveSchema(isDrop = true)("field1" -> "renamed") assert(outputSchema.getAttributeNames == List("field2", "field3")) } it should "raise IllegalArgumentException when dropping a non-existent attribute" in { // Unlike the exec, which silently ignores unknown names, // Schema.remove rejects them at schema-derivation time. - projectionOpDesc.isDrop = true - projectionOpDesc.attributes ++= List( - new AttributeUnit("field---5", "f5") - ) assertThrows[IllegalArgumentException] { - projectionOpDesc.getExternalOutputSchemas(Map(PortIdentity() -> schema)).values.head + deriveSchema(isDrop = true)("field---5" -> "f5") } } it should "derive an empty schema when dropping every attribute" in { - projectionOpDesc.isDrop = true - projectionOpDesc.attributes ++= List( - new AttributeUnit("field1", ""), - new AttributeUnit("field2", ""), - new AttributeUnit("field3", "") - ) val outputSchema = - projectionOpDesc.getExternalOutputSchemas(Map(PortIdentity() -> schema)).values.head + deriveSchema(isDrop = true)("field1" -> "", "field2" -> "", "field3" -> "") assert(outputSchema.getAttributes.isEmpty) } it should "match drop names case-insensitively when deriving the schema" in { // Schema.remove lowercases both sides, matching the exec's behavior. - projectionOpDesc.isDrop = true - projectionOpDesc.attributes ++= List( - new AttributeUnit("FIELD2", "") - ) - val outputSchema = - projectionOpDesc.getExternalOutputSchemas(Map(PortIdentity() -> schema)).values.head + val outputSchema = deriveSchema(isDrop = true)("FIELD2" -> "") assert(outputSchema.getAttributeNames == List("field1", "field3")) } @@ -179,13 +129,8 @@ class ProjectionOpDescSpec extends AnyFlatSpec with BeforeAndAfter { // The exec tolerates duplicates; the schema derivation folds // Schema.remove one entry at a time, so the second removal of the same name // rejects a now non-existent attribute. - projectionOpDesc.isDrop = true - projectionOpDesc.attributes ++= List( - new AttributeUnit("field2", ""), - new AttributeUnit("field2", "") - ) assertThrows[IllegalArgumentException] { - projectionOpDesc.getExternalOutputSchemas(Map(PortIdentity() -> schema)).values.head + deriveSchema(isDrop = true)("field2" -> "", "field2" -> "") } } @@ -220,11 +165,11 @@ class ProjectionOpDescSpec extends AnyFlatSpec with BeforeAndAfter { // in the order the user put them in. "ProjectionOpDesc.generateStandaloneCode" should "select or drop the named columns" in { val keep = new ProjectionOpDesc - keep.attributes = List(new AttributeUnit("a", ""), new AttributeUnit("b", "")) + keep.attributes = units("a" -> "", "b" -> "") assert(keep.generateStandaloneCode().contains("""in1df[["a", "b"]]""")) val drop = new ProjectionOpDesc - drop.attributes = List(new AttributeUnit("a", "")) + drop.attributes = units("a" -> "") drop.isDrop = true assert(drop.generateStandaloneCode() == """out1df = in1df.drop(columns=["a"])""") } @@ -236,14 +181,14 @@ class ProjectionOpDescSpec extends AnyFlatSpec with BeforeAndAfter { val inputSchemas = Map(PortIdentity() -> schema) val keep = new ProjectionOpDesc - keep.attributes = List(new AttributeUnit("FIELD1", ""), new AttributeUnit("Field2", "f2")) + keep.attributes = units("FIELD1" -> "", "Field2" -> "f2") assert( keep.generateStandaloneCode(inputSchemas) == """out1df = in1df[["field1", "field2"]].set_axis(["FIELD1", "f2"], axis=1)""" ) val drop = new ProjectionOpDesc - drop.attributes = List(new AttributeUnit("FIELD1", "")) + drop.attributes = units("FIELD1" -> "") drop.isDrop = true assert( drop.generateStandaloneCode(inputSchemas) == """out1df = in1df.drop(columns=["field1"])""" @@ -255,8 +200,7 @@ class ProjectionOpDescSpec extends AnyFlatSpec with BeforeAndAfter { // twice. So the script names the output columns by position. it should "keep both aliases of one column read twice" in { val keep = new ProjectionOpDesc - keep.attributes = - List(new AttributeUnit("field1", "first"), new AttributeUnit("FIELD1", "second")) + keep.attributes = units("field1" -> "first", "FIELD1" -> "second") assert( keep.getExternalOutputSchemas(Map(PortIdentity() -> schema)).values.head.getAttributeNames == List("first", "second") @@ -275,5 +219,4 @@ class ProjectionOpDescSpec extends AnyFlatSpec with BeforeAndAfter { assert(code.startsWith("raise ValueError(")) assert(code.contains("Please select at least one attribute to project.")) } - } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/projection/ProjectionOpExecSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/projection/ProjectionOpExecSpec.scala index 1d8ee9717bb..76e00eee657 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/projection/ProjectionOpExecSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/projection/ProjectionOpExecSpec.scala @@ -24,6 +24,7 @@ import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.BeforeAndAfter import org.scalatest.flatspec.AnyFlatSpec + class ProjectionOpExecSpec extends AnyFlatSpec with BeforeAndAfter { val tupleSchema: Schema = Schema() .add(new Attribute("field1", AttributeType.STRING)) @@ -34,65 +35,57 @@ class ProjectionOpExecSpec extends AnyFlatSpec with BeforeAndAfter { .builder(tupleSchema) .add(new Attribute("field1", AttributeType.STRING), "hello") .add(new Attribute("field2", AttributeType.INTEGER), 1) - .add( - new Attribute("field3", AttributeType.BOOLEAN), - true - ) + .add(new Attribute("field3", AttributeType.BOOLEAN), true) .build() val opDesc: ProjectionOpDesc = new ProjectionOpDesc() + before { + opDesc.isDrop = false + opDesc.attributes = List() + } + + private def units(pairs: (String, String)*): List[AttributeUnit] = + pairs.map { case (orig, alias) => new AttributeUnit(orig, alias) }.toList + + private def execProjection(isDrop: Boolean = false, openExec: Boolean = true)( + pairs: (String, String)* + ): TupleLike = { + opDesc.isDrop = isDrop + opDesc.attributes = units(pairs: _*) + val projectionOpExec = new ProjectionOpExec(objectMapper.writeValueAsString(opDesc)) + if (openExec) projectionOpExec.open() + projectionOpExec.processTuple(tuple, 0).next() + } + + private def enforce(outputSchema: Schema)(pairs: (String, String)*): Tuple = + execProjection()(pairs: _*).asInstanceOf[SchemaEnforceable].enforceSchema(outputSchema) + + private def dropOutput(pairs: (String, String)*): MapTupleLike = + execProjection(isDrop = true)(pairs: _*).asInstanceOf[MapTupleLike] + it should "open" in { - opDesc.attributes = List( - new AttributeUnit("field2", "f2"), - new AttributeUnit("field1", "f1") - ) + opDesc.attributes = units("field2" -> "f2", "field1" -> "f1") val projectionOpExec = new ProjectionOpExec(objectMapper.writeValueAsString(opDesc)) projectionOpExec.open() - } it should "process Tuple" in { - opDesc.attributes = List( - new AttributeUnit("field2", "f2"), - new AttributeUnit("field1", "f1") - ) val outputSchema = Schema() .add(new Attribute("f1", AttributeType.STRING)) .add(new Attribute("f2", AttributeType.INTEGER)) - - val projectionOpExec = new ProjectionOpExec(objectMapper.writeValueAsString(opDesc)) - projectionOpExec.open() - - val outputTuple = - projectionOpExec - .processTuple(tuple, 0) - .next() - .asInstanceOf[SchemaEnforceable] - .enforceSchema(outputSchema) + val outputTuple = enforce(outputSchema)("field2" -> "f2", "field1" -> "f1") assert(outputTuple.length == 2) assert(outputTuple.getField("f1").asInstanceOf[String] == "hello") assert(outputTuple.getField("f2").asInstanceOf[Int] == 1) assert(outputTuple.getField[String](0) == "hello") assert(outputTuple.getField[Int](1) == 1) } + it should "process Tuple with different order" in { - opDesc.attributes = List( - new AttributeUnit("field3", "f3"), - new AttributeUnit("field1", "f1") - ) val outputSchema = Schema() .add(new Attribute("f3", AttributeType.BOOLEAN)) .add(new Attribute("f1", AttributeType.STRING)) - - val projectionOpExec = new ProjectionOpExec(objectMapper.writeValueAsString(opDesc)) - projectionOpExec.open() - - val outputTuple = - projectionOpExec - .processTuple(tuple, 0) - .next() - .asInstanceOf[SchemaEnforceable] - .enforceSchema(outputSchema) + val outputTuple = enforce(outputSchema)("field3" -> "f3", "field1" -> "f1") assert(outputTuple.length == 2) assert(outputTuple.getField("f3").asInstanceOf[Boolean]) assert(outputTuple.getField("f1").asInstanceOf[String] == "hello") @@ -101,53 +94,28 @@ class ProjectionOpExecSpec extends AnyFlatSpec with BeforeAndAfter { } it should "meException on non-existing fields" in { - opDesc.attributes = List( - new AttributeUnit("field---5", "f5"), - new AttributeUnit("field---6", "f6") - ) - val projectionOpExec = new ProjectionOpExec(objectMapper.writeValueAsString(opDesc)) assertThrows[RuntimeException] { - projectionOpExec.processTuple(tuple, 0).next() + execProjection(openExec = false)("field---5" -> "f5", "field---6" -> "f6") } } it should "raise IllegalArgumentException on empty attributes" in { - opDesc.attributes = List() - val projectionOpExec = new ProjectionOpExec(objectMapper.writeValueAsString(opDesc)) assertThrows[IllegalArgumentException] { - projectionOpExec.processTuple(tuple, 0).next() + execProjection(openExec = false)() } } it should "raise RuntimeException on duplicate alias" in { - opDesc.attributes = List( - new AttributeUnit("field1", "f"), - new AttributeUnit("field2", "f") - ) - val projectionOpExec = new ProjectionOpExec(objectMapper.writeValueAsString(opDesc)) assertThrows[RuntimeException] { - projectionOpExec.processTuple(tuple, 0).next() + execProjection(openExec = false)("field1" -> "f", "field2" -> "f") } } it should "allow empty alias" in { - opDesc.attributes = List( - new AttributeUnit("field2", "f2"), - new AttributeUnit("field1", "") - ) val outputSchema = Schema() .add(new Attribute("field1", AttributeType.STRING)) .add(new Attribute("f2", AttributeType.INTEGER)) - - val projectionOpExec = new ProjectionOpExec(objectMapper.writeValueAsString(opDesc)) - projectionOpExec.open() - - val outputTuple = - projectionOpExec - .processTuple(tuple, 0) - .next() - .asInstanceOf[SchemaEnforceable] - .enforceSchema(outputSchema) + val outputTuple = enforce(outputSchema)("field2" -> "f2", "field1" -> "") assert(outputTuple.length == 2) assert(outputTuple.getField("field1").asInstanceOf[String] == "hello") assert(outputTuple.getField("f2").asInstanceOf[Int] == 1) @@ -156,18 +124,10 @@ class ProjectionOpExecSpec extends AnyFlatSpec with BeforeAndAfter { } it should "drop a single attribute and keep the rest under their original names" in { - opDesc.isDrop = true - opDesc.attributes = List( - new AttributeUnit("field2", "") - ) val outputSchema = Schema() .add(new Attribute("field1", AttributeType.STRING)) .add(new Attribute("field3", AttributeType.BOOLEAN)) - - val projectionOpExec = new ProjectionOpExec(objectMapper.writeValueAsString(opDesc)) - projectionOpExec.open() - - val output = projectionOpExec.processTuple(tuple, 0).next().asInstanceOf[MapTupleLike] + val output = dropOutput("field2" -> "") assert(output.fieldMappings.keySet == Set("field1", "field3")) val outputTuple = output.enforceSchema(outputSchema) @@ -177,69 +137,32 @@ class ProjectionOpExecSpec extends AnyFlatSpec with BeforeAndAfter { } it should "drop multiple attributes" in { - opDesc.isDrop = true - opDesc.attributes = List( - new AttributeUnit("field1", ""), - new AttributeUnit("field3", "") - ) - val projectionOpExec = new ProjectionOpExec(objectMapper.writeValueAsString(opDesc)) - projectionOpExec.open() - - val output = projectionOpExec.processTuple(tuple, 0).next().asInstanceOf[MapTupleLike] + val output = dropOutput("field1" -> "", "field3" -> "") assert(output.fieldMappings == Map("field2" -> 1)) } it should "ignore aliases in drop mode" in { - opDesc.isDrop = true - opDesc.attributes = List( - new AttributeUnit("field2", "renamed") - ) - val projectionOpExec = new ProjectionOpExec(objectMapper.writeValueAsString(opDesc)) - projectionOpExec.open() - - val output = projectionOpExec.processTuple(tuple, 0).next().asInstanceOf[MapTupleLike] + val output = dropOutput("field2" -> "renamed") assert(output.fieldMappings.keySet == Set("field1", "field3")) assert(!output.fieldMappings.contains("renamed")) } it should "silently ignore dropping a non-existent attribute" in { - opDesc.isDrop = true - opDesc.attributes = List( - new AttributeUnit("field---5", "f5") - ) - val projectionOpExec = new ProjectionOpExec(objectMapper.writeValueAsString(opDesc)) - projectionOpExec.open() - - val output = projectionOpExec.processTuple(tuple, 0).next().asInstanceOf[MapTupleLike] + val output = dropOutput("field---5" -> "f5") assert(output.fieldMappings == Map("field1" -> "hello", "field2" -> 1, "field3" -> true)) } it should "emit an empty tuple when dropping every attribute" in { - opDesc.isDrop = true - opDesc.attributes = List( - new AttributeUnit("field1", ""), - new AttributeUnit("field2", ""), - new AttributeUnit("field3", "") - ) - val projectionOpExec = new ProjectionOpExec(objectMapper.writeValueAsString(opDesc)) - projectionOpExec.open() - - val output = projectionOpExec.processTuple(tuple, 0).next().asInstanceOf[MapTupleLike] + val output = dropOutput("field1" -> "", "field2" -> "", "field3" -> "") assert(output.fieldMappings.isEmpty) } it should "emit exactly the attributes the descriptor derives for the same drop config" in { opDesc.isDrop = true - opDesc.attributes = List( - new AttributeUnit("field2", "") - ) + opDesc.attributes = units("field2" -> "") val derivedSchema = opDesc.getExternalOutputSchemas(Map(PortIdentity() -> tupleSchema)).values.head - - val projectionOpExec = new ProjectionOpExec(objectMapper.writeValueAsString(opDesc)) - projectionOpExec.open() - - val output = projectionOpExec.processTuple(tuple, 0).next().asInstanceOf[MapTupleLike] + val output = dropOutput("field2" -> "") assert(output.fieldMappings.keySet == derivedSchema.getAttributeNames.toSet) val outputTuple = output.enforceSchema(derivedSchema) @@ -250,27 +173,12 @@ class ProjectionOpExecSpec extends AnyFlatSpec with BeforeAndAfter { it should "match drop names case-insensitively" in { // Matches the descriptor, whose Schema.remove lowercases both sides. - opDesc.isDrop = true - opDesc.attributes = List( - new AttributeUnit("FIELD2", "") - ) - val projectionOpExec = new ProjectionOpExec(objectMapper.writeValueAsString(opDesc)) - projectionOpExec.open() - - val output = projectionOpExec.processTuple(tuple, 0).next().asInstanceOf[MapTupleLike] + val output = dropOutput("FIELD2" -> "") assert(output.fieldMappings == Map("field1" -> "hello", "field3" -> true)) } it should "tolerate duplicate entries in the drop list" in { - opDesc.isDrop = true - opDesc.attributes = List( - new AttributeUnit("field2", ""), - new AttributeUnit("field2", "") - ) - val projectionOpExec = new ProjectionOpExec(objectMapper.writeValueAsString(opDesc)) - projectionOpExec.open() - - val output = projectionOpExec.processTuple(tuple, 0).next().asInstanceOf[MapTupleLike] + val output = dropOutput("field2" -> "", "field2" -> "") assert(output.fieldMappings == Map("field1" -> "hello", "field3" -> true)) } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/set/BinarySetOpExecBehaviors.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/set/BinarySetOpExecBehaviors.scala new file mode 100644 index 00000000000..1a7a5da2db4 --- /dev/null +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/set/BinarySetOpExecBehaviors.scala @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.operator.set + +import org.apache.texera.amber.core.executor.OperatorExecutor +import org.apache.texera.amber.core.tuple._ +import org.scalatest.BeforeAndAfter +import org.scalatest.flatspec.AnyFlatSpecLike + +/** Shared fixture and behavioral cases for two-port set-difference operator executors. */ +trait BinarySetOpExecBehaviors extends AnyFlatSpecLike with BeforeAndAfter { + + protected val input1: Int = 0 + protected val input2: Int = 1 + protected var opExec: OperatorExecutor = _ + protected var counter: Int = 0 + + protected val schema: Schema = Schema() + .add(new Attribute("field1", AttributeType.STRING)) + .add(new Attribute("field2", AttributeType.INTEGER)) + .add(new Attribute("field3", AttributeType.BOOLEAN)) + + protected def newExecutor(): OperatorExecutor + + protected def tuple(): Tuple = { + counter += 1 + Tuple + .builder(schema) + .addSequentially(Array("hello", Int.box(counter), Boolean.box(true))) + .build() + } + + before { + opExec = newExecutor() + } + + protected def binarySetOpExecBehaviors( + includeCommon: Boolean, + includeLeftOnly: Boolean, + includeRightOnly: Boolean + ): Unit = { + def whenLeftOnly(tuples: Seq[Tuple]): Set[TupleLike] = + if (includeLeftOnly) tuples.toSet else Set.empty + + it should "open" in { + opExec.open() + } + + it should "work with basic two input streams with no duplicates" in { + opExec.open() + counter = 0 + val commonTuples = (1 to 10).map(_ => tuple()).toList + + (0 to 7).foreach(i => opExec.processTuple(commonTuples(i), input1)) + assert(opExec.onFinish(input1).isEmpty) + + (5 to 9).foreach(i => opExec.processTuple(commonTuples(i), input2)) + + val expected = + (if (includeCommon) commonTuples.slice(5, 8).toSet else Set.empty[TupleLike]) ++ + (if (includeLeftOnly) commonTuples.slice(0, 5).toSet else Set.empty[TupleLike]) ++ + (if (includeRightOnly) commonTuples.slice(8, 10).toSet else Set.empty[TupleLike]) + assert(opExec.onFinish(input2).toSet == expected) + opExec.close() + } + + it should "work with one empty input upstream after a data stream" in { + opExec.open() + counter = 0 + val commonTuples = (1 to 10).map(_ => tuple()).toList + + (0 to 9).foreach(i => opExec.processTuple(commonTuples(i), input1)) + assert(opExec.onFinish(input1).isEmpty) + assert(opExec.onFinish(input2).toSet == whenLeftOnly(commonTuples)) + opExec.close() + } + + it should "work with one empty input upstream after a data stream - other order" in { + opExec.open() + counter = 0 + val commonTuples = (1 to 10).map(_ => tuple()).toList + + (0 to 9).foreach(i => opExec.processTuple(commonTuples(i), input2)) + assert(opExec.onFinish(input2).isEmpty) + + val expected = if (includeRightOnly) commonTuples.toSet else Set.empty[TupleLike] + assert(opExec.onFinish(input1).toSet == expected) + opExec.close() + } + + it should "work with one empty input upstream before a data stream" in { + opExec.open() + counter = 0 + val commonTuples = (1 to 10).map(_ => tuple()).toList + + assert(opExec.onFinish(input2).isEmpty) + (0 to 9).foreach(i => opExec.processTuple(commonTuples(i), input1)) + assert(opExec.onFinish(input1).toSet == whenLeftOnly(commonTuples)) + opExec.close() + } + + it should "work with one empty input upstream during a data stream" in { + opExec.open() + counter = 0 + val commonTuples = (1 to 10).map(_ => tuple()).toList + + (0 to 5).foreach(i => opExec.processTuple(commonTuples(i), input1)) + assert(opExec.onFinish(input2).isEmpty) + (6 to 9).foreach(i => opExec.processTuple(commonTuples(i), input1)) + assert(opExec.onFinish(input1).toSet == whenLeftOnly(commonTuples)) + opExec.close() + } + + it should "work with two empty input upstreams" in { + opExec.open() + assert(opExec.onFinish(input1).isEmpty) + assert(opExec.onFinish(input2).isEmpty) + opExec.close() + } + } +} diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/set/difference/DifferenceOpExecSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/set/difference/DifferenceOpExecSpec.scala index 060130a2a15..440af3ed2dd 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/set/difference/DifferenceOpExecSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/set/difference/DifferenceOpExecSpec.scala @@ -19,135 +19,19 @@ package org.apache.texera.amber.operator.set.difference -import org.apache.texera.amber.core.tuple._ -import org.scalatest.BeforeAndAfter +import org.apache.texera.amber.core.executor.OperatorExecutor +import org.apache.texera.amber.operator.set.BinarySetOpExecBehaviors import org.scalatest.flatspec.AnyFlatSpec -class DifferenceOpExecSpec extends AnyFlatSpec with BeforeAndAfter { - var input1: Int = 0 - var input2: Int = 1 - var opExec: DifferenceOpExec = _ - var counter: Int = 0 - val schema: Schema = Schema() - .add(new Attribute("field1", AttributeType.STRING)) - .add(new Attribute("field2", AttributeType.INTEGER)) - .add(new Attribute("field3", AttributeType.BOOLEAN)) - def tuple(): Tuple = { - counter += 1 +class DifferenceOpExecSpec extends AnyFlatSpec with BinarySetOpExecBehaviors { - Tuple - .builder(schema) - .addSequentially(Array("hello", Int.box(counter), Boolean.box(true))) - .build() - } - - before { - opExec = new DifferenceOpExec() - } - - it should "open" in { - - opExec.open() - - } - - it should "work with basic two input streams with no duplicates" in { - opExec.open() - counter = 0 - val commonTuples = (1 to 10).map(_ => tuple()).toList - - (0 to 7).map(i => { - opExec.processTuple(commonTuples(i), input1) - }) - assert(opExec.onFinish(input1).isEmpty) - - (5 to 9).map(i => { - opExec.processTuple(commonTuples(i), input2) - }) + override protected def newExecutor(): OperatorExecutor = new DifferenceOpExec() - val outputTuples: Set[TupleLike] = - opExec.onFinish(input2).toSet - assert( - outputTuples.equals(commonTuples.slice(0, 5).toSet) - ) - - opExec.close() - } - - it should "work with one empty input upstream after a data stream" in { - opExec.open() - counter = 0 - val commonTuples = (1 to 10).map(_ => tuple()).toList - - (0 to 9).map(i => { - opExec.processTuple(commonTuples(i), input1) - }) - assert(opExec.onFinish(input1).isEmpty) - - val outputTuples: Set[TupleLike] = - opExec.onFinish(input2).toSet - assert(outputTuples.equals(commonTuples.toSet)) - opExec.close() - } - - it should "work with one empty input upstream after a data stream - other order" in { - opExec.open() - counter = 0 - val commonTuples = (1 to 10).map(_ => tuple()).toList - - (0 to 9).map(i => { - opExec.processTuple(commonTuples(i), input2) - }) - assert(opExec.onFinish(input2).isEmpty) - - val outputTuples: Set[TupleLike] = - opExec.onFinish(input1).toSet - assert(outputTuples.isEmpty) - opExec.close() - } - - it should "work with one empty input upstream before a data stream" in { - opExec.open() - counter = 0 - val commonTuples = (1 to 10).map(_ => tuple()).toList - - assert(opExec.onFinish(input2).isEmpty) - (0 to 9).map(i => { - opExec.processTuple(commonTuples(i), input1) - }) - - val outputTuples: Set[TupleLike] = - opExec.onFinish(input1).toSet - assert(outputTuples.equals(commonTuples.toSet)) - opExec.close() - } - - it should "work with one empty input upstream during a data stream" in { - opExec.open() - counter = 0 - val commonTuples = (1 to 10).map(_ => tuple()).toList - - (0 to 5).map(i => { - opExec.processTuple(commonTuples(i), input1) - }) - assert(opExec.onFinish(input2).isEmpty) - (6 to 9).map(i => { - opExec.processTuple(commonTuples(i), input1) - }) - - val outputTuples: Set[TupleLike] = - opExec.onFinish(input1).toSet - assert(outputTuples.equals(commonTuples.toSet)) - opExec.close() - } - - it should "work with two empty input upstreams" in { - - opExec.open() - assert(opExec.onFinish(input1).isEmpty) - assert(opExec.onFinish(input2).isEmpty) - opExec.close() - } + binarySetOpExecBehaviors( + includeCommon = false, + includeLeftOnly = true, + includeRightOnly = false + ) it should "start each open/close cycle empty when one instance is reused" in { counter = 0 @@ -168,5 +52,4 @@ class DifferenceOpExecSpec extends AnyFlatSpec with BeforeAndAfter { assert(opExec.onFinish(input2).toSet == Set(second(0))) opExec.close() } - } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/set/intersect/IntersectOpExecSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/set/intersect/IntersectOpExecSpec.scala index 72d0d9c577c..2032f332cd6 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/set/intersect/IntersectOpExecSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/set/intersect/IntersectOpExecSpec.scala @@ -19,21 +19,15 @@ package org.apache.texera.amber.operator.set.intersect -import org.apache.texera.amber.core.tuple._ +import org.apache.texera.amber.core.executor.OperatorExecutor import org.apache.texera.amber.core.virtualidentity.{OperatorIdentity, PhysicalOpIdentity} import org.apache.texera.amber.core.workflow.{PhysicalLink, PortIdentity} -import org.scalatest.BeforeAndAfter +import org.apache.texera.amber.operator.set.BinarySetOpExecBehaviors import org.scalatest.flatspec.AnyFlatSpec -import scala.util.Random -class IntersectOpExecSpec extends AnyFlatSpec with BeforeAndAfter { - var opExec: IntersectOpExec = _ - var counter: Int = 0 +class IntersectOpExecSpec extends AnyFlatSpec with BinarySetOpExecBehaviors { - val tupleSchema: Schema = Schema() - .add(new Attribute("field1", AttributeType.STRING)) - .add(new Attribute("field2", AttributeType.INTEGER)) - .add(new Attribute("field3", AttributeType.BOOLEAN)) + override protected def newExecutor(): OperatorExecutor = new IntersectOpExec() def physicalOpId(): PhysicalOpIdentity = { counter += 1 @@ -48,131 +42,9 @@ class IntersectOpExecSpec extends AnyFlatSpec with BeforeAndAfter { toPortId = PortIdentity() ) - def tuple(): Tuple = { - counter += 1 - Tuple - .builder(tupleSchema) - .add(new Attribute("field1", AttributeType.STRING), "hello") - .add(new Attribute("field2", AttributeType.INTEGER), counter) - .add( - new Attribute("field3", AttributeType.BOOLEAN), - true - ) - .build() - } - - before { - opExec = new IntersectOpExec() - } - - it should "open" in { - - opExec.open() - - } - - it should "work with basic two input streams with no duplicates" in { - val input1 = 0 - val input2 = 1 - opExec.open() - counter = 0 - val commonTuples = (1 to 10).map(_ => tuple()).toList - - (0 to 7).map(i => { - opExec.processTuple(commonTuples(i), input1) - }) - assert(opExec.onFinish(input1).isEmpty) - - (5 to 9).map(i => { - opExec.processTuple(commonTuples(i), input2) - }) - - val outputTuples: Set[TupleLike] = - opExec.onFinish(input2).toSet - assert(outputTuples.equals(commonTuples.slice(5, 8).toSet)) - - opExec.close() - } - - it should "work with one empty input upstream after a data stream" in { - val input0 = 0 - val input1 = 1 - opExec.open() - counter = 0 - val commonTuples = (1 to 10).map(_ => tuple()).toList - - (1 to 100).map(_ => { - opExec.processTuple(tuple(), input0) - opExec.processTuple(commonTuples(Random.nextInt(commonTuples.size)), input0) - }) - assert(opExec.onFinish(input0).isEmpty) - - assert(opExec.onFinish(input1).isEmpty) - opExec.close() - } - - it should "work with one empty input upstream after a data stream - other order" in { - val input1 = 0 - val input2 = 1 - opExec.open() - counter = 0 - val commonTuples = (1 to 10).map(_ => tuple()).toList - - (1 to 100).map(_ => { - opExec.processTuple(tuple(), input1) - opExec.processTuple(commonTuples(Random.nextInt(commonTuples.size)), input1) - }) - assert(opExec.onFinish(input2).isEmpty) - - assert(opExec.onFinish(input1).isEmpty) - opExec.close() - } - - it should "work with one empty input upstream before a data stream" in { - val input1 = 0 - val input2 = 1 - opExec.open() - counter = 0 - val commonTuples = (1 to 10).map(_ => tuple()).toList - - assert(opExec.onFinish(input2).isEmpty) - (1 to 100).map(_ => { - opExec.processTuple(tuple(), input1) - opExec.processTuple(commonTuples(Random.nextInt(commonTuples.size)), input1) - }) - assert(opExec.onFinish(input1).isEmpty) - - opExec.close() - } - - it should "work with one empty input upstream during a data stream" in { - val input1 = 0 - val input2 = 1 - opExec.open() - counter = 0 - val commonTuples = (1 to 10).map(_ => tuple()).toList - - (1 to 100).map(_ => { - opExec.processTuple(tuple(), input1) - opExec.processTuple(commonTuples(Random.nextInt(commonTuples.size)), input1) - }) - assert(opExec.onFinish(input2).isEmpty) - - (1 to 100).map(_ => { - opExec.processTuple(tuple(), input1) - opExec.processTuple(commonTuples(Random.nextInt(commonTuples.size)), input1) - }) - assert(opExec.onFinish(input1).isEmpty) - - opExec.close() - } - - it should "work with two empty input upstreams" in { - - opExec.open() - assert(opExec.onFinish(0).isEmpty) - assert(opExec.onFinish(1).isEmpty) - opExec.close() - } - + it should behave like binarySetOpExecBehaviors( + includeCommon = true, + includeLeftOnly = false, + includeRightOnly = false + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/set/symmetricDifference/SymmetricDifferenceOpExecSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/set/symmetricDifference/SymmetricDifferenceOpExecSpec.scala index 290fa16a5bb..43e956bfda6 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/set/symmetricDifference/SymmetricDifferenceOpExecSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/set/symmetricDifference/SymmetricDifferenceOpExecSpec.scala @@ -19,154 +19,17 @@ package org.apache.texera.amber.operator.set.symmetricDifference -import org.apache.texera.amber.core.tuple._ -import org.scalatest.BeforeAndAfter +import org.apache.texera.amber.core.executor.OperatorExecutor +import org.apache.texera.amber.operator.set.BinarySetOpExecBehaviors import org.scalatest.flatspec.AnyFlatSpec -class SymmetricDifferenceOpExecSpec extends AnyFlatSpec with BeforeAndAfter { - var opExec: SymmetricDifferenceOpExec = _ - var counter: Int = 0 - val schema: Schema = Schema() - .add(new Attribute("field1", AttributeType.STRING)) - .add(new Attribute("field2", AttributeType.INTEGER)) - .add(new Attribute("field3", AttributeType.BOOLEAN)) - def tuple(): Tuple = { - counter += 1 +class SymmetricDifferenceOpExecSpec extends AnyFlatSpec with BinarySetOpExecBehaviors { - Tuple - .builder(schema) - .addSequentially(Array("hello", Int.box(counter), Boolean.box(true))) - .build() - } - - before { - opExec = new SymmetricDifferenceOpExec() - } - - it should "open" in { - - opExec.open() - - } - - it should "work with basic two input streams with no duplicates" in { - val input1 = 0 - val input2 = 1 - opExec.open() - counter = 0 - val commonTuples = (1 to 10).map(_ => tuple()).toList - - (0 to 7).map(i => { - opExec.processTuple(commonTuples(i), input1) - }) - assert(opExec.onFinish(input1).isEmpty) - - (5 to 9).map(i => { - opExec.processTuple(commonTuples(i), input2) - }) - - val outputTuples: Set[TupleLike] = - opExec.onFinish(input2).toSet - assert( - outputTuples.equals(commonTuples.slice(0, 5).toSet.union(commonTuples.slice(8, 10).toSet)) - ) - - opExec.close() - } - - it should "work with one empty input upstream after a data stream" in { - val input1 = 0 - val input2 = 1 - opExec.open() - counter = 0 - val commonTuples = (1 to 10).map(_ => tuple()).toList - - (0 to 9).map(i => { - opExec.processTuple(commonTuples(i), input1) - }) - assert(opExec.onFinish(input1).isEmpty) - - val outputTuples: Set[Tuple] = - opExec - .onFinish(input2) - .map(tupleLike => tupleLike.asInstanceOf[SchemaEnforceable].enforceSchema(schema)) - .toSet - assert(outputTuples.equals(commonTuples.toSet)) - opExec.close() - } - - it should "work with one empty input upstream after a data stream - other order" in { - val input1 = 0 - val input2 = 1 - opExec.open() - counter = 0 - val commonTuples = (1 to 10).map(_ => tuple()).toList - - (0 to 9).map(i => { - opExec.processTuple(commonTuples(i), input2) - }) - assert(opExec.onFinish(input2).isEmpty) - - val outputTuples: Set[Tuple] = - opExec - .onFinish(input1) - .map(tupleLike => tupleLike.asInstanceOf[SchemaEnforceable].enforceSchema(schema)) - .toSet - assert(outputTuples.equals(commonTuples.toSet)) - opExec.close() - } - - it should "work with one empty input upstream before a data stream" in { - val input1 = 0 - val input2 = 1 - opExec.open() - counter = 0 - val commonTuples = (1 to 10).map(_ => tuple()).toList - - assert(opExec.onFinish(input2).isEmpty) - (0 to 9).map(i => { - opExec.processTuple(commonTuples(i), input1) - }) - - val outputTuples: Set[Tuple] = - opExec - .onFinish(input1) - .map(tupleLike => tupleLike.asInstanceOf[SchemaEnforceable].enforceSchema(schema)) - .toSet - assert(outputTuples.equals(commonTuples.toSet)) - opExec.close() - } - - it should "work with one empty input upstream during a data stream" in { - val input1 = 0 - val input2 = 1 - opExec.open() - counter = 0 - val commonTuples = (1 to 10).map(_ => tuple()).toList - - (0 to 5).map(i => { - opExec.processTuple(commonTuples(i), input1) - }) - assert(opExec.onFinish(input2).isEmpty) - (6 to 9).map(i => { - opExec.processTuple(commonTuples(i), input1) - }) - - val outputTuples: Set[Tuple] = - opExec - .onFinish(input1) - .map(tupleLike => tupleLike.asInstanceOf[SchemaEnforceable].enforceSchema(schema)) - .toSet - assert(outputTuples.equals(commonTuples.toSet)) - opExec.close() - } - - it should "work with two empty input upstreams" in { - - opExec.open() - assert(opExec.onFinish(0).isEmpty) - assert(opExec.onFinish(1).isEmpty) - opExec.close() - } + override protected def newExecutor(): OperatorExecutor = new SymmetricDifferenceOpExec() + binarySetOpExecBehaviors( + includeCommon = false, + includeLeftOnly = true, + includeRightOnly = true + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDescCodegenSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDescCodegenSpec.scala index 489a09ad1fd..dbc889ecb11 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDescCodegenSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDescCodegenSpec.scala @@ -19,146 +19,36 @@ package org.apache.texera.amber.operator.sklearn -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -import java.nio.charset.StandardCharsets -import java.util.Base64 - -/** - * Covers the vectorizer branches of the shared train-and-evaluate codegen - * template in SklearnClassifierOpDesc. The abstract base is exercised through - * a representative concrete subclass (SklearnKNNOpDesc), matching how the 25 - * classifier operators use it. - * - * The template distinguishes all four flag combinations: countVectorizer - * gates the text-column selection and the CountVectorizer() stage, while - * tfidfTransformer independently prepends a TfidfTransformer() stage (the UI - * hides it when countVectorizer is off, but codegen still honors it). - */ -class SklearnClassifierOpDescCodegenSpec extends AnyFlatSpec with Matchers { - - // UI-provided attribute names are EncodableStrings: the encoded template - // renders them as a runtime base64-decode expression, not a Python literal. - private def decodeExpr(value: String): String = - PythonTemplateBuilder.wrapWithPythonDecoderExpr( - Base64.getEncoder.encodeToString(value.getBytes(StandardCharsets.UTF_8)) - ) - - // Collapse space runs so assertions on the make_pipeline call are not - // coupled to the exact spacing the empty branches leave behind. - private def normalized(code: String): String = code.replaceAll(" +", " ") - - private def descriptor( - countVectorizer: Boolean = false, - tfidfTransformer: Boolean = false - ): SklearnKNNOpDesc = { - val d = new SklearnKNNOpDesc - d.target = "label" - d.text = List("docs") - d.countVectorizer = countVectorizer - d.tfidfTransformer = tfidfTransformer - d - } - - "SklearnClassifierOpDesc.generatePythonCode" should - "generate a plain feature pipeline when both vectorizer flags are off" in { - val code = descriptor().generatePythonCode() - code should include("from sklearn.neighbors import KNeighborsClassifier") - code should include(s"Y = table[${decodeExpr("label")}]") - code should include(s"X = table.drop(${decodeExpr("label")}, axis=1)") - // Feature-column path: every column an estimator can fit is kept, the rest are - // named on the console, and the text attribute is never read. - code should not include "ColumnTransformer(" - code should include("""_fittable = X.select_dtypes(include=["number", "bool"])""") - code should include("""print("Ignoring columns an estimator cannot fit:", _ignored)""") - code should include("X = _fittable") - code should not include decodeExpr("docs") - normalized(code) should include( - "self.model = make_pipeline( KNeighborsClassifier()).fit(X, Y)" - ) - code should include("predictions = self.model.predict(X)") - code should not include "CountVectorizer()" - code should not include "TfidfTransformer()" - } - - it should "select the text column and prepend CountVectorizer when countVectorizer is on" in { - val code = descriptor(countVectorizer = true).generatePythonCode() - // ColumnTransformer selects the columns itself, so X stays the whole frame, and - // narrowing it to the fittable columns would drop the text ones it reads. - code should not include "_fittable" - normalized(code) should include( - s"""self.model = make_pipeline(ColumnTransformer([("text0", CountVectorizer(), ${decodeExpr( - "docs" - )})]), KNeighborsClassifier()).fit(X, Y)""" - ) - code should not include "TfidfTransformer()" - } - - // One CountVectorizer per column: it reads a flat sequence of documents, so - // several columns handed to one would be read as a document each. The steps are - // named by position, keeping a column named with a double underscore away from - // the separator get_feature_names_out uses. - it should "give each named column its own CountVectorizer" in { - val d = descriptor(countVectorizer = true) - d.text = List("title", "body") - normalized(d.generatePythonCode()) should include( - s"""self.model = make_pipeline(ColumnTransformer([("text0", CountVectorizer(), ${decodeExpr( - "title" - )}), ("text1", CountVectorizer(), ${decodeExpr( - "body" - )})]), KNeighborsClassifier()).fit(X, Y)""" - ) - } - - it should "chain CountVectorizer before TfidfTransformer when both flags are on" in { - val code = - descriptor(countVectorizer = true, tfidfTransformer = true).generatePythonCode() - normalized(code) should include( - s"""self.model = make_pipeline(ColumnTransformer([("text0", CountVectorizer(), ${decodeExpr( - "docs" - )})]), TfidfTransformer(), KNeighborsClassifier()).fit(X, Y)""" - ) - } - - it should "prepend only TfidfTransformer and keep all features when tfidfTransformer is on alone" in { - val code = descriptor(tfidfTransformer = true).generatePythonCode() - // Without countVectorizer there is no text-column selection. - code should not include "ColumnTransformer(" - code should include("X = _fittable") - code should not include decodeExpr("docs") - normalized(code) should include( - "self.model = make_pipeline( TfidfTransformer(), KNeighborsClassifier()).fit(X, Y)" - ) - code should not include "CountVectorizer()" - } - - // --- the base's own model-identity defaults -------------------------------- +/** Covers the vectorizer branches of SklearnClassifierOpDesc and its empty placeholder hooks. */ +class SklearnClassifierOpDescCodegenSpec + extends AnyFlatSpec + with SklearnVectorizerCodegenBehaviors { + + vectorizerCodegenBehavior( + "SklearnClassifierOpDesc.generatePythonCode", + pipelinePrefix = "self.model = ", + makeDescriptor = { (cv, tfidf, cols) => + val d = new SklearnKNNOpDesc + d.target = "label" + d.text = cols + d.countVectorizer = cv + d.tfidfTransformer = tfidf + d + } + ) // Declared inside the spec class on purpose: PythonClassgraphScanner drops - // non-static enclosed classes, which is what keeps SklearnOpDescRegistrySpec's - // and PythonCodeRawInvalidTextSpec's classpath scans from treating this stub as - // a shipped operator. A top-level subclass here would break both suites. + // non-static enclosed classes, which keeps SklearnOpDescRegistrySpec and + // PythonCodeRawInvalidTextSpec from treating this stub as a shipped operator. private class BareClassifier extends SklearnClassifierOpDesc - // Overrides only the model name, so the two hooks hold different values. That - // is what separates the base's own getImportStatements body from a body that - // merely forwards to the other hook — on BareClassifier alone both return "", - // which makes an exchange between them invisible. private class NamedOnlyClassifier extends SklearnClassifierOpDesc { override def getUserFriendlyModelName = "ProbeModel" } "SklearnClassifierOpDesc" should "leave both model-identity hooks blank as base placeholders" in { - // NOT a claim that "" is the intended design. SklearnModelOpDesc declares both - // hooks abstract, and all of the shipped classifiers override them; these two - // bodies are placeholders that satisfy the abstract contract for the family. - // What is pinned is therefore what a subclass that forgets to override - // actually ships — a nameless operator whose generated pipeline stage comes - // out empty — not a default anyone should rely on. Leaving these two hooks - // abstract on SklearnClassifierOpDesc is filed as a follow-up rather than - // asserted here. val bare = new BareClassifier bare.getImportStatements shouldBe "" bare.getUserFriendlyModelName shouldBe "" diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnVectorizerCodegenBehaviors.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnVectorizerCodegenBehaviors.scala new file mode 100644 index 00000000000..3d2f740f487 --- /dev/null +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnVectorizerCodegenBehaviors.scala @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.operator.sklearn + +import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.nio.charset.StandardCharsets +import java.util.Base64 + +/** Shared vectorizer codegen contract for both classifier and training sklearn templates. */ +trait SklearnVectorizerCodegenBehaviors extends Matchers { self: AnyFlatSpec => + + // UI-provided attribute names are EncodableStrings: the encoded template + // renders them as a runtime base64-decode expression, not a Python literal. + protected def decodeExpr(value: String): String = + PythonTemplateBuilder.wrapWithPythonDecoderExpr( + Base64.getEncoder.encodeToString(value.getBytes(StandardCharsets.UTF_8)) + ) + + // Collapse space runs so assertions on the make_pipeline call are not + // coupled to the exact spacing the empty branches leave behind. + protected def normalized(code: String): String = code.replaceAll(" +", " ") + + protected def vectorizerCodegenBehavior( + subject: String, + pipelinePrefix: String, + makeDescriptor: (Boolean, Boolean, List[String]) => PythonOperatorDescriptor + ): Unit = { + subject should "generate a plain feature pipeline when both vectorizer flags are off" in { + val code = makeDescriptor(false, false, List("docs")).generatePythonCode() + code should include("from sklearn.neighbors import KNeighborsClassifier") + code should include(s"Y = table[${decodeExpr("label")}]") + code should include(s"X = table.drop(${decodeExpr("label")}, axis=1)") + // Feature-column path: every column an estimator can fit is kept, the rest are + // named on the console, and the text attribute is never read. + code should not include "ColumnTransformer(" + code should include("""_fittable = X.select_dtypes(include=["number", "bool"])""") + code should include("""print("Ignoring columns an estimator cannot fit:", _ignored)""") + code should include("X = _fittable") + code should not include decodeExpr("docs") + normalized(code) should include( + s"${pipelinePrefix}make_pipeline( KNeighborsClassifier()).fit(X, Y)" + ) + if (pipelinePrefix.nonEmpty) code should include("predictions = self.model.predict(X)") + code should not include "CountVectorizer()" + code should not include "TfidfTransformer()" + } + + it should "select the text column and prepend CountVectorizer when countVectorizer is on" in { + val code = makeDescriptor(true, false, List("docs")).generatePythonCode() + // ColumnTransformer selects the columns itself, so X stays the whole frame, and + // narrowing it to the fittable columns would drop the text ones it reads. + code should not include "_fittable" + normalized(code) should include( + s"""${pipelinePrefix}make_pipeline(ColumnTransformer([("text0", CountVectorizer(), ${decodeExpr( + "docs" + )})]), KNeighborsClassifier()).fit(X, Y)""" + ) + code should not include "TfidfTransformer()" + } + + // One CountVectorizer per column: it reads a flat sequence of documents, so + // several columns handed to one would be read as a document each. The steps are + // named by position, keeping a column named with a double underscore away from + // the separator get_feature_names_out uses. + it should "give each named column its own CountVectorizer" in { + val code = makeDescriptor(true, false, List("title", "body")).generatePythonCode() + normalized(code) should include( + s"""${pipelinePrefix}make_pipeline(ColumnTransformer([("text0", CountVectorizer(), ${decodeExpr( + "title" + )}), ("text1", CountVectorizer(), ${decodeExpr( + "body" + )})]), KNeighborsClassifier()).fit(X, Y)""" + ) + } + + it should "chain CountVectorizer before TfidfTransformer when both flags are on" in { + val code = makeDescriptor(true, true, List("docs")).generatePythonCode() + normalized(code) should include( + s"""${pipelinePrefix}make_pipeline(ColumnTransformer([("text0", CountVectorizer(), ${decodeExpr( + "docs" + )})]), TfidfTransformer(), KNeighborsClassifier()).fit(X, Y)""" + ) + } + + it should "prepend only TfidfTransformer and keep all features when tfidfTransformer is on alone" in { + val code = makeDescriptor(false, true, List("docs")).generatePythonCode() + // Without countVectorizer there is no text-column selection. + code should not include "ColumnTransformer(" + code should include("X = _fittable") + code should not include decodeExpr("docs") + normalized(code) should include( + s"${pipelinePrefix}make_pipeline( TfidfTransformer(), KNeighborsClassifier()).fit(X, Y)" + ) + code should not include "CountVectorizer()" + } + } +} diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDescCodegenSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDescCodegenSpec.scala index 23301e18c7d..46d1363c329 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDescCodegenSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDescCodegenSpec.scala @@ -19,115 +19,22 @@ package org.apache.texera.amber.operator.sklearn.training -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder +import org.apache.texera.amber.operator.sklearn.SklearnVectorizerCodegenBehaviors import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -import java.nio.charset.StandardCharsets -import java.util.Base64 - -/** - * Covers the vectorizer branches of the shared training codegen template in - * SklearnTrainingOpDesc. The template is exercised through a representative - * concrete subclass (SklearnTrainingKNNOpDesc) so the generated pipeline ends - * in a real estimator, matching how the 26 training operators use the base. - * - * The template distinguishes all four flag combinations: countVectorizer - * gates the text-column selection and the CountVectorizer() stage, while - * tfidfTransformer independently prepends a TfidfTransformer() stage (the UI - * hides it when countVectorizer is off, but codegen still honors it). - */ -class SklearnTrainingOpDescCodegenSpec extends AnyFlatSpec with Matchers { - - // UI-provided attribute names are EncodableStrings: the encoded template - // renders them as a runtime base64-decode expression, not a Python literal. - private def decodeExpr(value: String): String = - PythonTemplateBuilder.wrapWithPythonDecoderExpr( - Base64.getEncoder.encodeToString(value.getBytes(StandardCharsets.UTF_8)) - ) - - // Collapse space runs so assertions on the make_pipeline call are not - // coupled to the exact spacing the empty branches leave behind. - private def normalized(code: String): String = code.replaceAll(" +", " ") - - private def descriptor( - countVectorizer: Boolean = false, - tfidfTransformer: Boolean = false - ): SklearnTrainingKNNOpDesc = { - val d = new SklearnTrainingKNNOpDesc - d.target = "label" - d.text = List("docs") - d.countVectorizer = countVectorizer - d.tfidfTransformer = tfidfTransformer - d - } - - "SklearnTrainingOpDesc.generatePythonCode" should - "generate a plain feature pipeline when both vectorizer flags are off" in { - val code = descriptor().generatePythonCode() - code should include("from sklearn.neighbors import KNeighborsClassifier") - code should include(s"Y = table[${decodeExpr("label")}]") - code should include(s"X = table.drop(${decodeExpr("label")}, axis=1)") - // Feature-column path: every column an estimator can fit is kept, the rest are - // named on the console, and the text attribute is never read. - code should not include "ColumnTransformer(" - code should include("""_fittable = X.select_dtypes(include=["number", "bool"])""") - code should include("""print("Ignoring columns an estimator cannot fit:", _ignored)""") - code should include("X = _fittable") - code should not include decodeExpr("docs") - normalized(code) should include("make_pipeline( KNeighborsClassifier()).fit(X, Y)") - code should not include "CountVectorizer()" - code should not include "TfidfTransformer()" - } - - it should "select the text column and prepend CountVectorizer when countVectorizer is on" in { - val code = descriptor(countVectorizer = true).generatePythonCode() - // ColumnTransformer selects the columns itself, so X stays the whole frame, and - // narrowing it to the fittable columns would drop the text ones it reads. - code should not include "_fittable" - normalized(code) should include( - s"""make_pipeline(ColumnTransformer([("text0", CountVectorizer(), ${decodeExpr( - "docs" - )})]), KNeighborsClassifier()).fit(X, Y)""" - ) - code should not include "TfidfTransformer()" - } - - // One CountVectorizer per column: it reads a flat sequence of documents, so - // several columns handed to one would be read as a document each. The steps are - // named by position, keeping a column named with a double underscore away from - // the separator get_feature_names_out uses. - it should "give each named column its own CountVectorizer" in { - val d = descriptor(countVectorizer = true) - d.text = List("title", "body") - normalized(d.generatePythonCode()) should include( - s"""make_pipeline(ColumnTransformer([("text0", CountVectorizer(), ${decodeExpr( - "title" - )}), ("text1", CountVectorizer(), ${decodeExpr( - "body" - )})]), KNeighborsClassifier()).fit(X, Y)""" - ) - } - - it should "chain CountVectorizer before TfidfTransformer when both flags are on" in { - val code = - descriptor(countVectorizer = true, tfidfTransformer = true).generatePythonCode() - normalized(code) should include( - s"""make_pipeline(ColumnTransformer([("text0", CountVectorizer(), ${decodeExpr( - "docs" - )})]), TfidfTransformer(), KNeighborsClassifier()).fit(X, Y)""" - ) - } - - it should "prepend only TfidfTransformer and keep all features when tfidfTransformer is on alone" in { - val code = descriptor(tfidfTransformer = true).generatePythonCode() - // Without countVectorizer there is no text-column selection. - code should not include "ColumnTransformer(" - code should include("X = _fittable") - code should not include decodeExpr("docs") - normalized(code) should include( - "make_pipeline( TfidfTransformer(), KNeighborsClassifier()).fit(X, Y)" - ) - code should not include "CountVectorizer()" - } +/** Covers the vectorizer branches of the shared training codegen template in SklearnTrainingOpDesc. */ +class SklearnTrainingOpDescCodegenSpec extends AnyFlatSpec with SklearnVectorizerCodegenBehaviors { + + vectorizerCodegenBehavior( + "SklearnTrainingOpDesc.generatePythonCode", + pipelinePrefix = "", + makeDescriptor = { (cv, tfidf, cols) => + val d = new SklearnTrainingKNNOpDesc + d.target = "label" + d.text = cols + d.countVectorizer = cv + d.tfidfTransformer = tfidf + d + } + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDescSpec.scala index 96ad9662f91..052ebb9b317 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDescSpec.scala @@ -41,6 +41,15 @@ class StableMergeSortOpDescSpec extends AnyFlatSpec with Matchers { private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) + private def newDesc(attribute: String, preference: SortPreference): StableMergeSortOpDesc = { + val unit = new SortCriteriaUnit + unit.attributeName = attribute + unit.sortPreference = preference + val desc = new StableMergeSortOpDesc + desc.keys = ListBuffer(unit) + desc + } + "StableMergeSortOpDesc.operatorInfo" should "advertise the name, Sort group, and a single blocking output" in { val info = (new StableMergeSortOpDesc).operatorInfo @@ -86,12 +95,7 @@ class StableMergeSortOpDescSpec extends AnyFlatSpec with Matchers { if (!canImport(python, "pandas", "numpy")) cancel(s"'$python' cannot import pandas") def blockFor(pref: SortPreference): String = { - val unit = new SortCriteriaUnit - unit.attributeName = "v" - unit.sortPreference = pref - val d = new StableMergeSortOpDesc - d.keys = ListBuffer(unit) - d.generateStandaloneCode().linesIterator.map(" " + _).mkString("\n") + newDesc("v", pref).generateStandaloneCode().linesIterator.map(" " + _).mkString("\n") } val driver = @@ -137,14 +141,7 @@ class StableMergeSortOpDescSpec extends AnyFlatSpec with Matchers { val values: Seq[java.lang.Double] = Seq(0.0, -0.0, Double.NaN, 1.0, null, -0.0) val schema = Schema().add("v", AttributeType.DOUBLE).add("i", AttributeType.INTEGER) - def descFor(pref: SortPreference): StableMergeSortOpDesc = { - val unit = new SortCriteriaUnit - unit.attributeName = "v" - unit.sortPreference = pref - val d = new StableMergeSortOpDesc - d.keys = ListBuffer(unit) - d - } + def descFor(pref: SortPreference): StableMergeSortOpDesc = newDesc("v", pref) val prefs = Seq(SortPreference.ASC, SortPreference.DESC) val engine = prefs.map { pref => val exec = new StableMergeSortOpExec(objectMapper.writeValueAsString(descFor(pref))) @@ -190,11 +187,7 @@ class StableMergeSortOpDescSpec extends AnyFlatSpec with Matchers { val python = configuredPython().getOrElse(cancel("No runnable python executable")) if (!canImport(python, "pandas", "numpy")) cancel(s"'$python' cannot import pandas") - val unit = new SortCriteriaUnit - unit.attributeName = "x" - unit.sortPreference = SortPreference.ASC - val desc = new StableMergeSortOpDesc - desc.keys = ListBuffer(unit) + val desc = newDesc("x", SortPreference.ASC) val driver = s"""import pandas as pd @@ -231,14 +224,7 @@ class StableMergeSortOpDescSpec extends AnyFlatSpec with Matchers { if (v == null) "NULL" else v.codePoints().toArray.map(c => f"$c%X").mkString("+") val schema = Schema().add(new Attribute("s", AttributeType.STRING)) - def descFor(pref: SortPreference): StableMergeSortOpDesc = { - val unit = new SortCriteriaUnit - unit.attributeName = "s" - unit.sortPreference = pref - val d = new StableMergeSortOpDesc - d.keys = ListBuffer(unit) - d - } + def descFor(pref: SortPreference): StableMergeSortOpDesc = newDesc("s", pref) def engine(pref: SortPreference): String = { val exec = new StableMergeSortOpExec(objectMapper.writeValueAsString(descFor(pref))) exec.open() diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpExecSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpExecSpec.scala index 1c05fe26a91..06e1cd3bfbd 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpExecSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpExecSpec.scala @@ -51,6 +51,11 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { // Helpers // =========================================================================== + private val integerSchema = schemaOf("value" -> AttributeType.INTEGER) + private val integerLabelSchema = + schemaOf("value" -> AttributeType.INTEGER, "label" -> AttributeType.STRING) + private val keyIdSchema = schemaOf("key" -> AttributeType.INTEGER, "id" -> AttributeType.STRING) + /** Build a Schema with (name, type) pairs, in-order. */ private def schemaOf(attributes: (String, AttributeType)*): Schema = { attributes.foldLeft(Schema()) { @@ -84,21 +89,18 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { criteria } - /** Convert varargs keys into the operator config buffer. */ - private def sortKeysBuffer(keys: SortCriteriaUnit*): ListBuffer[SortCriteriaUnit] = - ListBuffer(keys: _*) + private def newExecutor(keys: SortCriteriaUnit*): StableMergeSortOpExec = { + val desc = new StableMergeSortOpDesc() + desc.keys = ListBuffer(keys: _*) + new StableMergeSortOpExec(objectMapper.writeValueAsString(desc)) + } /** * Run the operator on an in-memory sequence of tuples and capture all output. * Output is only emitted at onFinish to preserve determinism. */ - private def runStableMergeSort( - schema: Schema, - tuples: Seq[Tuple] - )(configure: StableMergeSortOpDesc => Unit): List[Tuple] = { - val desc = new StableMergeSortOpDesc() - configure(desc) - val exec = new StableMergeSortOpExec(objectMapper.writeValueAsString(desc)) + private def runStableMergeSort(tuples: Seq[Tuple])(keys: SortCriteriaUnit*): List[Tuple] = { + val exec = newExecutor(keys: _*) exec.open() tuples.foreach(t => exec.processTuple(t, 0)) val result = exec.onFinish(0).map(_.asInstanceOf[Tuple]).toList @@ -128,15 +130,14 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { // =========================================================================== "StableMergeSortOpExec" should "sort integers ascending and preserve duplicate order" in { - val schema = schemaOf("value" -> AttributeType.INTEGER, "label" -> AttributeType.STRING) val tuples = List( - tupleOf(schema, "value" -> 3, "label" -> "a"), - tupleOf(schema, "value" -> 1, "label" -> "first-1"), - tupleOf(schema, "value" -> 2, "label" -> "b"), - tupleOf(schema, "value" -> 1, "label" -> "first-2"), - tupleOf(schema, "value" -> 3, "label" -> "c") + tupleOf(integerLabelSchema, "value" -> 3, "label" -> "a"), + tupleOf(integerLabelSchema, "value" -> 1, "label" -> "first-1"), + tupleOf(integerLabelSchema, "value" -> 2, "label" -> "b"), + tupleOf(integerLabelSchema, "value" -> 1, "label" -> "first-2"), + tupleOf(integerLabelSchema, "value" -> 3, "label" -> "c") ) - val result = runStableMergeSort(schema, tuples) { _.keys = sortKeysBuffer(sortKey("value")) } + val result = runStableMergeSort(tuples)(sortKey("value")) assert(result.map(_.getField[Int]("value")) == List(1, 1, 2, 3, 3)) val labelsForOnes = result.filter(_.getField[Int]("value") == 1).map(_.getField[String]("label")) @@ -144,16 +145,13 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { } it should "sort integers descending while preserving stability" in { - val schema = schemaOf("value" -> AttributeType.INTEGER, "label" -> AttributeType.STRING) val tuples = List( - tupleOf(schema, "value" -> 2, "label" -> "first"), - tupleOf(schema, "value" -> 2, "label" -> "second"), - tupleOf(schema, "value" -> 1, "label" -> "third"), - tupleOf(schema, "value" -> 3, "label" -> "fourth") + tupleOf(integerLabelSchema, "value" -> 2, "label" -> "first"), + tupleOf(integerLabelSchema, "value" -> 2, "label" -> "second"), + tupleOf(integerLabelSchema, "value" -> 1, "label" -> "third"), + tupleOf(integerLabelSchema, "value" -> 3, "label" -> "fourth") ) - val result = runStableMergeSort(schema, tuples) { - _.keys = sortKeysBuffer(sortKey("value", SortPreference.DESC)) - } + val result = runStableMergeSort(tuples)(sortKey("value", SortPreference.DESC)) assert(result.map(_.getField[Int]("value")) == List(3, 2, 2, 1)) val labelsForTwos = result.filter(_.getField[Int]("value") == 2).map(_.getField[String]("label")) @@ -168,30 +166,27 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { tupleOf(schema, "name" -> "banana"), tupleOf(schema, "name" -> "APPLE") ) - val sorted = runStableMergeSort(schema, tuples) { - _.keys = sortKeysBuffer(sortKey("name", SortPreference.ASC)) - } + val sorted = runStableMergeSort(tuples)(sortKey("name", SortPreference.ASC)) assert(sorted.map(_.getField[String]("name")) == List("APPLE", "Banana", "apple", "banana")) } it should "order ASCII strings by Java compareTo (punctuation < digits < uppercase < lowercase)" in { val schema = schemaOf("str" -> AttributeType.STRING) val tuples = List("a", "A", "0", "~", "!").map(s => tupleOf(schema, "str" -> s)) - val result = runStableMergeSort(schema, tuples) { _.keys = sortKeysBuffer(sortKey("str")) } + val result = runStableMergeSort(tuples)(sortKey("str")) assert(result.map(_.getField[String]("str")) == List("!", "0", "A", "a", "~")) } it should "sort negatives and zeros correctly" in { - val schema = schemaOf("value" -> AttributeType.INTEGER) - val tuples = List(0, -1, -10, 5, -3, 2).map(v => tupleOf(schema, "value" -> v)) - val result = runStableMergeSort(schema, tuples) { _.keys = sortKeysBuffer(sortKey("value")) } + val tuples = List(0, -1, -10, 5, -3, 2).map(v => tupleOf(integerSchema, "value" -> v)) + val result = runStableMergeSort(tuples)(sortKey("value")) assert(result.map(_.getField[Int]("value")) == List(-10, -3, -1, 0, 2, 5)) } it should "sort LONG values ascending" in { val schema = schemaOf("id" -> AttributeType.LONG) val tuples = List(5L, 1L, 3L, 9L, 0L).map(v => tupleOf(schema, "id" -> v)) - val result = runStableMergeSort(schema, tuples) { _.keys = sortKeysBuffer(sortKey("id")) } + val result = runStableMergeSort(tuples)(sortKey("id")) assert(result.map(_.getField[Long]("id")) == List(0L, 1L, 3L, 5L, 9L)) } @@ -204,9 +199,7 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { new Timestamp(base.getTime + 3000), new Timestamp(base.getTime + 2000) ).map(ts => tupleOf(schema, "timestamp" -> ts)) - val result = runStableMergeSort(schema, tuples) { - _.keys = sortKeysBuffer(sortKey("timestamp", SortPreference.ASC)) - } + val result = runStableMergeSort(tuples)(sortKey("timestamp", SortPreference.ASC)) val times = result.map(_.getField[Timestamp]("timestamp").getTime) assert(times == times.sorted) } @@ -220,9 +213,7 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { new Timestamp(base.getTime + 1000), new Timestamp(base.getTime + 2000) ).map(ts => tupleOf(schema, "timestamp" -> ts)) - val result = runStableMergeSort(schema, tuples) { - _.keys = sortKeysBuffer(sortKey("timestamp", SortPreference.DESC)) - } + val result = runStableMergeSort(tuples)(sortKey("timestamp", SortPreference.DESC)) val times = result.map(_.getField[Timestamp]("timestamp").getTime) assert(times == times.sorted(Ordering.Long.reverse)) } @@ -230,20 +221,16 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { it should "treat numeric strings as strings (lexicographic ordering)" in { val schema = schemaOf("str" -> AttributeType.STRING) val tuples = List("2", "10", "1", "11", "20").map(s => tupleOf(schema, "str" -> s)) - val result = runStableMergeSort(schema, tuples) { _.keys = sortKeysBuffer(sortKey("str")) } + val result = runStableMergeSort(tuples)(sortKey("str")) assert(result.map(_.getField[String]("str")) == List("1", "10", "11", "2", "20")) } it should "sort BOOLEAN ascending (false < true) and descending" in { val schema = schemaOf("bool" -> AttributeType.BOOLEAN) val tuples = List(true, false, true, false).map(v => tupleOf(schema, "bool" -> v)) - val asc = runStableMergeSort(schema, tuples) { - _.keys = sortKeysBuffer(sortKey("bool", SortPreference.ASC)) - } + val asc = runStableMergeSort(tuples)(sortKey("bool", SortPreference.ASC)) assert(asc.map(_.getField[Boolean]("bool")) == List(false, false, true, true)) - val desc = runStableMergeSort(schema, tuples) { - _.keys = sortKeysBuffer(sortKey("bool", SortPreference.DESC)) - } + val desc = runStableMergeSort(tuples)(sortKey("bool", SortPreference.DESC)) assert(desc.map(_.getField[Boolean]("bool")) == List(true, true, false, false)) } @@ -261,7 +248,7 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { val inputTuples = List(bytes80, bytes0000, bytesEmpty, bytesFF, bytes0001, bytes00, bytes7F) .map(arr => tupleOf(schema, "bin" -> arr)) - val sorted = runStableMergeSort(schema, inputTuples) { _.keys = sortKeysBuffer(sortKey("bin")) } + val sorted = runStableMergeSort(inputTuples)(sortKey("bin")) val actualUnsigned = sorted.map(_.getField[Array[Byte]]("bin").toSeq.map(b => b & 0xff)) val expectedUnsigned = @@ -280,9 +267,7 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { val tuples = List(Double.NaN, Double.PositiveInfinity, 1.5, -0.0, 0.0, -3.2, Double.NegativeInfinity) .map(v => tupleOf(schema, "x" -> v)) - val result = runStableMergeSort(schema, tuples) { - _.keys = sortKeysBuffer(sortKey("x")) - } + val result = runStableMergeSort(tuples)(sortKey("x")) val values = result.map(_.getField[Double]("x")) assert(values.head == Double.NegativeInfinity) assert(values(1) == -3.2) @@ -303,7 +288,7 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { tupleOf(schema, "x" -> Double.PositiveInfinity), tupleOf(schema, "x" -> null) ) - val result = runStableMergeSort(schema, tuples) { _.keys = sortKeysBuffer(sortKey("x")) } + val result = runStableMergeSort(tuples)(sortKey("x")) val values = result.map(_.getField[java.lang.Double]("x")) assert(values.take(4).forall(_ != null)) // first 4 are non-null assert(values(0).isInfinite && values(0) == Double.NegativeInfinity) @@ -314,21 +299,16 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { } it should "place nulls last regardless of ascending or descending" in { - val schema = schemaOf("value" -> AttributeType.INTEGER, "label" -> AttributeType.STRING) val tuples = List( - tupleOf(schema, "value" -> null, "label" -> "null-1"), - tupleOf(schema, "value" -> 5, "label" -> "five"), - tupleOf(schema, "value" -> null, "label" -> "null-2"), - tupleOf(schema, "value" -> 3, "label" -> "three") + tupleOf(integerLabelSchema, "value" -> null, "label" -> "null-1"), + tupleOf(integerLabelSchema, "value" -> 5, "label" -> "five"), + tupleOf(integerLabelSchema, "value" -> null, "label" -> "null-2"), + tupleOf(integerLabelSchema, "value" -> 3, "label" -> "three") ) - val asc = runStableMergeSort(schema, tuples) { - _.keys = sortKeysBuffer(sortKey("value", SortPreference.ASC)) - } + val asc = runStableMergeSort(tuples)(sortKey("value", SortPreference.ASC)) assert(asc.map(_.getField[String]("label")) == List("three", "five", "null-1", "null-2")) - val desc = runStableMergeSort(schema, tuples) { - _.keys = sortKeysBuffer(sortKey("value", SortPreference.DESC)) - } + val desc = runStableMergeSort(tuples)(sortKey("value", SortPreference.DESC)) assert(desc.map(_.getField[String]("label")) == List("five", "three", "null-1", "null-2")) } @@ -348,10 +328,10 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { tupleOf(schema, "group" -> "A", "score" -> null, "label" -> "null-1"), tupleOf(schema, "group" -> "A", "score" -> null, "label" -> "null-2") ) - val result = runStableMergeSort(schema, tuples) { desc => - desc.keys = - sortKeysBuffer(sortKey("group", SortPreference.ASC), sortKey("score", SortPreference.DESC)) - } + val result = runStableMergeSort(tuples)( + sortKey("group", SortPreference.ASC), + sortKey("score", SortPreference.DESC) + ) assert( result.map(_.getField[String]("label")) == List("nan", "pinf", "one", "zero", "neg", "ninf", "null-1", "null-2") @@ -375,9 +355,7 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { tupleOf(schema, "bin" -> null, "id" -> "null-1") ) - val sorted = runStableMergeSort(schema, inputTuples) { - _.keys = sortKeysBuffer(sortKey("bin", SortPreference.DESC)) - } + val sorted = runStableMergeSort(inputTuples)(sortKey("bin", SortPreference.DESC)) val idsInOrder = sorted.map(_.getField[String]("id")) assert(idsInOrder == List("ff-1", "00-1", "00-2", "null-1")) @@ -425,13 +403,11 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { "hired" -> new Timestamp(base.getTime + 4000) ) ) - val result = runStableMergeSort(schema, tuples) { desc => - desc.keys = sortKeysBuffer( - sortKey("dept", SortPreference.ASC), - sortKey("score", SortPreference.DESC), - sortKey("name", SortPreference.ASC) - ) - } + val result = runStableMergeSort(tuples)( + sortKey("dept", SortPreference.ASC), + sortKey("score", SortPreference.DESC), + sortKey("name", SortPreference.ASC) + ) assert(result.map(_.getField[String]("name")) == List("Dave", "Alice", "Bob", "Carol", "Eve")) } @@ -450,10 +426,10 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { (3, 0, 5), (3, 2, 6) ).map { case (ma, mi, i) => tupleOf(schema, "major" -> ma, "minor" -> mi, "idx" -> i) } - val result = runStableMergeSort(schema, tuples) { desc => - desc.keys = - sortKeysBuffer(sortKey("major", SortPreference.DESC), sortKey("minor", SortPreference.ASC)) - } + val result = runStableMergeSort(tuples)( + sortKey("major", SortPreference.DESC), + sortKey("minor", SortPreference.ASC) + ) val pairs = result.map(t => (t.getField[Int]("major"), t.getField[Int]("minor"))) assert(pairs == List((3, 0), (3, 2), (2, 3), (2, 5), (1, 1), (1, 1), (1, 9))) val idxFor11 = result @@ -477,10 +453,11 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { ).map { case (a, b, c, id) => tupleOf(schema, "keyA" -> a, "keyB" -> b, "keyC" -> c, "id" -> id) } - val result = runStableMergeSort(schema, tuples) { - _.keys = - sortKeysBuffer(sortKey("keyA"), sortKey("keyB"), sortKey("keyC", SortPreference.DESC)) - } + val result = runStableMergeSort(tuples)( + sortKey("keyA"), + sortKey("keyB"), + sortKey("keyC", SortPreference.DESC) + ) assert(result.map(_.getField[String]("id")) == List("y9", "x3", "x2", "x1")) } @@ -494,10 +471,10 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { ("a", 9), ("a", 2) ).map { case (s, i) => tupleOf(schema, "keyA" -> s, "keyB" -> i) } - val result = runStableMergeSort(schema, tuples) { desc => - desc.keys = - sortKeysBuffer(sortKey("keyA", SortPreference.ASC), sortKey("keyB", SortPreference.DESC)) - } + val result = runStableMergeSort(tuples)( + sortKey("keyA", SortPreference.ASC), + sortKey("keyB", SortPreference.DESC) + ) val out = result.map(t => (t.getField[String]("keyA"), t.getField[Int]("keyB"))) assert(out == List(("a", 9), ("a", 2), ("x", 2), ("x", 1), (null, 5), (null, 1))) } @@ -514,9 +491,7 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { tupleOf(schema, "keyA" -> null, "keyB" -> 1, "id" -> "null-a-1"), tupleOf(schema, "keyA" -> "B", "keyB" -> 9, "id" -> "non-null-b") ) - val result = runStableMergeSort(schema, tuples) { - _.keys = sortKeysBuffer(sortKey("keyA"), sortKey("keyB")) - } + val result = runStableMergeSort(tuples)(sortKey("keyA"), sortKey("keyB")) assert( result .map(_.getField[String]("id")) == List("non-null-a", "non-null-b", "null-a-1", "null-a-5") @@ -539,12 +514,10 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { tupleOf(schema, "bin" -> key01, "score" -> 2, "label" -> "01-score2") ) - val sorted = runStableMergeSort(schema, inputTuples) { desc => - desc.keys = sortKeysBuffer( - sortKey("bin", SortPreference.ASC), // primary: binary ascending - sortKey("score", SortPreference.DESC) // secondary: integer descending - ) - } + val sorted = runStableMergeSort(inputTuples)( + sortKey("bin", SortPreference.ASC), // primary: binary ascending + sortKey("score", SortPreference.DESC) // secondary: integer descending + ) val labelsInOrder = sorted.map(_.getField[String]("label")) assert(labelsInOrder == List("00-score9", "01-score2", "01-score1")) @@ -556,7 +529,7 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { it should "preserve original order among tuples with equal keys" in { val schema = schemaOf("key" -> AttributeType.INTEGER, "index" -> AttributeType.INTEGER) val tuples = (0 until 100).map(i => tupleOf(schema, "key" -> (i % 5), "index" -> i)) - val result = runStableMergeSort(schema, tuples) { _.keys = sortKeysBuffer(sortKey("key")) } + val result = runStableMergeSort(tuples)(sortKey("key")) val grouped = result.groupBy(_.getField[Int]("key")).values grouped.foreach { group => val indices = group.map(_.getField[Int]("index")) @@ -565,12 +538,9 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { } it should "act as a stable pass-through when keys are empty" in { - val schema = schemaOf("value" -> AttributeType.INTEGER, "label" -> AttributeType.STRING) val tuples = List(3, 1, 4, 1, 5, 9).zipWithIndex - .map { case (v, i) => tupleOf(schema, "value" -> v, "label" -> s"row-$i") } - val result = runStableMergeSort(schema, tuples) { desc => - desc.keys = ListBuffer.empty[SortCriteriaUnit] - } + .map { case (v, i) => tupleOf(integerLabelSchema, "value" -> v, "label" -> s"row-$i") } + val result = runStableMergeSort(tuples)() assert( result.map(_.getField[String]("label")) == List("row-0", "row-1", "row-2", "row-3", "row-4", "row-5") @@ -578,10 +548,8 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { } it should "buffer tuples until onFinish is called" in { - val schema = schemaOf("value" -> AttributeType.INTEGER) - val tuple = tupleOf(schema, "value" -> 2) - val desc = new StableMergeSortOpDesc(); desc.keys = sortKeysBuffer(sortKey("value")) - val exec = new StableMergeSortOpExec(objectMapper.writeValueAsString(desc)) + val tuple = tupleOf(integerSchema, "value" -> 2) + val exec = newExecutor(sortKey("value")) exec.open() val immediate = exec.processTuple(tuple, 0) assert(immediate.isEmpty) @@ -591,23 +559,19 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { } it should "return empty for empty input" in { - val schema = schemaOf("value" -> AttributeType.INTEGER) - val result = runStableMergeSort(schema, Seq.empty) { _.keys = sortKeysBuffer(sortKey("value")) } + val result = runStableMergeSort(Seq.empty)(sortKey("value")) assert(result.isEmpty) } it should "handle single element input" in { - val schema = schemaOf("value" -> AttributeType.INTEGER) - val result = runStableMergeSort(schema, Seq(tupleOf(schema, "value" -> 42))) { - _.keys = sortKeysBuffer(sortKey("value")) - } + val result = runStableMergeSort(Seq(tupleOf(integerSchema, "value" -> 42)))(sortKey("value")) assert(result.map(_.getField[Int]("value")) == List(42)) } it should "sort large inputs efficiently (sanity on boundaries)" in { - val schema = schemaOf("value" -> AttributeType.INTEGER, "label" -> AttributeType.STRING) - val tuples = (50000 to 1 by -1).map(i => tupleOf(schema, "value" -> i, "label" -> s"row-$i")) - val result = runStableMergeSort(schema, tuples) { _.keys = sortKeysBuffer(sortKey("value")) } + val tuples = + (50000 to 1 by -1).map(i => tupleOf(integerLabelSchema, "value" -> i, "label" -> s"row-$i")) + val result = runStableMergeSort(tuples)(sortKey("value")) assert(result.head.getField[Int]("value") == 1) assert(result(1).getField[Int]("value") == 2) assert(result.takeRight(2).map(_.getField[Int]("value")) == List(49999, 50000)) @@ -618,14 +582,12 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { // =========================================================================== it should "merge incrementally: bucket sizes match binary decomposition after each push" in { - val schema = schemaOf("value" -> AttributeType.INTEGER) - val desc = new StableMergeSortOpDesc(); desc.keys = sortKeysBuffer(sortKey("value")) - val exec = new StableMergeSortOpExec(objectMapper.writeValueAsString(desc)) + val exec = newExecutor(sortKey("value")) exec.open() val totalCount = 64 for (index <- (totalCount - 1) to 0 by -1) { - exec.processTuple(tupleOf(schema, "value" -> index), 0) + exec.processTuple(tupleOf(integerSchema, "value" -> index), 0) val sizes = getBucketSizes(exec).sorted assert(sizes == binaryDecomposition(totalCount - index)) } @@ -634,15 +596,13 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { } it should "maintain bucket-stack invariant (no adjacent equal sizes) after each insertion" in { - val schema = schemaOf("value" -> AttributeType.INTEGER) - val desc = new StableMergeSortOpDesc(); desc.keys = sortKeysBuffer(sortKey("value")) - val exec = new StableMergeSortOpExec(objectMapper.writeValueAsString(desc)) + val exec = newExecutor(sortKey("value")) exec.open() val totalCount = 200 val stream = (0 until totalCount by 2) ++ (1 until totalCount by 2) stream.foreach { index => - exec.processTuple(tupleOf(schema, "value" -> (totalCount - 1 - index)), 0) + exec.processTuple(tupleOf(integerSchema, "value" -> (totalCount - 1 - index)), 0) val sizes = getBucketSizes(exec) sizes.sliding(2).foreach { pair => if (pair.length == 2) assert(pair.head != pair.last) @@ -653,12 +613,10 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { } it should "form expected bucket sizes at milestones (1,2,3,4,7,8,15,16)" in { - val schema = schemaOf("value" -> AttributeType.INTEGER) - val desc = new StableMergeSortOpDesc(); desc.keys = sortKeysBuffer(sortKey("value")) - val exec = new StableMergeSortOpExec(objectMapper.writeValueAsString(desc)) + val exec = newExecutor(sortKey("value")) exec.open() - val inputSequence = (100 to 1 by -1).map(i => tupleOf(schema, "value" -> i)) + val inputSequence = (100 to 1 by -1).map(i => tupleOf(integerSchema, "value" -> i)) val milestones = Set(1, 2, 3, 4, 7, 8, 15, 16) var pushed = 0 inputSequence.foreach { t => @@ -677,20 +635,18 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { // =========================================================================== "mergeSortedBuckets" should "be stable: left bucket wins on equal keys" in { - val schema = schemaOf("key" -> AttributeType.INTEGER, "id" -> AttributeType.STRING) - val desc = new StableMergeSortOpDesc(); desc.keys = sortKeysBuffer(sortKey("key")) - val exec = new StableMergeSortOpExec(objectMapper.writeValueAsString(desc)); exec.open() + val exec = newExecutor(sortKey("key")); exec.open() // Seed to resolve schema/keys once. - exec.processTuple(tupleOf(schema, "key" -> 0, "id" -> "seed"), 0) + exec.processTuple(tupleOf(keyIdSchema, "key" -> 0, "id" -> "seed"), 0) val left = ArrayBuffer( - tupleOf(schema, "key" -> 1, "id" -> "L1"), - tupleOf(schema, "key" -> 2, "id" -> "L2") + tupleOf(keyIdSchema, "key" -> 1, "id" -> "L1"), + tupleOf(keyIdSchema, "key" -> 2, "id" -> "L2") ) val right = ArrayBuffer( - tupleOf(schema, "key" -> 1, "id" -> "R1"), - tupleOf(schema, "key" -> 3, "id" -> "R3") + tupleOf(keyIdSchema, "key" -> 1, "id" -> "R1"), + tupleOf(keyIdSchema, "key" -> 3, "id" -> "R3") ) val merged = exec.mergeSortedBuckets(left, right) @@ -700,15 +656,13 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { } "mergeSortedBuckets" should "handle empty left bucket" in { - val schema = schemaOf("key" -> AttributeType.INTEGER, "id" -> AttributeType.STRING) - val desc = new StableMergeSortOpDesc(); desc.keys = sortKeysBuffer(sortKey("key")) - val exec = new StableMergeSortOpExec(objectMapper.writeValueAsString(desc)); exec.open() - exec.processTuple(tupleOf(schema, "key" -> 0, "id" -> "seed"), 0) // seed keys + val exec = newExecutor(sortKey("key")); exec.open() + exec.processTuple(tupleOf(keyIdSchema, "key" -> 0, "id" -> "seed"), 0) // seed keys val left = ArrayBuffer.empty[Tuple] val right = ArrayBuffer( - tupleOf(schema, "key" -> 1, "id" -> "r1"), - tupleOf(schema, "key" -> 2, "id" -> "r2") + tupleOf(keyIdSchema, "key" -> 1, "id" -> "r1"), + tupleOf(keyIdSchema, "key" -> 2, "id" -> "r2") ) val merged = exec.mergeSortedBuckets(left, right) assert(merged.map(_.getField[String]("id")).toList == List("r1", "r2")) @@ -716,14 +670,12 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { } "mergeSortedBuckets" should "handle empty right bucket" in { - val schema = schemaOf("key" -> AttributeType.INTEGER, "id" -> AttributeType.STRING) - val desc = new StableMergeSortOpDesc(); desc.keys = sortKeysBuffer(sortKey("key")) - val exec = new StableMergeSortOpExec(objectMapper.writeValueAsString(desc)); exec.open() - exec.processTuple(tupleOf(schema, "key" -> 0, "id" -> "seed"), 0) + val exec = newExecutor(sortKey("key")); exec.open() + exec.processTuple(tupleOf(keyIdSchema, "key" -> 0, "id" -> "seed"), 0) val left = ArrayBuffer( - tupleOf(schema, "key" -> 1, "id" -> "l1"), - tupleOf(schema, "key" -> 2, "id" -> "l2") + tupleOf(keyIdSchema, "key" -> 1, "id" -> "l1"), + tupleOf(keyIdSchema, "key" -> 2, "id" -> "l2") ) val right = ArrayBuffer.empty[Tuple] val merged = exec.mergeSortedBuckets(left, right) @@ -736,16 +688,16 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { // =========================================================================== "pushBucketAndCombine" should "merge two size-2 buckets into size-4 on push (with existing size-1 seed)" in { - val schema = schemaOf("value" -> AttributeType.INTEGER) - val desc = new StableMergeSortOpDesc(); desc.keys = sortKeysBuffer(sortKey("value")) - val exec = new StableMergeSortOpExec(objectMapper.writeValueAsString(desc)); exec.open() + val exec = newExecutor(sortKey("value")); exec.open() // seed to compile keys -> results in one size-1 bucket in the stack - exec.processTuple(tupleOf(schema, "value" -> 0), 0) + exec.processTuple(tupleOf(integerSchema, "value" -> 0), 0) // two pre-sorted buckets of size 2 - val bucket1 = ArrayBuffer(tupleOf(schema, "value" -> 1), tupleOf(schema, "value" -> 3)) - val bucket2 = ArrayBuffer(tupleOf(schema, "value" -> 2), tupleOf(schema, "value" -> 4)) + val bucket1 = + ArrayBuffer(tupleOf(integerSchema, "value" -> 1), tupleOf(integerSchema, "value" -> 3)) + val bucket2 = + ArrayBuffer(tupleOf(integerSchema, "value" -> 2), tupleOf(integerSchema, "value" -> 4)) exec.pushBucketAndCombine(bucket1) // sizes now [1,2] exec.pushBucketAndCombine(bucket2) // equal top [2,2] => merged to 4; sizes [1,4] @@ -756,10 +708,8 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { } it should "return the same sorted output if onFinish is called twice in a row" in { - val schema = schemaOf("value" -> AttributeType.INTEGER) - val desc = new StableMergeSortOpDesc(); desc.keys = sortKeysBuffer(sortKey("value")) - val exec = new StableMergeSortOpExec(objectMapper.writeValueAsString(desc)); exec.open() - List(3, 1, 2).foreach(i => exec.processTuple(tupleOf(schema, "value" -> i), 0)) + val exec = newExecutor(sortKey("value")); exec.open() + List(3, 1, 2).foreach(i => exec.processTuple(tupleOf(integerSchema, "value" -> i), 0)) val first = exec.onFinish(0).map(_.asInstanceOf[Tuple]).toList.map(_.getField[Int]("value")) val second = exec.onFinish(0).map(_.asInstanceOf[Tuple]).toList.map(_.getField[Int]("value")) @@ -769,10 +719,9 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { } it should "have processTuple always return empty iterators until finish" in { - val schema = schemaOf("value" -> AttributeType.INTEGER) - val desc = new StableMergeSortOpDesc(); desc.keys = sortKeysBuffer(sortKey("value")) - val exec = new StableMergeSortOpExec(objectMapper.writeValueAsString(desc)); exec.open() - val immediates = (10 to 1 by -1).map(i => exec.processTuple(tupleOf(schema, "value" -> i), 0)) + val exec = newExecutor(sortKey("value")); exec.open() + val immediates = + (10 to 1 by -1).map(i => exec.processTuple(tupleOf(integerSchema, "value" -> i), 0)) assert(immediates.forall(_.isEmpty)) val out = exec.onFinish(0).map(_.asInstanceOf[Tuple]).toList.map(_.getField[Int]("value")) assert(out == (1 to 10).toList) @@ -794,9 +743,7 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { // which field that guard reads and what close() actually does to the stack. "debugBucketSizes" should "report no buckets before open and the live stack after it" in { - val schema = schemaOf("value" -> AttributeType.INTEGER) - val desc = new StableMergeSortOpDesc(); desc.keys = sortKeysBuffer(sortKey("value")) - val exec = new StableMergeSortOpExec(objectMapper.writeValueAsString(desc)) + val exec = newExecutor(sortKey("value")) assert(getBucketSizes(exec) == Nil) // open() allocates the bucket stack but leaves the sort keys uncompiled — @@ -806,14 +753,13 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { // distinguishes which of the two fields the Nil guard is reading; a single // size-1 push performs no merge, so no comparison is attempted. exec.open() - exec.pushBucketAndCombine(ArrayBuffer(tupleOf(schema, "value" -> 1))) + exec.pushBucketAndCombine(ArrayBuffer(tupleOf(integerSchema, "value" -> 1))) assert(getBucketSizes(exec) == List(1)) exec.close() } "close" should "be a no-op when it runs before open" in { - val desc = new StableMergeSortOpDesc(); desc.keys = sortKeysBuffer(sortKey("value")) - val exec = new StableMergeSortOpExec(objectMapper.writeValueAsString(desc)) + val exec = newExecutor(sortKey("value")) // Pure "does not throw": the executor was never opened, so this case can only // catch a crash in the guard, never a change in what close() does. The // companion case below is the one that pins the effect. @@ -822,10 +768,8 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { } it should "drop the buffered buckets when it runs after open" in { - val schema = schemaOf("value" -> AttributeType.INTEGER) - val desc = new StableMergeSortOpDesc(); desc.keys = sortKeysBuffer(sortKey("value")) - val exec = new StableMergeSortOpExec(objectMapper.writeValueAsString(desc)); exec.open() - List(3, 1, 2).foreach(i => exec.processTuple(tupleOf(schema, "value" -> i), 0)) + val exec = newExecutor(sortKey("value")); exec.open() + List(3, 1, 2).foreach(i => exec.processTuple(tupleOf(integerSchema, "value" -> i), 0)) assert(getBucketSizes(exec) == List(2, 1)) exec.close() assert(getBucketSizes(exec) == Nil) @@ -839,8 +783,7 @@ class StableMergeSortOpExecSpec extends AnyFlatSpec { // makes the arm's breadth a contract rather than a case for one type. def rejects(attrType: AttributeType, first: Any, second: Any): IllegalStateException = { val schema = schemaOf("value" -> attrType) - val desc = new StableMergeSortOpDesc(); desc.keys = sortKeysBuffer(sortKey("value")) - val exec = new StableMergeSortOpExec(objectMapper.writeValueAsString(desc)); exec.open() + val exec = newExecutor(sortKey("value")); exec.open() // The first tuple only compiles the sort keys. The second makes the stack's // top two buckets equal-sized, and the binary carry that merges them is the // first comparison of two values. diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/apis/twitter/v2/TwitterFullArchiveSearchSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/apis/twitter/v2/TwitterFullArchiveSearchSourceOpDescSpec.scala index 83dac4e2a11..3375859f35b 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/apis/twitter/v2/TwitterFullArchiveSearchSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/apis/twitter/v2/TwitterFullArchiveSearchSourceOpDescSpec.scala @@ -19,30 +19,22 @@ package org.apache.texera.amber.operator.source.apis.twitter.v2 -import org.apache.texera.amber.core.executor.OpExecWithClassName -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers import scala.annotation.nowarn // The Twitter source operators are @deprecated (no longer executable) but retained so // legacy workflows still deserialize; the coverage below pins that backward-compatible contract. @nowarn("cat=deprecation") -class TwitterFullArchiveSearchSourceOpDescSpec extends AnyFlatSpec with Matchers { +class TwitterFullArchiveSearchSourceOpDescSpec + extends AnyFlatSpec + with TwitterSourceOpDescSpecSupport { "TwitterFullArchiveSearchSourceOpDesc.operatorInfo" should "advertise the Twitter Full Archive Search API source in the External API group" in { - val info = (new TwitterFullArchiveSearchSourceOpDesc).operatorInfo - info.userFriendlyName shouldBe "Twitter Full Archive Search API" - info.operatorDescription shouldBe "Retrieve data from Twitter Full Archive Search API" - info.operatorGroupName shouldBe OperatorGroupConstants.API_GROUP - info.inputPorts shouldBe empty - info.outputPorts should have length 1 + checkOperatorInfo(new TwitterFullArchiveSearchSourceOpDesc, "Twitter Full Archive Search API") } "TwitterFullArchiveSearchSourceOpDesc" should @@ -60,12 +52,7 @@ class TwitterFullArchiveSearchSourceOpDescSpec extends AnyFlatSpec with Matchers "TwitterFullArchiveSearchSourceOpDesc.sourceSchema" should "describe the fixed 33-column tweet schema" in { - val schema = (new TwitterFullArchiveSearchSourceOpDesc).sourceSchema() - schema.getAttributes should have length 33 - schema.getAttribute("id").getType shouldBe AttributeType.STRING - schema.getAttribute("created_at").getType shouldBe AttributeType.TIMESTAMP - schema.getAttribute("retweet_count").getType shouldBe AttributeType.LONG - schema.getAttribute("user_verified").getType shouldBe AttributeType.BOOLEAN + checkTweetSchema(new TwitterFullArchiveSearchSourceOpDesc) } "TwitterFullArchiveSearchSourceOpDesc" should @@ -95,21 +82,6 @@ class TwitterFullArchiveSearchSourceOpDescSpec extends AnyFlatSpec with Matchers val d = new TwitterFullArchiveSearchSourceOpDesc d.searchQuery = "texera" d.limit = 50 - val p = d.getPhysicalOp(WorkflowIdentity(1L), ExecutionIdentity(1L)) - - p.opExecInitInfo match { - case OpExecWithClassName(className, descString) => - className shouldBe - "org.apache.texera.amber.operator.source.apis.twitter.v2.TwitterFullArchiveSearchSourceOpExec" - descString should include("texera") - case other => fail(s"expected OpExecWithClassName, got $other") - } - - p.inputPorts shouldBe empty - p.outputPorts should have size 1 - - // Invoke the schema-propagation closure so its body (not just the wiring) is exercised. - val propagated = p.propagateSchema.func(Map.empty) - propagated.keySet shouldBe d.operatorInfo.outputPorts.map(_.id).toSet + checkSourceWiring(d, "TwitterFullArchiveSearchSourceOpExec") } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/apis/twitter/v2/TwitterSearchSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/apis/twitter/v2/TwitterSearchSourceOpDescSpec.scala index 74d5533a4c7..4f69019ffdb 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/apis/twitter/v2/TwitterSearchSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/apis/twitter/v2/TwitterSearchSourceOpDescSpec.scala @@ -24,6 +24,7 @@ import org.apache.texera.amber.core.tuple.AttributeType import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants +import org.apache.texera.amber.operator.source.apis.twitter.TwitterSourceOpDesc import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -33,16 +34,11 @@ import scala.annotation.nowarn // The Twitter source operators are @deprecated (no longer executable) but retained so // legacy workflows still deserialize; the coverage below pins that backward-compatible contract. @nowarn("cat=deprecation") -class TwitterSearchSourceOpDescSpec extends AnyFlatSpec with Matchers { +class TwitterSearchSourceOpDescSpec extends AnyFlatSpec with TwitterSourceOpDescSpecSupport { "TwitterSearchSourceOpDesc.operatorInfo" should "advertise the Twitter Search API source in the External API group" in { - val info = (new TwitterSearchSourceOpDesc).operatorInfo - info.userFriendlyName shouldBe "Twitter Search API" - info.operatorDescription shouldBe "Retrieve data from Twitter Search API" - info.operatorGroupName shouldBe OperatorGroupConstants.API_GROUP - info.inputPorts shouldBe empty - info.outputPorts should have length 1 + checkOperatorInfo(new TwitterSearchSourceOpDesc, "Twitter Search API") } "TwitterSearchSourceOpDesc" should "default its query/credential fields (limit is 0, not 100)" in { @@ -57,12 +53,7 @@ class TwitterSearchSourceOpDescSpec extends AnyFlatSpec with Matchers { "TwitterSearchSourceOpDesc.sourceSchema" should "describe the fixed 33-column tweet schema" in { - val schema = (new TwitterSearchSourceOpDesc).sourceSchema() - schema.getAttributes should have length 33 - schema.getAttribute("id").getType shouldBe AttributeType.STRING - schema.getAttribute("created_at").getType shouldBe AttributeType.TIMESTAMP - schema.getAttribute("like_count").getType shouldBe AttributeType.LONG - schema.getAttribute("user_protected").getType shouldBe AttributeType.BOOLEAN + checkTweetSchema(new TwitterSearchSourceOpDesc) } "TwitterSearchSourceOpDesc" should "round-trip its config fields through the polymorphic base" in { @@ -89,21 +80,45 @@ class TwitterSearchSourceOpDescSpec extends AnyFlatSpec with Matchers { val d = new TwitterSearchSourceOpDesc d.searchQuery = "texera" d.limit = 50 - val p = d.getPhysicalOp(WorkflowIdentity(1L), ExecutionIdentity(1L)) + checkSourceWiring(d, "TwitterSearchSourceOpExec") + } +} - p.opExecInitInfo match { +@nowarn("cat=deprecation") +private[v2] trait TwitterSourceOpDescSpecSupport extends Matchers { + + protected def checkOperatorInfo(desc: TwitterSourceOpDesc, title: String): Unit = { + val info = desc.operatorInfo + info.userFriendlyName shouldBe title + info.operatorDescription shouldBe s"Retrieve data from $title" + info.operatorGroupName shouldBe OperatorGroupConstants.API_GROUP + info.inputPorts shouldBe empty + info.outputPorts should have length 1 + } + + protected def checkTweetSchema(desc: TwitterSourceOpDesc): Unit = { + val schema = desc.sourceSchema() + schema.getAttributes should have length 33 + schema.getAttribute("id").getType shouldBe AttributeType.STRING + schema.getAttribute("created_at").getType shouldBe AttributeType.TIMESTAMP + schema.getAttribute("retweet_count").getType shouldBe AttributeType.LONG + schema.getAttribute("user_verified").getType shouldBe AttributeType.BOOLEAN + schema.getAttribute("like_count").getType shouldBe AttributeType.LONG + schema.getAttribute("user_protected").getType shouldBe AttributeType.BOOLEAN + } + + protected def checkSourceWiring(desc: TwitterSourceOpDesc, executorName: String): Unit = { + val physical = desc.getPhysicalOp(WorkflowIdentity(1L), ExecutionIdentity(1L)) + physical.opExecInitInfo match { case OpExecWithClassName(className, descString) => - className shouldBe - "org.apache.texera.amber.operator.source.apis.twitter.v2.TwitterSearchSourceOpExec" + className shouldBe s"org.apache.texera.amber.operator.source.apis.twitter.v2.$executorName" descString should include("texera") case other => fail(s"expected OpExecWithClassName, got $other") } + physical.inputPorts shouldBe empty + physical.outputPorts should have size 1 - p.inputPorts shouldBe empty - p.outputPorts should have size 1 - - // Invoke the schema-propagation closure so its body (not just the wiring) is exercised. - val propagated = p.propagateSchema.func(Map.empty) - propagated.keySet shouldBe d.operatorInfo.outputPorts.map(_.id).toSet + val propagated = physical.propagateSchema.func(Map.empty) + propagated.keySet shouldBe desc.operatorInfo.outputPorts.map(_.id).toSet } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala index eb17bbe3a3a..3439e547d5c 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala @@ -35,6 +35,7 @@ import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, Workflow import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} import org.apache.texera.amber.operator.metadata.OperatorGroupConstants import org.apache.texera.amber.operator.source.scan.FileDecodingMethod +import org.apache.texera.amber.operator.source.scan.arrow.ArrowSourceFixtures._ import org.apache.texera.amber.util.ArrowUtils import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec @@ -51,39 +52,6 @@ class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) - private def writeArrowFile( - schema: Schema, - rows: Seq[Array[Any]], - schemaMetadata: Map[String, String] = Map.empty - ): File = { - val file = File.createTempFile("arrow-src-", ".arrow") - file.deleteOnExit() - val allocator = new RootAllocator() - val root = VectorSchemaRoot.create( - new ArrowFileSchema(ArrowUtils.fromTexeraSchema(schema).getFields, schemaMetadata.asJava), - allocator - ) - val out = new FileOutputStream(file) - val writer = new ArrowFileWriter(root, null, Channels.newChannel(out)) - try { - writer.start() - root.allocateNew() - rows.zipWithIndex.foreach { - case (values, i) => - ArrowUtils.setTexeraTuple(Tuple.builder(schema).addSequentially(values).build(), i, root) - } - root.setRowCount(rows.size) - writer.writeBatch() - writer.end() - } finally { - writer.close() - root.close() - allocator.close() - out.close() - } - file - } - "ArrowSourceOpDesc.operatorInfo" should "advertise the Arrow file-scan name in the Data Input group with no input and one output" in { val info = (new ArrowSourceOpDesc).operatorInfo @@ -220,9 +188,7 @@ class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { } it should "throw a friendly error when the file is not a valid Arrow file" in { - val bogus = File.createTempFile("not-arrow-", ".arrow") - bogus.deleteOnExit() - Files.write(bogus.toPath, "this is not arrow".getBytes) + val bogus = invalidArrowFile() val d = new ArrowSourceOpDesc d.fileName = Some(bogus.toURI.toString) val ex = intercept[RuntimeException](d.inferSchema()) @@ -291,24 +257,8 @@ class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { // the schema rather than read as the -1 its storage counts down to. it should "refuse an unsigned 64-bit column, having no Texera type for it" in { val fields = List(new Field("u", FieldType.nullable(new ArrowType.Int(64, false)), null)) - val file = File.createTempFile("arrow-u64-", ".arrow") - file.deleteOnExit() - val allocator = new RootAllocator() - val root = VectorSchemaRoot.create(new ArrowFileSchema(fields.asJava), allocator) - val out = new FileOutputStream(file) - val writer = new ArrowFileWriter(root, null, Channels.newChannel(out)) - try { - writer.start() - root.allocateNew() + val file = writeRawArrowFile("arrow-u64-", new ArrowFileSchema(fields.asJava), 1) { root => root.getVector("u").asInstanceOf[UInt8Vector].setSafe(0, -1L) - root.setRowCount(1) - writer.writeBatch() - writer.end() - } finally { - writer.close() - root.close() - allocator.close() - out.close() } val d = new ArrowSourceOpDesc @@ -325,3 +275,76 @@ class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { private def scriptHeader(d: ArrowSourceOpDesc): String = (d.standaloneImports() :+ "import pandas as pd" :+ bindSourceFile(d)).mkString("\n") } + +private[arrow] object ArrowSourceFixtures { + def writeArrowFile( + schema: Schema, + rows: Seq[Array[Any]], + schemaMetadata: Map[String, String] = Map.empty + ): File = + writeArrowBatches(schema, Seq(rows), schemaMetadata) + + def writeArrowBatches( + schema: Schema, + batches: Seq[Seq[Array[Any]]], + schemaMetadata: Map[String, String] = Map.empty + ): File = + writeRawArrowBatches( + "arrow-src-", + new ArrowFileSchema(ArrowUtils.fromTexeraSchema(schema).getFields, schemaMetadata.asJava), + batches.map { rows => + rows.size -> { (root: VectorSchemaRoot) => + rows.zipWithIndex.foreach { + case (values, i) => + ArrowUtils.setTexeraTuple( + Tuple.builder(schema).addSequentially(values).build(), + i, + root + ) + } + } + } + ) + + def writeRawArrowFile(prefix: String, schema: => ArrowFileSchema, rowCount: Int)( + populate: VectorSchemaRoot => Unit + ): File = + writeRawArrowBatches(prefix, schema, Seq(rowCount -> populate)) + + private def writeRawArrowBatches( + prefix: String, + schema: => ArrowFileSchema, + batches: Seq[(Int, VectorSchemaRoot => Unit)] + ): File = { + val file = File.createTempFile(prefix, ".arrow") + file.deleteOnExit() + val allocator = new RootAllocator() + val root = VectorSchemaRoot.create(schema, allocator) + val out = new FileOutputStream(file) + val writer = new ArrowFileWriter(root, null, Channels.newChannel(out)) + try { + writer.start() + batches.foreach { + case (rowCount, populate) => + root.allocateNew() + populate(root) + root.setRowCount(rowCount) + writer.writeBatch() + } + writer.end() + } finally { + writer.close() + root.close() + allocator.close() + out.close() + } + file + } + + def invalidArrowFile(): File = { + val file = File.createTempFile("not-arrow-", ".arrow") + file.deleteOnExit() + Files.write(file.toPath, "this is not arrow".getBytes) + file + } +} diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpExecSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpExecSpec.scala index 440c83dcfec..288cb5cc5cb 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpExecSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpExecSpec.scala @@ -19,55 +19,20 @@ package org.apache.texera.amber.operator.source.scan.arrow -import org.apache.arrow.memory.RootAllocator -import org.apache.arrow.vector.VectorSchemaRoot -import org.apache.arrow.vector.ipc.ArrowFileWriter import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema, Tuple} -import org.apache.texera.amber.util.ArrowUtils import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -import java.io.{File, FileOutputStream} -import java.nio.channels.Channels -import java.nio.file.Files +import java.io.File class ArrowSourceOpExecSpec extends AnyFlatSpec with Matchers { private val schema = Schema(List(new Attribute("s", AttributeType.STRING))) /** Writes one record batch per argument, so a file can hold several or none. */ - private def writeArrowFile(batches: Seq[String]*): File = { - val file = File.createTempFile("arrow-src-", ".arrow") - file.deleteOnExit() - val allocator = new RootAllocator() - val root = VectorSchemaRoot.create(ArrowUtils.fromTexeraSchema(schema), allocator) - val out = new FileOutputStream(file) - val writer = new ArrowFileWriter(root, null, Channels.newChannel(out)) - try { - writer.start() - batches.foreach { rows => - root.allocateNew() - rows.zipWithIndex.foreach { - case (value, i) => - ArrowUtils.setTexeraTuple( - Tuple.builder(schema).addSequentially(Array[Any](value)).build(), - i, - root - ) - } - root.setRowCount(rows.size) - writer.writeBatch() - } - writer.end() - } finally { - writer.close() - root.close() - allocator.close() - out.close() - } - file - } + private def writeArrowFile(batches: Seq[String]*): File = + ArrowSourceFixtures.writeArrowBatches(schema, batches.map(_.map(value => Array[Any](value)))) private def descString( file: File, @@ -129,9 +94,7 @@ class ArrowSourceOpExecSpec extends AnyFlatSpec with Matchers { } it should "wrap failures to open an invalid Arrow file" in { - val bogus = File.createTempFile("not-arrow-", ".arrow") - bogus.deleteOnExit() - Files.write(bogus.toPath, "this is not arrow".getBytes) + val bogus = ArrowSourceFixtures.invalidArrowFile() val exec = new ArrowSourceOpExec(descString(bogus)) val ex = intercept[RuntimeException](exec.open()) ex.getMessage shouldBe "Failed to open Arrow source" diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/ChartConfigSpecSupport.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/ChartConfigSpecSupport.scala new file mode 100644 index 00000000000..1f8320b4920 --- /dev/null +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/ChartConfigSpecSupport.scala @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.operator.visualization + +import com.fasterxml.jackson.annotation.JsonProperty +import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName +import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.scalatest.Assertions + +import javax.validation.constraints.NotNull + +/** Shared assertion helpers for chart config serialization, annotation, and bounds specs. */ +private[visualization] trait ChartConfigSpecSupport extends Assertions { + + protected def checkRoundTrip[T](value: T, clazz: Class[T]): T = + objectMapper.readValue(objectMapper.writeValueAsString(value), clazz) + + protected def assertAttrJsonPropertyRequired(clazz: Class[_]): Unit = { + val jp = clazz.getDeclaredField("attributeName").getAnnotation(classOf[JsonProperty]) + assert(jp != null && jp.required, "attributeName must carry @JsonProperty(required = true)") + } + + protected def assertAttrAutofill(clazz: Class[_], msg: String): Unit = { + val ann = clazz.getDeclaredField("attributeName").getAnnotation(classOf[AutofillAttributeName]) + assert(ann != null, msg) + } + + protected def assertAttrNotNull(clazz: Class[_]): Unit = { + val ann = clazz.getDeclaredField("attributeName").getAnnotation(classOf[NotNull]) + assert(ann != null && ann.message == "Attribute Name cannot be empty") + } + + protected def assertStepWireKeys[T](step: T, expectedStart: Double, expectedEnd: Double): Unit = { + val tree = objectMapper.readTree(objectMapper.writeValueAsString(step)) + assert( + tree + .has("start") && tree.get("start").isNumber && tree.get("start").asDouble() == expectedStart + ) + assert(tree.has("end") && tree.get("end").isNumber && tree.get("end").asDouble() == expectedEnd) + } + + protected def assertBoundsPair[T]( + clazz: Class[T], + json: String, + boundsOf: T => (Option[Double], Option[Double]), + expectedStart: Double, + expectedEnd: Double + ): Unit = { + val (s, e) = boundsOf(objectMapper.readValue(json, clazz)) + assert(s.contains(expectedStart)) + assert(e.contains(expectedEnd)) + } + + protected def assertUnsetBounds[T](clazz: Class[T], startOf: T => Option[Double]): Unit = { + assert(startOf(objectMapper.readValue("""{}""", clazz)).isEmpty) + assert(startOf(objectMapper.readValue("""{"start":null}""", clazz)).isEmpty) + assert(startOf(objectMapper.readValue("""{"start":""}""", clazz)).isEmpty) + } + + protected def assertDoubleBoundsConversion[T]( + clazz: Class[T], + startOf: T => Option[Double] + ): Unit = { + assert(startOf(objectMapper.readValue("""{"start":"1.5"}""", clazz)).map(_ * 2).contains(3.0)) + } +} diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/bulletChart/BulletChartStepDefinitionSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/bulletChart/BulletChartStepDefinitionSpec.scala index 35861db4619..d9271ab5a1b 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/bulletChart/BulletChartStepDefinitionSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/bulletChart/BulletChartStepDefinitionSpec.scala @@ -20,125 +20,89 @@ package org.apache.texera.amber.operator.visualization.bulletChart import com.fasterxml.jackson.annotation.{JsonCreator, JsonProperty} -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.visualization.ChartConfigSpecSupport import org.scalatest.flatspec.AnyFlatSpec -class BulletChartStepDefinitionSpec extends AnyFlatSpec { - - // --------------------------------------------------------------------------- - // Construction — @JsonCreator constructor accepts both fields - // --------------------------------------------------------------------------- +class BulletChartStepDefinitionSpec extends AnyFlatSpec with ChartConfigSpecSupport { "BulletChartStepDefinition" should "store both constructor arguments" in { val d = new BulletChartStepDefinition(Some(10), Some(90)) - assert(d.start.contains(10)) - assert(d.end.contains(90)) + assert(d.start.contains(10) && d.end.contains(90)) } - // --------------------------------------------------------------------------- - // Mutability - // --------------------------------------------------------------------------- - it should "allow both fields to be reassigned post-construction" in { val d = new BulletChartStepDefinition(Some(0), Some(1)) d.start = Some(2.5) d.end = Some(7.5) - assert(d.start.contains(2.5)) - assert(d.end.contains(7.5)) + assert(d.start.contains(2.5) && d.end.contains(7.5)) } - // --------------------------------------------------------------------------- - // JSON round-trip — wire keys are `start` / `end` - // --------------------------------------------------------------------------- - "BulletChartStepDefinition JSON round-trip" should "serialize start and end as numbers under the canonical wire keys" in { - val d = new BulletChartStepDefinition(Some(1.5), Some(9.5)) - val tree = objectMapper.readTree(objectMapper.writeValueAsString(d)) - assert(tree.has("start")) - assert(tree.get("start").isNumber) - assert(tree.get("start").asDouble() == 1.5) - assert(tree.has("end")) - assert(tree.get("end").isNumber) - assert(tree.get("end").asDouble() == 9.5) + assertStepWireKeys(new BulletChartStepDefinition(Some(1.5), Some(9.5)), 1.5, 9.5) } it should "round-trip both fields cleanly" in { - val d = new BulletChartStepDefinition(Some(33), Some(66)) - val restored = objectMapper.readValue( - objectMapper.writeValueAsString(d), + val restored = checkRoundTrip( + new BulletChartStepDefinition(Some(33), Some(66)), classOf[BulletChartStepDefinition] ) - assert(restored.start.contains(33)) - assert(restored.end.contains(66)) + assert(restored.start.contains(33) && restored.end.contains(66)) } - /** Reads the shapes a stored workflow can hold; a round trip cannot cover them, - * since it writes a number back. See GaugeChartStepsSpec for why `contentAs` is - * what these pin. - */ - private def read(json: String): BulletChartStepDefinition = - objectMapper.readValue(json, classOf[BulletChartStepDefinition]) - "BulletChartStepDefinition bounds" should "deserialize JSON numbers" in { - val d = read("""{"start":1.5,"end":9.5}""") - assert(d.start.contains(1.5)) - assert(d.end.contains(9.5)) + assertBoundsPair[BulletChartStepDefinition]( + classOf[BulletChartStepDefinition], + """{"start":1.5,"end":9.5}""", + d => (d.start, d.end), + 1.5, + 9.5 + ) } it should "deserialize the numeric strings a workflow saved before the bounds were numeric" in { - val d = read("""{"start":"1.5","end":"9.5"}""") - assert(d.start.contains(1.5)) - assert(d.end.contains(9.5)) + assertBoundsPair[BulletChartStepDefinition]( + classOf[BulletChartStepDefinition], + """{"start":"1.5","end":"9.5"}""", + d => (d.start, d.end), + 1.5, + 9.5 + ) } it should "read absent, null and blank bounds as unset rather than as zero" in { - assert(read("""{}""").start.isEmpty) - assert(read("""{"start":null}""").start.isEmpty) - assert(read("""{"start":""}""").start.isEmpty) + assertUnsetBounds[BulletChartStepDefinition](classOf[BulletChartStepDefinition], _.start) } it should "hold a Double, not the raw JSON value" in { - assert(read("""{"start":"1.5"}""").start.map(_ * 2).contains(3.0)) + assertDoubleBoundsConversion[BulletChartStepDefinition]( + classOf[BulletChartStepDefinition], + _.start + ) } - // --------------------------------------------------------------------------- - // Annotations — on the @JsonCreator constructor parameters (Scala places - // annotations on `var` ctor params on the parameter, not the synthesized - // field, unless `@(JsonProperty @meta.field)` is used). - // --------------------------------------------------------------------------- - - // Select the @JsonCreator-annotated constructor by its annotation rather than - // by reflection order (`getDeclaredConstructors.head`), so the test stays - // deterministic if an auxiliary constructor is ever added. private val jsonCreatorCtor = classOf[BulletChartStepDefinition].getDeclaredConstructors .find(_.isAnnotationPresent(classOf[JsonCreator])) - .getOrElse( - fail("expected a @JsonCreator constructor on BulletChartStepDefinition") - ) + .getOrElse(fail("expected a @JsonCreator constructor on BulletChartStepDefinition")) private def ctorParamJsonProperty(paramIndex: Int): JsonProperty = { - val annotations = jsonCreatorCtor.getParameterAnnotations()(paramIndex) - annotations.collectFirst { case jp: JsonProperty => jp }.orNull + jsonCreatorCtor + .getParameterAnnotations()(paramIndex) + .collectFirst { case jp: JsonProperty => jp } + .orNull } "BulletChartStepDefinition ctor param[0] (start)" should "carry @JsonProperty(\"start\")" in { val jp = ctorParamJsonProperty(0) - assert(jp != null) - assert(jp.value == "start") + assert(jp != null && jp.value == "start") } "BulletChartStepDefinition ctor param[1] (end)" should "carry @JsonProperty(\"end\")" in { val jp = ctorParamJsonProperty(1) - assert(jp != null) - assert(jp.value == "end") + assert(jp != null && jp.value == "end") } - // --------------------------------------------------------------------------- - // Instance independence - // --------------------------------------------------------------------------- - it should "construct two independent instances (no static state shared)" in { val a = new BulletChartStepDefinition(Some(1), Some(2)) val b = new BulletChartStepDefinition(Some(3), Some(4)) diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/figureFactoryTable/FigureFactoryTableConfigSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/figureFactoryTable/FigureFactoryTableConfigSpec.scala index e7a63291593..6016ef22d8d 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/figureFactoryTable/FigureFactoryTableConfigSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/figureFactoryTable/FigureFactoryTableConfigSpec.scala @@ -19,77 +19,47 @@ package org.apache.texera.amber.operator.visualization.figureFactoryTable -import com.fasterxml.jackson.annotation.JsonProperty -import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.visualization.ChartConfigSpecSupport import org.scalatest.flatspec.AnyFlatSpec -class FigureFactoryTableConfigSpec extends AnyFlatSpec { - - // --------------------------------------------------------------------------- - // Default state - // --------------------------------------------------------------------------- +class FigureFactoryTableConfigSpec extends AnyFlatSpec with ChartConfigSpecSupport { "FigureFactoryTableConfig" should "default attributeName to the empty string" in { - val c = new FigureFactoryTableConfig - assert(c.attributeName == "") + assert(new FigureFactoryTableConfig().attributeName == "") } - // --------------------------------------------------------------------------- - // Mutability - // --------------------------------------------------------------------------- - it should "allow attributeName to be assigned post-construction" in { val c = new FigureFactoryTableConfig c.attributeName = "col-z" assert(c.attributeName == "col-z") } - // --------------------------------------------------------------------------- - // JSON round-trip - // --------------------------------------------------------------------------- - "FigureFactoryTableConfig JSON round-trip" should "preserve attributeName" in { val original = new FigureFactoryTableConfig original.attributeName = "x" - val restored = objectMapper.readValue( - objectMapper.writeValueAsString(original), - classOf[FigureFactoryTableConfig] - ) - assert(restored.attributeName == "x") + assert(checkRoundTrip(original, classOf[FigureFactoryTableConfig]).attributeName == "x") } it should "round-trip the default empty attributeName" in { - val restored = objectMapper.readValue( - objectMapper.writeValueAsString(new FigureFactoryTableConfig), - classOf[FigureFactoryTableConfig] + assert( + checkRoundTrip( + new FigureFactoryTableConfig, + classOf[FigureFactoryTableConfig] + ).attributeName == "" ) - assert(restored.attributeName == "") } - // --------------------------------------------------------------------------- - // Annotations - // --------------------------------------------------------------------------- - "FigureFactoryTableConfig#attributeName" should "carry @JsonProperty(required = true)" in { - val jp = classOf[FigureFactoryTableConfig] - .getDeclaredField("attributeName") - .getAnnotation(classOf[JsonProperty]) - assert(jp != null && jp.required) + assertAttrJsonPropertyRequired(classOf[FigureFactoryTableConfig]) } - it should - "carry @AutofillAttributeName (UI relies on this to populate a dropdown)" in { - val ann = classOf[FigureFactoryTableConfig] - .getDeclaredField("attributeName") - .getAnnotation(classOf[AutofillAttributeName]) - assert(ann != null, "@AutofillAttributeName must be present so the UI auto-populates") + it should "carry @AutofillAttributeName (UI relies on this to populate a dropdown)" in { + assertAttrAutofill( + classOf[FigureFactoryTableConfig], + "@AutofillAttributeName must be present so the UI auto-populates" + ) } - // --------------------------------------------------------------------------- - // Instance independence - // --------------------------------------------------------------------------- - it should "construct two independent instances (no static state shared)" in { val a = new FigureFactoryTableConfig val b = new FigureFactoryTableConfig diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/gaugeChart/GaugeChartStepsSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/gaugeChart/GaugeChartStepsSpec.scala index 2989e262356..8fd34e9d07b 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/gaugeChart/GaugeChartStepsSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/gaugeChart/GaugeChartStepsSpec.scala @@ -20,119 +20,77 @@ package org.apache.texera.amber.operator.visualization.gaugeChart import com.fasterxml.jackson.annotation.JsonProperty -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.visualization.ChartConfigSpecSupport import org.scalatest.flatspec.AnyFlatSpec -class GaugeChartStepsSpec extends AnyFlatSpec { - - // --------------------------------------------------------------------------- - // Defaults - // --------------------------------------------------------------------------- +class GaugeChartStepsSpec extends AnyFlatSpec with ChartConfigSpecSupport { "GaugeChartSteps" should "default start and end to unset" in { val s = new GaugeChartSteps - assert(s.start.isEmpty) - assert(s.end.isEmpty) + assert(s.start.isEmpty && s.end.isEmpty) } - // --------------------------------------------------------------------------- - // Mutability - // --------------------------------------------------------------------------- - it should "allow start and end to be assigned post-construction" in { val s = new GaugeChartSteps s.start = Some(10) s.end = Some(90) - assert(s.start.contains(10)) - assert(s.end.contains(90)) + assert(s.start.contains(10) && s.end.contains(90)) } - // --------------------------------------------------------------------------- - // JSON round-trip — wire keys are `start` / `end` - // --------------------------------------------------------------------------- - "GaugeChartSteps JSON round-trip" should "serialize start and end as numbers under the canonical wire keys" in { val s = new GaugeChartSteps s.start = Some(1.5) s.end = Some(9.5) - val tree = objectMapper.readTree(objectMapper.writeValueAsString(s)) - assert(tree.has("start")) - assert(tree.get("start").isNumber) - assert(tree.get("start").asDouble() == 1.5) - assert(tree.has("end")) - assert(tree.get("end").isNumber) - assert(tree.get("end").asDouble() == 9.5) + assertStepWireKeys(s, 1.5, 9.5) } it should "round-trip both fields cleanly" in { val s = new GaugeChartSteps s.start = Some(0) s.end = Some(100) - val restored = objectMapper.readValue( - objectMapper.writeValueAsString(s), - classOf[GaugeChartSteps] - ) - assert(restored.start.contains(0)) - assert(restored.end.contains(100)) + val restored = checkRoundTrip(s, classOf[GaugeChartSteps]) + assert(restored.start.contains(0) && restored.end.contains(100)) } - /** `Option[Double]` erases its element type, so Jackson needs - * `@JsonDeserialize(contentAs = ...)` to know what to build. Without it a JSON - * string is left inside the Option unconverted and the first arithmetic use - * throws ClassCastException — which a round trip cannot catch, since it writes a - * number back. These read the shapes a stored workflow can actually hold. - */ - private def read(json: String): GaugeChartSteps = - objectMapper.readValue(json, classOf[GaugeChartSteps]) - "GaugeChartSteps bounds" should "deserialize JSON numbers" in { - val s = read("""{"start":1.5,"end":9.5}""") - assert(s.start.contains(1.5)) - assert(s.end.contains(9.5)) + assertBoundsPair[GaugeChartSteps]( + classOf[GaugeChartSteps], + """{"start":1.5,"end":9.5}""", + s => (s.start, s.end), + 1.5, + 9.5 + ) } it should "deserialize the numeric strings a workflow saved before the bounds were numeric" in { - val s = read("""{"start":"1.5","end":"9.5"}""") - assert(s.start.contains(1.5)) - assert(s.end.contains(9.5)) + assertBoundsPair[GaugeChartSteps]( + classOf[GaugeChartSteps], + """{"start":"1.5","end":"9.5"}""", + s => (s.start, s.end), + 1.5, + 9.5 + ) } it should "read absent, null and blank bounds as unset rather than as zero" in { - assert(read("""{}""").start.isEmpty) - assert(read("""{"start":null}""").start.isEmpty) - assert(read("""{"start":""}""").start.isEmpty) + assertUnsetBounds[GaugeChartSteps](classOf[GaugeChartSteps], _.start) } it should "hold a Double, not the raw JSON value" in { - // The ClassCastException surfaces here, at the first use, not at read time. - assert(read("""{"start":"1.5"}""").start.map(_ * 2).contains(3.0)) + assertDoubleBoundsConversion[GaugeChartSteps](classOf[GaugeChartSteps], _.start) } - // --------------------------------------------------------------------------- - // Annotations - // --------------------------------------------------------------------------- - "GaugeChartSteps#start" should "carry @JsonProperty(\"start\")" in { - val jp = classOf[GaugeChartSteps] - .getDeclaredField("start") - .getAnnotation(classOf[JsonProperty]) - assert(jp != null) - assert(jp.value == "start") + val jp = classOf[GaugeChartSteps].getDeclaredField("start").getAnnotation(classOf[JsonProperty]) + assert(jp != null && jp.value == "start") } "GaugeChartSteps#end" should "carry @JsonProperty(\"end\")" in { - val jp = classOf[GaugeChartSteps] - .getDeclaredField("end") - .getAnnotation(classOf[JsonProperty]) - assert(jp != null) - assert(jp.value == "end") + val jp = classOf[GaugeChartSteps].getDeclaredField("end").getAnnotation(classOf[JsonProperty]) + assert(jp != null && jp.value == "end") } - // --------------------------------------------------------------------------- - // Instance independence - // --------------------------------------------------------------------------- - it should "construct two independent instances (no static state shared)" in { val a = new GaugeChartSteps val b = new GaugeChartSteps diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/hierarchychart/HierarchySectionSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/hierarchychart/HierarchySectionSpec.scala index cb40df93f85..c40ea201e38 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/hierarchychart/HierarchySectionSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/hierarchychart/HierarchySectionSpec.scala @@ -19,86 +19,43 @@ package org.apache.texera.amber.operator.visualization.hierarchychart -import com.fasterxml.jackson.annotation.JsonProperty -import javax.validation.constraints.NotNull -import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.visualization.ChartConfigSpecSupport import org.scalatest.flatspec.AnyFlatSpec -class HierarchySectionSpec extends AnyFlatSpec { - - // --------------------------------------------------------------------------- - // Defaults - // --------------------------------------------------------------------------- +class HierarchySectionSpec extends AnyFlatSpec with ChartConfigSpecSupport { "HierarchySection" should "default attributeName to the empty string" in { - val s = new HierarchySection - assert(s.attributeName == "") + assert(new HierarchySection().attributeName == "") } - // --------------------------------------------------------------------------- - // Mutability - // --------------------------------------------------------------------------- - it should "allow attributeName to be reassigned post-construction" in { val s = new HierarchySection s.attributeName = "country" assert(s.attributeName == "country") } - // --------------------------------------------------------------------------- - // JSON round-trip - // --------------------------------------------------------------------------- - "HierarchySection JSON round-trip" should "preserve attributeName" in { val s = new HierarchySection s.attributeName = "region" - val restored = objectMapper.readValue( - objectMapper.writeValueAsString(s), - classOf[HierarchySection] - ) - assert(restored.attributeName == "region") + assert(checkRoundTrip(s, classOf[HierarchySection]).attributeName == "region") } it should "round-trip default (empty) values cleanly" in { - val restored = objectMapper.readValue( - objectMapper.writeValueAsString(new HierarchySection), - classOf[HierarchySection] - ) - assert(restored.attributeName == "") + assert(checkRoundTrip(new HierarchySection, classOf[HierarchySection]).attributeName == "") } - // --------------------------------------------------------------------------- - // Annotations — required=true + @AutofillAttributeName + @NotNull - // --------------------------------------------------------------------------- - "HierarchySection#attributeName" should "carry @JsonProperty(required = true)" in { - val jp = classOf[HierarchySection] - .getDeclaredField("attributeName") - .getAnnotation(classOf[JsonProperty]) - assert(jp != null) - assert(jp.required) + assertAttrJsonPropertyRequired(classOf[HierarchySection]) } it should "carry @AutofillAttributeName" in { - val ann = classOf[HierarchySection] - .getDeclaredField("attributeName") - .getAnnotation(classOf[AutofillAttributeName]) - assert(ann != null) + assertAttrAutofill(classOf[HierarchySection], "@AutofillAttributeName expected") } it should "carry @NotNull with the canonical error message" in { - val ann = classOf[HierarchySection] - .getDeclaredField("attributeName") - .getAnnotation(classOf[NotNull]) - assert(ann != null) - assert(ann.message == "Attribute Name cannot be empty") + assertAttrNotNull(classOf[HierarchySection]) } - // --------------------------------------------------------------------------- - // Instance independence - // --------------------------------------------------------------------------- - it should "construct two independent instances (no static state shared)" in { val a = new HierarchySection val b = new HierarchySection diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/tablesChart/TablesConfigSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/tablesChart/TablesConfigSpec.scala index 48ea3fe1364..270af547c9f 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/tablesChart/TablesConfigSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/tablesChart/TablesConfigSpec.scala @@ -19,60 +19,31 @@ package org.apache.texera.amber.operator.visualization.tablesChart -import com.fasterxml.jackson.annotation.JsonProperty -import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.visualization.ChartConfigSpecSupport import org.scalatest.flatspec.AnyFlatSpec -import javax.validation.constraints.NotNull - -class TablesConfigSpec extends AnyFlatSpec { - - // --------------------------------------------------------------------------- - // Default state - // --------------------------------------------------------------------------- +class TablesConfigSpec extends AnyFlatSpec with ChartConfigSpecSupport { "TablesConfig" should "default attributeName to the empty string" in { - val c = new TablesConfig - assert(c.attributeName == "") + assert(new TablesConfig().attributeName == "") } - // --------------------------------------------------------------------------- - // Mutability — the @JsonProperty bag is `var`-based by design - // --------------------------------------------------------------------------- - it should "allow attributeName to be assigned post-construction" in { val c = new TablesConfig c.attributeName = "col-a" assert(c.attributeName == "col-a") } - // --------------------------------------------------------------------------- - // JSON round-trip preserves the field - // --------------------------------------------------------------------------- - - "TablesConfig JSON round-trip" should - "serialize and deserialize attributeName unchanged" in { + "TablesConfig JSON round-trip" should "serialize and deserialize attributeName unchanged" in { val original = new TablesConfig original.attributeName = "my-attr" - val json = objectMapper.writeValueAsString(original) - val restored = objectMapper.readValue(json, classOf[TablesConfig]) - assert(restored.attributeName == "my-attr") + assert(checkRoundTrip(original, classOf[TablesConfig]).attributeName == "my-attr") } it should "round-trip the default empty attributeName" in { - val original = new TablesConfig - val restored = objectMapper.readValue( - objectMapper.writeValueAsString(original), - classOf[TablesConfig] - ) - assert(restored.attributeName == "") + assert(checkRoundTrip(new TablesConfig, classOf[TablesConfig]).attributeName == "") } - // --------------------------------------------------------------------------- - // Independent instances — no static-field leakage - // --------------------------------------------------------------------------- - it should "construct two independent instances (no static state shared)" in { val a = new TablesConfig val b = new TablesConfig @@ -80,29 +51,18 @@ class TablesConfigSpec extends AnyFlatSpec { assert(b.attributeName == "", "second instance must not see first instance's mutation") } - // --------------------------------------------------------------------------- - // Annotations — verified via reflection (the Jackson + validation - // contract is what consumers actually depend on) - // --------------------------------------------------------------------------- - "TablesConfig#attributeName" should "carry @JsonProperty(required = true)" in { - val field = classOf[TablesConfig].getDeclaredField("attributeName") - val jp = field.getAnnotation(classOf[JsonProperty]) - assert(jp != null, "attributeName must carry @JsonProperty") - assert(jp.required, "attributeName must be marked required") + assertAttrJsonPropertyRequired(classOf[TablesConfig]) } it should "carry @NotNull (javax.validation contract)" in { - val field = classOf[TablesConfig].getDeclaredField("attributeName") - val notNull = field.getAnnotation(classOf[NotNull]) - assert(notNull != null, "attributeName must carry @NotNull for javax.validation") + assertAttrNotNull(classOf[TablesConfig]) } - it should - "carry @AutofillAttributeName (UI populates the dropdown from the input schema)" in { - val ann = classOf[TablesConfig] - .getDeclaredField("attributeName") - .getAnnotation(classOf[AutofillAttributeName]) - assert(ann != null, "@AutofillAttributeName must be present so the UI auto-populates") + it should "carry @AutofillAttributeName (UI populates the dropdown from the input schema)" in { + assertAttrAutofill( + classOf[TablesConfig], + "@AutofillAttributeName must be present so the UI auto-populates" + ) } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/pybuilder/DescriptorChecker.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/pybuilder/DescriptorChecker.scala index 97a725330f1..3224e6317e0 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/pybuilder/DescriptorChecker.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/pybuilder/DescriptorChecker.scala @@ -349,96 +349,93 @@ final class DescriptorChecker(private val rawInvalidText: String, private val ma * This is intentionally heuristic: the goal is to create a "usable enough" object graph for codegen * without knowing real business semantics. */ - private def bestEffortFillJsonPropertyDefaults( + private final case class JsonFieldSite( + owner: AnyRef, + declaringClass: Class[_], + field: Field, + jsonProp: JsonProperty, + typeEnv: TypeEnv, + resolvedType: Type, + currentValue: AnyRef, + remainingDepth: Int + ) + + private def walkJsonPropertyFields( rootDescriptor: AnyRef, recursionDepthLimit: Int - ): Unit = { + )(visitField: (JsonFieldSite, (AnyRef, Int) => Unit) => Unit): Unit = { val visitedIdentityHashes = mutable.Set.empty[Int] - def fillRecursively(currentObject: AnyRef, remainingDepth: Int): Unit = { + def visitObject(currentObject: AnyRef, remainingDepth: Int): Unit = { if (currentObject == null || remainingDepth < 0) return val objectId = System.identityHashCode(currentObject) - if (visitedIdentityHashes.contains(objectId)) return - visitedIdentityHashes += objectId + if (!visitedIdentityHashes.add(objectId)) return val currentTypeEnv = envFor(currentObject) - - walkHierarchy(currentObject.getClass) { declaringClassInHierarchy => - declaringClassInHierarchy.getDeclaredFields.foreach { declaredField => - if (!shouldSkipField(declaredField)) { - val jsonPropertyOpt = - jsonPropertyForFieldOrAccessors(declaringClassInHierarchy, declaredField) - jsonPropertyOpt.foreach { jsonPropertyAnn => - declaredField.setAccessible(true) - - val currentFieldValue = Try(declaredField.get(currentObject)).toOption.orNull - val defaultValueText = Option(jsonPropertyAnn.defaultValue()).getOrElse("").trim - val isRequired = jsonPropertyAnn.required() - - val resolvedFieldType = resolveType(declaredField.getGenericType, currentTypeEnv) - val needsInitialization = - (currentFieldValue == null) && (isRequired || defaultValueText.nonEmpty) - - val ensuredValue: AnyRef = - if (needsInitialization) { - val defaultValue = defaultValueForResolvedType( - targetType = resolvedFieldType, - defaultValueText = defaultValueText, - remainingDepth = remainingDepth, - typeEnvAtParent = currentTypeEnv - ) - if (defaultValue != null) { - trySet(currentObject, declaringClassInHierarchy, declaredField, defaultValue) - defaultValue - } else currentFieldValue - } else currentFieldValue - - val updatedValue = - ensureNonEmptyIfRequired( - owningInstance = currentObject, - declaringClass = declaringClassInHierarchy, - field = declaredField, - currentFieldValue = ensuredValue, - jsonPropertyAnn = jsonPropertyAnn, - resolvedFieldType = resolvedFieldType, - typeEnvAtField = currentTypeEnv, - remainingDepth = remainingDepth - ) - - recurseIntoValue(updatedValue, remainingDepth - 1, fillRecursively) + walkHierarchy(currentObject.getClass) { declaringClass => + declaringClass.getDeclaredFields.foreach { field => + if (!shouldSkipField(field)) { + jsonPropertyForFieldOrAccessors(declaringClass, field).foreach { jsonProp => + field.setAccessible(true) + val site = JsonFieldSite( + owner = currentObject, + declaringClass = declaringClass, + field = field, + jsonProp = jsonProp, + typeEnv = currentTypeEnv, + resolvedType = resolveType(field.getGenericType, currentTypeEnv), + currentValue = Try(field.get(currentObject)).toOption.orNull, + remainingDepth = remainingDepth + ) + visitField(site, visitObject) } } } } } - fillRecursively(rootDescriptor, recursionDepthLimit) + visitObject(rootDescriptor, recursionDepthLimit) } - private def ensureNonEmptyIfRequired( - owningInstance: AnyRef, - declaringClass: Class[_], - field: Field, - currentFieldValue: AnyRef, - jsonPropertyAnn: JsonProperty, - resolvedFieldType: Type, - typeEnvAtField: TypeEnv, - remainingDepth: Int - ): AnyRef = { - if (!jsonPropertyAnn.required() || remainingDepth <= 0) return currentFieldValue + private def bestEffortFillJsonPropertyDefaults( + rootDescriptor: AnyRef, + recursionDepthLimit: Int + ): Unit = { + walkJsonPropertyFields(rootDescriptor, recursionDepthLimit) { (site, recurse) => + val defaultValueText = Option(site.jsonProp.defaultValue()).getOrElse("").trim + val needsInitialization = + (site.currentValue == null) && (site.jsonProp.required() || defaultValueText.nonEmpty) + + val ensuredValue: AnyRef = + if (needsInitialization) { + val defaultValue = defaultValueForResolvedType( + targetType = site.resolvedType, + defaultValueText = defaultValueText, + remainingDepth = site.remainingDepth, + typeEnvAtParent = site.typeEnv + ) + if (defaultValue != null) { + trySet(site.owner, site.declaringClass, site.field, defaultValue) + defaultValue + } else site.currentValue + } else site.currentValue + + val updatedValue = ensureNonEmptyIfRequired(site, ensuredValue) + recurseIntoValue(updatedValue, site.remainingDepth - 1, recurse) + } + } + + private def ensureNonEmptyIfRequired(site: JsonFieldSite, currentFieldValue: AnyRef): AnyRef = { + if (!site.jsonProp.required() || site.remainingDepth <= 0) return currentFieldValue // If required and null, try to initialize collection containers too val ensuredNonNullValue: AnyRef = if (currentFieldValue != null) currentFieldValue else { - val rawFieldClass = typeToClass(resolvedFieldType).getOrElse(field.getType) - val defaultValue = defaultValueForResolvedType( - targetType = rawFieldClass, - defaultValueText = "", - remainingDepth = remainingDepth, - typeEnvAtParent = typeEnvAtField - ) - if (defaultValue != null) trySet(owningInstance, declaringClass, field, defaultValue) + val rawFieldClass = typeToClass(site.resolvedType).getOrElse(site.field.getType) + val defaultValue = + defaultValueForResolvedType(rawFieldClass, "", site.remainingDepth, site.typeEnv) + if (defaultValue != null) trySet(site.owner, site.declaringClass, site.field, defaultValue) defaultValue } @@ -446,38 +443,70 @@ final class DescriptorChecker(private val rawInvalidText: String, private val ma val runtimeValueClass = ensuredNonNullValue.getClass val elementTypeOpt = - elementTypeOfResolved(resolvedFieldType).map(et => resolveType(et, typeEnvAtField)) + elementTypeOfResolved(site.resolvedType).map(et => resolveType(et, site.typeEnv)) - def makeElementValue(): AnyRef = { - val elementType = elementTypeOpt.getOrElse(classOf[String]) + def makeElementValue(): AnyRef = defaultValueForResolvedType( - targetType = elementType, + targetType = elementTypeOpt.getOrElse(classOf[String]), defaultValueText = "", - remainingDepth = remainingDepth - 1, - typeEnvAtParent = typeEnvAtField + remainingDepth = site.remainingDepth - 1, + typeEnvAtParent = site.typeEnv ) - } if (isJavaList(runtimeValueClass)) { val javaList = ensuredNonNullValue.asInstanceOf[util.List[AnyRef]] if (javaList.isEmpty) { - val elementValue = makeElementValue() - if (elementValue != null) javaList.add(elementValue) + Option(makeElementValue()).foreach(javaList.add) } } else if (isScalaIterable(runtimeValueClass)) { val scalaIterable = ensuredNonNullValue.asInstanceOf[scala.collection.Iterable[Any]] if (scalaIterable.isEmpty) { - val elementValue = makeElementValue() - if (elementValue != null) - trySet(owningInstance, declaringClass, field, List(elementValue).asInstanceOf[AnyRef]) + Option(makeElementValue()).foreach(e => + trySet(site.owner, site.declaringClass, site.field, List(e).asInstanceOf[AnyRef]) + ) } } else if (runtimeValueClass.isArray && runtimeValueClass.getComponentType == classOf[String]) { val stringArray = ensuredNonNullValue.asInstanceOf[scala.Array[String]] if (stringArray.isEmpty) - trySet(owningInstance, declaringClass, field, scala.Array("x").asInstanceOf[AnyRef]) + trySet(site.owner, site.declaringClass, site.field, scala.Array("x").asInstanceOf[AnyRef]) } - Try(field.get(owningInstance)).toOption.orNull + Try(site.field.get(site.owner)).toOption.orNull + } + + private def defaultValueForClass( + rawClass: Class[_], + trimmedDefaultValueText: String, + remainingDepth: Int, + parameterizedOpt: Option[ParameterizedType], + typeEnvAtParent: TypeEnv + ): AnyRef = { + if (rawClass.isEnum) { + chooseEnumConstant(rawClass, trimmedDefaultValueText) + } else if (isJavaList(rawClass)) { + new util.ArrayList[AnyRef]() + } else if (isScalaIterable(rawClass)) { + List.empty[Any] + } else if ( + parameterizedOpt.isEmpty && rawClass.isArray && rawClass.getComponentType == classOf[String] + ) { + scala.Array.empty[String] + } else if (classOf[scala.Option[_]].isAssignableFrom(rawClass)) { + None + } else if ( + !rawClass.isInterface && !Modifier.isAbstract(rawClass.getModifiers) && remainingDepth > 0 + ) { + val instanceOpt = instantiateBestEffort(rawClass) + for { + pt <- parameterizedOpt + newInstance <- instanceOpt + } { + val newInstanceTypeEnv = + envForParameterizedInstance(rawClass, pt.getActualTypeArguments, typeEnvAtParent) + envByObj.put(newInstance, newInstanceTypeEnv) + } + instanceOpt.orNull + } else null } private def defaultValueForResolvedType( @@ -487,19 +516,12 @@ final class DescriptorChecker(private val rawInvalidText: String, private val ma typeEnvAtParent: TypeEnv ): AnyRef = { val trimmedDefaultValueText = Option(defaultValueText).getOrElse("").trim - val resolvedTargetType = resolveType(targetType, typeEnvAtParent) - - resolvedTargetType match { + resolveType(targetType, typeEnvAtParent) match { case rawClass: Class[_] => if (rawClass == classOf[String]) { if (trimmedDefaultValueText.nonEmpty) trimmedDefaultValueText else "x" } else if (rawClass == java.lang.Boolean.TYPE || rawClass == classOf[java.lang.Boolean]) { - val booleanValue = trimmedDefaultValueText.toLowerCase match { - case "true" => true - case "false" => false - case _ => false - } - java.lang.Boolean.valueOf(booleanValue) + java.lang.Boolean.valueOf(trimmedDefaultValueText.toLowerCase == "true") } else if (rawClass == java.lang.Integer.TYPE || rawClass == classOf[java.lang.Integer]) { java.lang.Integer.valueOf(Try(trimmedDefaultValueText.toInt).getOrElse(1)) } else if (rawClass == java.lang.Long.TYPE || rawClass == classOf[java.lang.Long]) { @@ -508,49 +530,28 @@ final class DescriptorChecker(private val rawInvalidText: String, private val ma java.lang.Double.valueOf(Try(trimmedDefaultValueText.toDouble).getOrElse(1.0d)) } else if (rawClass == java.lang.Float.TYPE || rawClass == classOf[java.lang.Float]) { java.lang.Float.valueOf(Try(trimmedDefaultValueText.toFloat).getOrElse(1.0f)) - } else if (rawClass.isEnum) { - chooseEnumConstant(rawClass, trimmedDefaultValueText) - } else if (isJavaList(rawClass)) { - new util.ArrayList[AnyRef]() - } else if (isScalaIterable(rawClass)) { - List.empty[Any] - } else if (rawClass.isArray && rawClass.getComponentType == classOf[String]) { - scala.Array.empty[String] - } else if (classOf[scala.Option[_]].isAssignableFrom(rawClass)) { - None - } else if ( - !rawClass.isInterface && !Modifier.isAbstract(rawClass.getModifiers) && remainingDepth > 0 - ) { - instantiateBestEffort(rawClass).orNull - } else null + } else { + defaultValueForClass( + rawClass, + trimmedDefaultValueText, + remainingDepth, + None, + typeEnvAtParent + ) + } case parameterizedType: ParameterizedType => - val rawClass = typeToClass(parameterizedType.getRawType).orNull - if (rawClass == null) return null - - if (rawClass.isEnum) { - chooseEnumConstant(rawClass, trimmedDefaultValueText) - } else if (isJavaList(rawClass)) { - new util.ArrayList[AnyRef]() - } else if (isScalaIterable(rawClass)) { - List.empty[Any] - } else if (classOf[scala.Option[_]].isAssignableFrom(rawClass)) { - None - } else if ( - !rawClass.isInterface && !Modifier.isAbstract(rawClass.getModifiers) && remainingDepth > 0 - ) { - val instanceOpt = instantiateBestEffort(rawClass) - instanceOpt.foreach { newInstance => - val newInstanceTypeEnv = - envForParameterizedInstance( - rawClass, - parameterizedType.getActualTypeArguments, - typeEnvAtParent - ) - envByObj.put(newInstance, newInstanceTypeEnv) - } - instanceOpt.orNull - } else null + typeToClass(parameterizedType.getRawType) match { + case Some(rawClass) => + defaultValueForClass( + rawClass, + trimmedDefaultValueText, + remainingDepth, + Some(parameterizedType), + typeEnvAtParent + ) + case None => null + } case _ => null @@ -621,118 +622,56 @@ final class DescriptorChecker(private val rawInvalidText: String, private val ma ): RawInvalidTextResult = { val changedMembers = mutable.ArrayBuffer.empty[String] val failedMembers = mutable.ArrayBuffer.empty[String] - val visitedIdentityHashes = mutable.Set.empty[Int] - def rawInvalidTextRecursively(currentObject: AnyRef, remainingDepth: Int): Unit = { - if (currentObject == null || remainingDepth < 0) return - val objectId = System.identityHashCode(currentObject) - if (visitedIdentityHashes.contains(objectId)) return - visitedIdentityHashes += objectId - - val currentTypeEnv = envFor(currentObject) + def hasStringElementType(site: JsonFieldSite): Boolean = + elementTypeOfResolved(site.resolvedType) + .map(et => resolveType(et, site.typeEnv)) + .flatMap(typeToClass) + .contains(classOf[String]) + + def injectFieldValue(site: JsonFieldSite, value: AnyRef, elementSuffix: String = ""): Unit = { + val jsonPropertyName = effectiveJsonPropName(site.jsonProp, fallback = site.field.getName) + val prefix = s"${site.declaringClass.getSimpleName}.${site.field.getName}" + if (Try(trySet(site.owner, site.declaringClass, site.field, value)).isSuccess) + changedMembers += s"""$prefix$elementSuffix(@JsonProperty("$jsonPropertyName"))""" + else + failedMembers += prefix + } - walkHierarchy(currentObject.getClass) { declaringClassInHierarchy => - declaringClassInHierarchy.getDeclaredFields.foreach { declaredField => - if (!shouldSkipField(declaredField)) { - val jsonPropertyOpt = - jsonPropertyForFieldOrAccessors(declaringClassInHierarchy, declaredField) - jsonPropertyOpt.foreach { jsonPropertyAnn => - declaredField.setAccessible(true) - val jsonPropertyName = - effectiveJsonPropName(jsonPropertyAnn, fallback = declaredField.getName) - - val resolvedFieldType = resolveType(declaredField.getGenericType, currentTypeEnv) - val rawFieldClass = typeToClass(resolvedFieldType).getOrElse(declaredField.getType) - val currentFieldValue = Try(declaredField.get(currentObject)).toOption.orNull - - if (rawFieldClass == classOf[String]) { - val didInjected = Try { - trySet( - currentObject, - declaringClassInHierarchy, - declaredField, - rawInvalidTextMarker - ) - }.isSuccess - if (didInjected) - changedMembers += s"""${declaringClassInHierarchy.getSimpleName}.${declaredField.getName}(@JsonProperty("$jsonPropertyName"))""" - else - failedMembers += s"${declaringClassInHierarchy.getSimpleName}.${declaredField.getName}" - - } else if (isJavaList(rawFieldClass)) { - val javaListValue = - if (currentFieldValue != null) currentFieldValue.asInstanceOf[util.List[AnyRef]] - else { - val newList = new util.ArrayList[AnyRef]() - Try(trySet(currentObject, declaringClassInHierarchy, declaredField, newList)) - newList - } - - val isElementTypeString = elementTypeOfResolved(resolvedFieldType) - .map(et => resolveType(et, currentTypeEnv)) - .flatMap(typeToClass) - .contains(classOf[String]) - - if (isElementTypeString) { - Try { javaListValue.clear(); javaListValue.add(rawInvalidTextMarker) } - changedMembers += s"""${declaringClassInHierarchy.getSimpleName}.${declaredField.getName}[0](@JsonProperty("$jsonPropertyName"))""" - } else { - javaListValue.asScala.foreach(elementObj => - rawInvalidTextRecursively(elementObj, remainingDepth - 1) - ) - } - - } else if (isScalaIterable(rawFieldClass)) { - val isElementTypeString = elementTypeOfResolved(resolvedFieldType) - .map(et => resolveType(et, currentTypeEnv)) - .flatMap(typeToClass) - .contains(classOf[String]) - - if (isElementTypeString) { - val didSetList = - Try( - trySet( - currentObject, - declaringClassInHierarchy, - declaredField, - List(rawInvalidTextMarker).asInstanceOf[AnyRef] - ) - ).isSuccess - if (didSetList) - changedMembers += s"""${declaringClassInHierarchy.getSimpleName}.${declaredField.getName}[0](@JsonProperty("$jsonPropertyName"))""" - else - failedMembers += s"${declaringClassInHierarchy.getSimpleName}.${declaredField.getName}" - } else { - recurseIntoValue(currentFieldValue, remainingDepth - 1, rawInvalidTextRecursively) - } - - } else if ( - rawFieldClass.isArray && rawFieldClass.getComponentType == classOf[String] - ) { - val didInjectedArray = - Try( - trySet( - currentObject, - declaringClassInHierarchy, - declaredField, - scala.Array(rawInvalidTextMarker).asInstanceOf[AnyRef] - ) - ).isSuccess - if (didInjectedArray) - changedMembers += s"""${declaringClassInHierarchy.getSimpleName}.${declaredField.getName}[0](@JsonProperty("$jsonPropertyName"))""" - else - failedMembers += s"${declaringClassInHierarchy.getSimpleName}.${declaredField.getName}" - - } else { - recurseIntoValue(currentFieldValue, remainingDepth - 1, rawInvalidTextRecursively) - } - } + walkJsonPropertyFields(rootDescriptor, recursionDepthLimit) { (site, recurse) => + val rawFieldClass = typeToClass(site.resolvedType).getOrElse(site.field.getType) + + if (rawFieldClass == classOf[String]) { + injectFieldValue(site, rawInvalidTextMarker) + } else if (isJavaList(rawFieldClass)) { + val javaListValue = + if (site.currentValue != null) site.currentValue.asInstanceOf[util.List[AnyRef]] + else { + val newList = new util.ArrayList[AnyRef]() + Try(trySet(site.owner, site.declaringClass, site.field, newList)) + newList } + + if (hasStringElementType(site)) { + Try { javaListValue.clear(); javaListValue.add(rawInvalidTextMarker) } + val jsonPropertyName = effectiveJsonPropName(site.jsonProp, fallback = site.field.getName) + changedMembers += s"""${site.declaringClass.getSimpleName}.${site.field.getName}[0](@JsonProperty("$jsonPropertyName"))""" + } else { + javaListValue.asScala.foreach(elementObj => recurse(elementObj, site.remainingDepth - 1)) + } + } else if (isScalaIterable(rawFieldClass)) { + if (hasStringElementType(site)) { + injectFieldValue(site, List(rawInvalidTextMarker).asInstanceOf[AnyRef], "[0]") + } else { + recurseIntoValue(site.currentValue, site.remainingDepth - 1, recurse) } + } else if (rawFieldClass.isArray && rawFieldClass.getComponentType == classOf[String]) { + injectFieldValue(site, scala.Array(rawInvalidTextMarker).asInstanceOf[AnyRef], "[0]") + } else { + recurseIntoValue(site.currentValue, site.remainingDepth - 1, recurse) } } - rawInvalidTextRecursively(rootDescriptor, recursionDepthLimit) RawInvalidTextResult(changedMembers.distinct.toSeq, failedMembers.distinct.toSeq) } @@ -865,27 +804,20 @@ final class DescriptorChecker(private val rawInvalidText: String, private val ma ): Unit = { if (value == null || remainingDepth < 0) return - value match { - case javaList: util.List[_] => - javaList.asScala.foreach { - case elementRef: AnyRef => visitFn(elementRef, remainingDepth) - case _ => () - } - - case scalaIterable: scala.collection.Iterable[_] => - scalaIterable.foreach { - case elementRef: AnyRef => visitFn(elementRef, remainingDepth) - case _ => () - } - - case arrayValue: scala.Array[_] => - arrayValue.foreach { + val elements: Option[Iterator[_]] = value match { + case javaList: util.List[_] => Some(javaList.asScala.iterator) + case scalaIterable: scala.collection.Iterable[_] => Some(scalaIterable.iterator) + case arrayValue: scala.Array[_] => Some(arrayValue.iterator) + case _ => None + } + elements match { + case Some(it) => + it.foreach { case elementRef: AnyRef => visitFn(elementRef, remainingDepth) case _ => () } - - case otherValue => - val runtimeClass = otherValue.getClass + case None => + val runtimeClass = value.getClass val isLeafValue = runtimeClass.isPrimitive || runtimeClass == classOf[String] || @@ -896,7 +828,7 @@ final class DescriptorChecker(private val rawInvalidText: String, private val ma runtimeClass.getName.startsWith("javax.") || runtimeClass.getName.startsWith("scala.") - if (!isLeafValue) visitFn(otherValue, remainingDepth) + if (!isLeafValue) visitFn(value, remainingDepth) } } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/util/ArrowUtilsSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/util/ArrowUtilsSpec.scala index 665dc4fc488..16d429ab210 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/util/ArrowUtilsSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/util/ArrowUtilsSpec.scala @@ -21,15 +21,16 @@ package org.apache.texera.amber.util import org.apache.texera.amber.core.tuple.AttributeTypeUtils.AttributeTypeException import org.apache.texera.amber.core.tuple.{AttributeType, LargeBinary, Schema, Tuple} -import org.apache.arrow.memory.{BufferAllocator, RootAllocator} +import org.apache.arrow.memory.RootAllocator import org.apache.arrow.vector.VectorSchemaRoot -import org.apache.arrow.vector.types.pojo.{ArrowType, Field, FieldType} +import org.apache.arrow.vector.types.pojo.{ArrowType, Field, FieldType, Schema => ArrowSchema} import org.apache.arrow.vector.types.{DateUnit, FloatingPointPrecision, IntervalUnit, TimeUnit} import org.scalatest.flatspec.AnyFlatSpec import java.sql.Timestamp import java.util import scala.jdk.CollectionConverters.IterableHasAsJava +import scala.util.Using class ArrowUtilsSpec extends AnyFlatSpec { val unsignedShortInt = new ArrowType.Int(16, false) @@ -65,6 +66,20 @@ class ArrowUtilsSpec extends AnyFlatSpec { ).toList.asJava ) + private def sampleTuple(longValue: java.lang.Long, time: Timestamp, text: String): Tuple = + Tuple + .builder(texeraSchema) + .addSequentially(Array(Int.box(2), longValue, Boolean.box(true), Double.box(1.1), time, text)) + .build() + + private def withVectorRoot(schema: ArrowSchema)(test: VectorSchemaRoot => Unit): Unit = + Using.Manager { use => + val allocator = use(new RootAllocator()) + val root = use(VectorSchemaRoot.create(schema, allocator)) + root.allocateNew() + test(root) + }.get + behavior of "ArrowUtils" it should "convert to AttributeTypes correctly" in { @@ -84,7 +99,6 @@ class ArrowUtilsSpec extends AnyFlatSpec { assert(ArrowUtils.toAttributeType(double) == AttributeType.DOUBLE) assert(ArrowUtils.toAttributeType(half) == AttributeType.DOUBLE) - assert(ArrowUtils.toAttributeType(timestamp) == AttributeType.TIMESTAMP) assert(ArrowUtils.toAttributeType(timestamp) == AttributeType.TIMESTAMP) assert(ArrowUtils.toAttributeType(string) == AttributeType.STRING) @@ -109,24 +123,16 @@ class ArrowUtilsSpec extends AnyFlatSpec { } it should "raise AttributeTypeException when converting unsupported types" in { - assertThrows[AttributeTypeException] { - ArrowUtils.toAttributeType(new ArrowType.Null) - } - - assertThrows[AttributeTypeException] { - ArrowUtils.toAttributeType(new ArrowType.Date(DateUnit.DAY)) - } - - assertThrows[AttributeTypeException] { - ArrowUtils.toAttributeType(new ArrowType.Map(true)) - } - - assertThrows[AttributeTypeException] { - ArrowUtils.toAttributeType(new ArrowType.List) - } - - assertThrows[AttributeTypeException] { - ArrowUtils.toAttributeType(new ArrowType.Interval(IntervalUnit.DAY_TIME)) + Seq[ArrowType]( + new ArrowType.Null, + new ArrowType.Date(DateUnit.DAY), + new ArrowType.Map(true), + new ArrowType.List, + new ArrowType.Interval(IntervalUnit.DAY_TIME) + ).foreach { arrowType => + withClue(s"${arrowType}: ") { + assertThrows[AttributeTypeException](ArrowUtils.toAttributeType(arrowType)) + } } } @@ -144,114 +150,70 @@ class ArrowUtilsSpec extends AnyFlatSpec { it should "set Arrow Fields from Texera Tuple correctly" in { - val tuple = Tuple - .builder(texeraSchema) - .addSequentially( - Array( - Int.box(2), - Long.box(1L), - Boolean.box(true), - Double.box(1.1), - // Stated as a wall clock, which is what a Texera TIMESTAMP holds. Built - // from an epoch instead, the wall clock would be whichever one the - // machine's zone gives that instant, and what gets stored below would - // move with it. - Timestamp.valueOf("1970-01-01 00:00:10"), - "hello world" - ) - ) - .build() - val allocator: BufferAllocator = new RootAllocator() - val vectorSchemaRoot = VectorSchemaRoot.create(arrowSchema, allocator) - vectorSchemaRoot.allocateNew() + // Texera holds a wall clock. Constructing it from an epoch would make + // the stored value depend on the machine's time zone. + val tuple = sampleTuple(Long.box(1L), Timestamp.valueOf("1970-01-01 00:00:10"), "hello world") - val rowCount = vectorSchemaRoot.getRowCount - val index = 1 + withVectorRoot(arrowSchema) { vectorSchemaRoot => + val rowCount = vectorSchemaRoot.getRowCount + val index = 1 - // set Tuple into the Vectors - ArrowUtils.setTexeraTuple(tuple, index, vectorSchemaRoot) + // set Tuple into the Vectors + ArrowUtils.setTexeraTuple(tuple, index, vectorSchemaRoot) - assert(vectorSchemaRoot.getVector(0).getObject(index).asInstanceOf[Int] == 2) - assert(vectorSchemaRoot.getVector(1).getObject(index).asInstanceOf[Long] == 1L) - assert(vectorSchemaRoot.getVector(2).getObject(index).asInstanceOf[Boolean] == true) - assert(vectorSchemaRoot.getVector(3).getObject(index).asInstanceOf[Double] == 1.1) + assert(vectorSchemaRoot.getVector(0).getObject(index).asInstanceOf[Int] == 2) + assert(vectorSchemaRoot.getVector(1).getObject(index).asInstanceOf[Long] == 1L) + assert(vectorSchemaRoot.getVector(2).getObject(index).asInstanceOf[Boolean] == true) + assert(vectorSchemaRoot.getVector(3).getObject(index).asInstanceOf[Double] == 1.1) - // The field carries no zone, so the vector holds the wall clock above and - // hands it back as one: ten seconds past the epoch, on a server anywhere. - assert( - vectorSchemaRoot.getVector(4).getObject(index) == - java.time.LocalDateTime.of(1970, 1, 1, 0, 0, 10) - ) - - // the arrow storage type of string is Text - assert( - vectorSchemaRoot.getVector(5).getObject(index) == - new org.apache.arrow.vector.util.Text("hello world") - ) + // The field carries no zone, so the vector holds the wall clock above and + // hands it back as one: ten seconds past the epoch, on a server anywhere. + assert( + vectorSchemaRoot.getVector(4).getObject(index) == + java.time.LocalDateTime.of(1970, 1, 1, 0, 0, 10) + ) - // should not have more vectors created. - assertThrows[IllegalArgumentException](vectorSchemaRoot.getVector(7).getObject(index)) + // the arrow storage type of string is Text + assert( + vectorSchemaRoot.getVector(5).getObject(index) == + new org.apache.arrow.vector.util.Text("hello world") + ) - // other fields at other untouched indices should be null - assert(vectorSchemaRoot.getVector(0).getObject(index + 1) == null) + // should not have more vectors created. + assertThrows[IllegalArgumentException](vectorSchemaRoot.getVector(7).getObject(index)) - // now the rowCount should be incremented by 1 - assert(vectorSchemaRoot.getRowCount == rowCount + 1) + // other fields at other untouched indices should be null + assert(vectorSchemaRoot.getVector(0).getObject(index + 1) == null) + // now the rowCount should be incremented by 1 + assert(vectorSchemaRoot.getRowCount == rowCount + 1) + } } it should "get Texera Tuple from Arrow Fields correctly" in { - val tuple = Tuple - .builder(texeraSchema) - .addSequentially( - Array( - Int.box(2), - Long.box(1L), - Boolean.box(true), - Double.box(1.1), - new Timestamp(10000L), - "hello world" - ) - ) - .build() - val allocator: BufferAllocator = new RootAllocator() - val vectorSchemaRoot = VectorSchemaRoot.create(arrowSchema, allocator) - vectorSchemaRoot.allocateNew() - - // set Tuple into the Vectors - ArrowUtils.appendTexeraTuple(tuple, vectorSchemaRoot) + val tuple = sampleTuple(Long.box(1L), new Timestamp(10000L), "hello world") - // get the Tuple from the Vectors - assert(ArrowUtils.getTexeraTuple(0, vectorSchemaRoot) == tuple) + withVectorRoot(arrowSchema) { vectorSchemaRoot => + // set Tuple into the Vectors + ArrowUtils.appendTexeraTuple(tuple, vectorSchemaRoot) + // get the Tuple from the Vectors + assert(ArrowUtils.getTexeraTuple(0, vectorSchemaRoot) == tuple) + } } it should "get Texera Tuple from Arrow Fields with null values correctly" in { - val tuple = Tuple - .builder(texeraSchema) - .addSequentially( - Array( - Int.box(2), - null, - Boolean.box(true), - Double.box(1.1), - null, - null - ) - ) - .build() - val allocator: BufferAllocator = new RootAllocator() - val vectorSchemaRoot = VectorSchemaRoot.create(arrowSchema, allocator) - vectorSchemaRoot.allocateNew() - - // set Tuple into the Vectors - ArrowUtils.appendTexeraTuple(tuple, vectorSchemaRoot) + val tuple = sampleTuple(null, null, null) - // get the Tuple from the Vectors - assert(ArrowUtils.getTexeraTuple(0, vectorSchemaRoot) == tuple) + withVectorRoot(arrowSchema) { vectorSchemaRoot => + // set Tuple into the Vectors + ArrowUtils.appendTexeraTuple(tuple, vectorSchemaRoot) + // get the Tuple from the Vectors + assert(ArrowUtils.getTexeraTuple(0, vectorSchemaRoot) == tuple) + } } it should "convert from AttributeType to ArrowType for LARGE_BINARY correctly" in { @@ -325,22 +287,19 @@ class ArrowUtilsSpec extends AnyFlatSpec { ) .build() - val allocator: BufferAllocator = new RootAllocator() - val arrowSchema = ArrowUtils.fromTexeraSchema(texeraSchemaWithLargeBinary) - val vectorSchemaRoot = VectorSchemaRoot.create(arrowSchema, allocator) - vectorSchemaRoot.allocateNew() - - // Set Tuple into the Vectors - ArrowUtils.appendTexeraTuple(tuple, vectorSchemaRoot) + withVectorRoot(ArrowUtils.fromTexeraSchema(texeraSchemaWithLargeBinary)) { vectorSchemaRoot => + // Set Tuple into the Vectors + ArrowUtils.appendTexeraTuple(tuple, vectorSchemaRoot) - // Verify the LARGE_BINARY is stored as string (URI) in Arrow - val storedValue = vectorSchemaRoot.getVector(0).getObject(0) - assert(storedValue.toString == "s3://test-bucket/path/to/object") + // Verify the LARGE_BINARY is stored as string (URI) in Arrow + val storedValue = vectorSchemaRoot.getVector(0).getObject(0) + assert(storedValue.toString == "s3://test-bucket/path/to/object") - // Get the Tuple from the Vectors - val retrievedTuple = ArrowUtils.getTexeraTuple(0, vectorSchemaRoot) - assert(retrievedTuple.getField[LargeBinary](0) == largeBinary) - assert(retrievedTuple.getField[String](1) == "regular string value") + // Get the Tuple from the Vectors + val retrievedTuple = ArrowUtils.getTexeraTuple(0, vectorSchemaRoot) + assert(retrievedTuple.getField[LargeBinary](0) == largeBinary) + assert(retrievedTuple.getField[String](1) == "regular string value") + } } it should "handle null LARGE_BINARY values correctly" in { @@ -356,20 +315,17 @@ class ArrowUtilsSpec extends AnyFlatSpec { ) .build() - val allocator: BufferAllocator = new RootAllocator() - val arrowSchema = ArrowUtils.fromTexeraSchema(texeraSchemaWithLargeBinary) - val vectorSchemaRoot = VectorSchemaRoot.create(arrowSchema, allocator) - vectorSchemaRoot.allocateNew() - - // Set Tuple into the Vectors - ArrowUtils.appendTexeraTuple(tuple, vectorSchemaRoot) + withVectorRoot(ArrowUtils.fromTexeraSchema(texeraSchemaWithLargeBinary)) { vectorSchemaRoot => + // Set Tuple into the Vectors + ArrowUtils.appendTexeraTuple(tuple, vectorSchemaRoot) - // Verify null is stored correctly - assert(vectorSchemaRoot.getVector(0).getObject(0) == null) + // Verify null is stored correctly + assert(vectorSchemaRoot.getVector(0).getObject(0) == null) - // Get the Tuple from the Vectors - val retrievedTuple = ArrowUtils.getTexeraTuple(0, vectorSchemaRoot) - assert(retrievedTuple.getField[LargeBinary](0) == null) + // Get the Tuple from the Vectors + val retrievedTuple = ArrowUtils.getTexeraTuple(0, vectorSchemaRoot) + assert(retrievedTuple.getField[LargeBinary](0) == null) + } } it should "round-trip LARGE_BINARY schema conversion correctly" in {