From 2a319878af561b45694242c3d7d9ec2be5b337f0 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Thu, 19 Nov 2026 01:28:54 -0800 Subject: [PATCH] refactor(pyamber, amber): remove redundant Python code --- .../handlers/actorcommand/backpressure_handler.py | 2 -- .../architecture/managers/exception_manager.py | 2 +- .../core/architecture/rpc/async_rpc_client.py | 2 +- .../sendsemantics/one_to_one_partitioner.py | 4 ++-- .../main/python/core/models/internal_marker.py | 2 -- .../python/core/models/schema/attribute_type.py | 4 ++-- .../core/models/schema/attribute_type_utils.py | 3 +-- .../src/main/python/core/models/schema/schema.py | 4 ++-- amber/src/main/python/core/models/table.py | 3 +-- amber/src/main/python/core/proxy/proxy_client.py | 2 +- amber/src/main/python/core/proxy/proxy_server.py | 15 ++++++--------- amber/src/main/python/core/runnables/heartbeat.py | 3 +-- .../main/python/core/storage/document_factory.py | 14 ++++++-------- .../storage/iceberg/iceberg_catalog_instance.py | 5 ++--- .../python/core/storage/iceberg/iceberg_utils.py | 13 ++++--------- .../core/storage/model/buffered_item_writer.py | 4 ---- .../storage/model/readonly_virtual_document.py | 6 ------ .../python/core/util/console_message/timestamp.py | 4 +--- .../main/python/core/util/expression_evaluator.py | 4 ++-- .../src/main/python/pytexera/udf/udf_operator.py | 8 -------- .../aiassistant/type_annotation_visitor.py | 11 +++++++---- 21 files changed, 40 insertions(+), 75 deletions(-) diff --git a/amber/src/main/python/core/architecture/handlers/actorcommand/backpressure_handler.py b/amber/src/main/python/core/architecture/handlers/actorcommand/backpressure_handler.py index 764348b1d13..55c9d4f410a 100644 --- a/amber/src/main/python/core/architecture/handlers/actorcommand/backpressure_handler.py +++ b/amber/src/main/python/core/architecture/handlers/actorcommand/backpressure_handler.py @@ -59,5 +59,3 @@ def __call__( ), ) ) - - return None diff --git a/amber/src/main/python/core/architecture/managers/exception_manager.py b/amber/src/main/python/core/architecture/managers/exception_manager.py index f3ea4dd4bf1..f54c2e120ec 100644 --- a/amber/src/main/python/core/architecture/managers/exception_manager.py +++ b/amber/src/main/python/core/architecture/managers/exception_manager.py @@ -23,7 +23,7 @@ class ExceptionManager: def __init__(self): self.exc_info: Optional[ExceptionInfo] = None - self.exc_info_history: List[ExceptionInfo] = list() + self.exc_info_history: List[ExceptionInfo] = [] def set_exception_info(self, exc_info: ExceptionInfo) -> None: self.exc_info = exc_info diff --git a/amber/src/main/python/core/architecture/rpc/async_rpc_client.py b/amber/src/main/python/core/architecture/rpc/async_rpc_client.py index 5da6c7d3e37..f3e07fa7f63 100644 --- a/amber/src/main/python/core/architecture/rpc/async_rpc_client.py +++ b/amber/src/main/python/core/architecture/rpc/async_rpc_client.py @@ -57,7 +57,7 @@ def __init__(self, output_queue: InternalQueue, context: Context): self._context = context self._output_queue = output_queue self._send_sequences: Dict[ActorVirtualIdentity, int] = defaultdict(int) - self._unfulfilled_promises: Dict[(ActorVirtualIdentity, int), Future] = dict() + self._unfulfilled_promises: Dict[(ActorVirtualIdentity, int), Future] = {} # TODO: is this correct? self._coordinator_service_stub = CoordinatorServiceStub("") rpc_context = AsyncRpcContext( diff --git a/amber/src/main/python/core/architecture/sendsemantics/one_to_one_partitioner.py b/amber/src/main/python/core/architecture/sendsemantics/one_to_one_partitioner.py index d6a568be6a3..2b59072219c 100644 --- a/amber/src/main/python/core/architecture/sendsemantics/one_to_one_partitioner.py +++ b/amber/src/main/python/core/architecture/sendsemantics/one_to_one_partitioner.py @@ -35,7 +35,7 @@ class OneToOnePartitioner(Partitioner): def __init__(self, partitioning: OneToOnePartitioning, worker_id: str): super().__init__(set_one_of(Partitioning, partitioning)) self.batch_size = partitioning.batch_size - self.batch: list[Tuple] = list() + self.batch: list[Tuple] = [] for channel in partitioning.channels: if channel.from_worker_id.name == worker_id: self.receiver = channel.to_worker_id @@ -72,4 +72,4 @@ def flush_state( @overrides def reset(self) -> None: - self.batch = list() + self.batch = [] diff --git a/amber/src/main/python/core/models/internal_marker.py b/amber/src/main/python/core/models/internal_marker.py index 209c33c5101..a8e9438ec16 100644 --- a/amber/src/main/python/core/models/internal_marker.py +++ b/amber/src/main/python/core/models/internal_marker.py @@ -25,8 +25,6 @@ class InternalMarker: Markers preserve ordering and signal input lifecycle changes. """ - pass - @dataclass(frozen=True) class PortMarker(InternalMarker): diff --git a/amber/src/main/python/core/models/schema/attribute_type.py b/amber/src/main/python/core/models/schema/attribute_type.py index f8f54e0fae8..fd7e4056580 100644 --- a/amber/src/main/python/core/models/schema/attribute_type.py +++ b/amber/src/main/python/core/models/schema/attribute_type.py @@ -124,7 +124,7 @@ def _local_wall_clock(moment): # machine's standard offset. Before 1900 the engine reads every zone at # its standard offset, where a named zone would give its local mean time. zone = _named_local_zone() - utc = moment.astimezone(datetime.timezone.utc).replace(tzinfo=None) + utc = moment.astimezone(datetime.UTC).replace(tzinfo=None) if utc < datetime.datetime(1900, 1, 1): return utc + _standard_offset(zone) if zone is not None: @@ -146,7 +146,7 @@ def _named_local_zone(): return _zone_by_name(os.environ.get("TZ")) -@functools.lru_cache(maxsize=None) +@functools.cache def _zone_by_name(tz): # TZ names the zone when it is set, and /etc/localtime links to it when # not. A zone neither of them names is left to localtime. diff --git a/amber/src/main/python/core/models/schema/attribute_type_utils.py b/amber/src/main/python/core/models/schema/attribute_type_utils.py index 3918fdfc342..86fdc416642 100644 --- a/amber/src/main/python/core/models/schema/attribute_type_utils.py +++ b/amber/src/main/python/core/models/schema/attribute_type_utils.py @@ -49,8 +49,7 @@ def detect_attribute_type_from_arrow_field(field: pa.Field) -> AttributeType: if is_large_binary: return AttributeType.LARGE_BINARY - else: - return FROM_ARROW_MAPPING[field.type.id] + return FROM_ARROW_MAPPING[field.type.id] def create_arrow_field_with_metadata( diff --git a/amber/src/main/python/core/models/schema/schema.py b/amber/src/main/python/core/models/schema/schema.py index d349807ab79..b76d9839cb4 100644 --- a/amber/src/main/python/core/models/schema/schema.py +++ b/amber/src/main/python/core/models/schema/schema.py @@ -66,7 +66,7 @@ def add(self, attr_name: str, attr_type: AttributeType) -> None: :return: """ if attr_name in self._name_type_mapping: - raise KeyError(f"Adding a duplicated attribute {repr(attr_name)}.") + raise KeyError(f"Adding a duplicated attribute {attr_name!r}.") self._name_type_mapping[attr_name] = attr_type def _from_raw_schema(self, raw_schema: Mapping[str, str]) -> None: @@ -144,7 +144,7 @@ def __eq__(self, other: object) -> bool: def __str__(self) -> str: content = ",\n".join( - f"({index}){repr(attr_name)} -> {attr_type}" + f"({index}){attr_name!r} -> {attr_type}" for index, (attr_name, attr_type) in enumerate(self.as_key_value_pairs(), 0) ) return f"Schema[\n{content}\n]" diff --git a/amber/src/main/python/core/models/table.py b/amber/src/main/python/core/models/table.py index 708e0580f86..40b613f98b7 100644 --- a/amber/src/main/python/core/models/table.py +++ b/amber/src/main/python/core/models/table.py @@ -95,8 +95,7 @@ def as_tuples(self) -> Iterator[Tuple]: def __eq__(self, other: "Table") -> bool: if isinstance(other, Table): return all(a == b for a, b in zip(self.as_tuples(), other.as_tuples())) - else: - return super().__eq__(other).all() + return super().__eq__(other).all() def all_output_to_tuple(output) -> Iterator[Tuple]: diff --git a/amber/src/main/python/core/proxy/proxy_client.py b/amber/src/main/python/core/proxy/proxy_client.py index 4f0055e5391..09b35d62df3 100644 --- a/amber/src/main/python/core/proxy/proxy_client.py +++ b/amber/src/main/python/core/proxy/proxy_client.py @@ -50,7 +50,7 @@ def __init__( def call_action( self, action_name: str, - payload: bytes = bytes(), + payload: bytes = b"", options: Optional[FlightCallOptions] = None, ) -> bytes: """ diff --git a/amber/src/main/python/core/proxy/proxy_server.py b/amber/src/main/python/core/proxy/proxy_server.py index 0dee156bbf0..c45402a2021 100644 --- a/amber/src/main/python/core/proxy/proxy_server.py +++ b/amber/src/main/python/core/proxy/proxy_server.py @@ -112,14 +112,14 @@ def __init__( if port is None: port = get_free_local_port() location = f"{scheme}://{host}:{port}" - super(ProxyServer, self).__init__(location) + super().__init__(location) logger.debug(f"Serving on {location}") self._port_number = port # action name to callable map, will contain registered actions, # identified by action name. - self._procedures: Dict[str, Tuple[Callable, str]] = dict() + self._procedures: Dict[str, Tuple[Callable, str]] = {} # register heartbeat, this is the default action for the client to # check the aliveness of the server. @@ -213,7 +213,7 @@ def list_actions(self, context: ServerCallContext) -> Iterator[Tuple[str, str]]: :param context: server context, containing information of middlewares. :return: iterator of (action_name, action_description) pairs. """ - return map(lambda x: (x[0], x[1][1]), self._procedures.items()) + return ((k, v[1]) for k, v in self._procedures.items()) @overrides(check_signature=False) def do_action(self, context: ServerCallContext, action: Action) -> Iterator[Result]: @@ -234,14 +234,11 @@ def do_action(self, context: ServerCallContext, action: Action) -> Iterator[Resu if action_name in self._procedures: procedure, _ = self._procedures.get(action_name) if not action: - raise KeyError("Unknown action {!r}".format(action_name)) + raise KeyError(f"Unknown action {action_name!r}") payload = action.body.to_pybytes() # invoke the action - if payload: - result = procedure(payload) - else: - result = procedure() + result = procedure(payload) if payload else procedure() # serialize the result if isinstance(result, bytes): @@ -254,7 +251,7 @@ def do_action(self, context: ServerCallContext, action: Action) -> Iterator[Resu if action_name == "shutdown": threading.Thread(target=self.graceful_shutdown).start() else: - raise KeyError("Unknown action {!r}".format(action_name)) + raise KeyError(f"Unknown action {action_name!r}") @logger.catch(reraise=True) def register(self, name: str, action: Callable, description: str = "") -> None: diff --git a/amber/src/main/python/core/runnables/heartbeat.py b/amber/src/main/python/core/runnables/heartbeat.py index 941782f6998..1535d5b6d28 100644 --- a/amber/src/main/python/core/runnables/heartbeat.py +++ b/amber/src/main/python/core/runnables/heartbeat.py @@ -110,7 +110,6 @@ def stop(self): os.kill(child.pid, signal.SIGKILL) except Exception as e: logger.warning( - "Exception during process termination " - f"PID {str(child.pid)}: {e} " + f"Exception during process termination PID {child.pid!s}: {e} " ) os.kill(os.getpid(), signal.SIGTERM) diff --git a/amber/src/main/python/core/storage/document_factory.py b/amber/src/main/python/core/storage/document_factory.py index 0db1b413922..d29a1f7f802 100644 --- a/amber/src/main/python/core/storage/document_factory.py +++ b/amber/src/main/python/core/storage/document_factory.py @@ -104,10 +104,9 @@ def create_document(uri: str, schema: Schema) -> VirtualDocument: warehouse, ) - else: - raise NotImplementedError( - f"Unsupported URI scheme: {parsed_uri.scheme} for creating the document" - ) + raise NotImplementedError( + f"Unsupported URI scheme: {parsed_uri.scheme} for creating the document" + ) @staticmethod def document_exists(uri: str) -> bool: @@ -166,7 +165,6 @@ def open_document(uri: str) -> typing.Tuple[VirtualDocument, Optional[Schema]]: ) return document, amber_schema - else: - raise NotImplementedError( - f"Unsupported URI scheme: {parsed_uri.scheme} for opening the document" - ) + raise NotImplementedError( + f"Unsupported URI scheme: {parsed_uri.scheme} for opening the document" + ) diff --git a/amber/src/main/python/core/storage/iceberg/iceberg_catalog_instance.py b/amber/src/main/python/core/storage/iceberg/iceberg_catalog_instance.py index 61987c2594d..56ff394ca52 100644 --- a/amber/src/main/python/core/storage/iceberg/iceberg_catalog_instance.py +++ b/amber/src/main/python/core/storage/iceberg/iceberg_catalog_instance.py @@ -59,7 +59,7 @@ def get_instance(cls, warehouse: Optional[str] = None) -> Catalog: StorageConfig.ICEBERG_POSTGRES_CATALOG_PASSWORD, ) return cls._catalogs[cls._POSTGRES_KEY] - elif catalog_type == "rest": + if catalog_type == "rest": key = warehouse or StorageConfig.ICEBERG_REST_CATALOG_WAREHOUSE_NAME if key not in cls._catalogs: cls._catalogs[key] = create_rest_catalog( @@ -68,8 +68,7 @@ def get_instance(cls, warehouse: Optional[str] = None) -> Catalog: StorageConfig.ICEBERG_REST_CATALOG_URI, ) return cls._catalogs[key] - else: - raise ValueError(f"Unsupported catalog type: {catalog_type}") + raise ValueError(f"Unsupported catalog type: {catalog_type}") @classmethod def replace_instance(cls, catalog: Catalog, warehouse: Optional[str] = None): diff --git a/amber/src/main/python/core/storage/iceberg/iceberg_utils.py b/amber/src/main/python/core/storage/iceberg/iceberg_utils.py index 63148598c35..0f9c9db59f3 100644 --- a/amber/src/main/python/core/storage/iceberg/iceberg_utils.py +++ b/amber/src/main/python/core/storage/iceberg/iceberg_utils.py @@ -32,7 +32,6 @@ from typing import Optional, Iterable from pyiceberg import types as iceberg_types -import core import core.models from core.models import ArrowTableTupleProvider, Tuple from core.models.schema.attribute_type import AttributeType, TO_ARROW_MAPPING @@ -242,11 +241,9 @@ def create_rest_catalog( # server, whose I/O goes through S3FileIO, not the local filesystem. return load_catalog( catalog_name, - **{ - "type": "rest", - "uri": rest_uri, - "warehouse": warehouse_name, - }, + type="rest", + uri=rest_uri, + warehouse=warehouse_name, ) @@ -278,14 +275,12 @@ def create_table( if catalog.table_exists(identifier) and override_if_exists: catalog.drop_table(identifier) - table = catalog.create_table( + return catalog.create_table( identifier=identifier, schema=table_schema, partition_spec=UNPARTITIONED_PARTITION_SPEC, ) - return table - def load_table_metadata( catalog: Catalog, table_namespace: str, table_name: str diff --git a/amber/src/main/python/core/storage/model/buffered_item_writer.py b/amber/src/main/python/core/storage/model/buffered_item_writer.py index 846ca119e25..49f315c2af2 100644 --- a/amber/src/main/python/core/storage/model/buffered_item_writer.py +++ b/amber/src/main/python/core/storage/model/buffered_item_writer.py @@ -37,7 +37,6 @@ def buffer_size(self) -> int: The size of the buffer. :return: the buffer size. """ - pass @abstractmethod def open(self) -> None: @@ -45,7 +44,6 @@ def open(self) -> None: Open the writer, initializing any necessary resources. This method should be called before any write operations. """ - pass @abstractmethod def close(self) -> None: @@ -53,7 +51,6 @@ def close(self) -> None: Close the writer, flushing any remaining items in the buffer to the underlying storage and releasing any held resources. """ - pass @abstractmethod def put_one(self, item: T) -> None: @@ -62,4 +59,3 @@ def put_one(self, item: T) -> None: the underlying storage. :param item: the data item to be written. """ - pass diff --git a/amber/src/main/python/core/storage/model/readonly_virtual_document.py b/amber/src/main/python/core/storage/model/readonly_virtual_document.py index 986aaac77b1..a0f05677969 100644 --- a/amber/src/main/python/core/storage/model/readonly_virtual_document.py +++ b/amber/src/main/python/core/storage/model/readonly_virtual_document.py @@ -37,7 +37,6 @@ def get_uri(self) -> ParseResult: Get the URI of the corresponding document. :return: the URI of the document as a ParseResult object """ - pass @abstractmethod def get_item(self, i: int) -> T: @@ -46,7 +45,6 @@ def get_item(self, i: int) -> T: :param i: index starting from 0 :return: data item of type T """ - pass @abstractmethod def get(self) -> Iterator[T]: @@ -54,7 +52,6 @@ def get(self) -> Iterator[T]: Get an iterator that iterates over all indexed items. :return: an iterator that returns data items of type T """ - pass @abstractmethod def get_range(self, from_index: int, until_index: int) -> Iterator[T]: @@ -65,7 +62,6 @@ def get_range(self, from_index: int, until_index: int) -> Iterator[T]: :param until_index: the ending index (exclusive) :return: an iterator that returns data items of type T """ - pass @abstractmethod def get_after(self, offset: int) -> Iterator[T]: @@ -74,7 +70,6 @@ def get_after(self, offset: int) -> Iterator[T]: :param offset: the starting index (exclusive) :return: an iterator that returns data items of type T """ - pass @abstractmethod def get_count(self) -> int: @@ -82,4 +77,3 @@ def get_count(self) -> int: Get the count of items in the document. :return: the count of items """ - pass diff --git a/amber/src/main/python/core/util/console_message/timestamp.py b/amber/src/main/python/core/util/console_message/timestamp.py index 9ced3a712bc..f6c9eb2e378 100644 --- a/amber/src/main/python/core/util/console_message/timestamp.py +++ b/amber/src/main/python/core/util/console_message/timestamp.py @@ -24,6 +24,4 @@ def current_time_in_local_timezone(): local_timezone = tzlocal.get_localzone() # Get the current time in the local timezone - local_time = datetime.datetime.now(local_timezone) - - return local_time + return datetime.datetime.now(local_timezone) diff --git a/amber/src/main/python/core/util/expression_evaluator.py b/amber/src/main/python/core/util/expression_evaluator.py index d0f9158fdef..0007e2dbe7d 100644 --- a/amber/src/main/python/core/util/expression_evaluator.py +++ b/amber/src/main/python/core/util/expression_evaluator.py @@ -74,7 +74,7 @@ def evaluate( value_str = repr(value) type_str = type(value).__name__ - to_be_expanded = list() + to_be_expanded = [] if ExpressionEvaluator._has_attributes(value): to_be_expanded += ExpressionEvaluator._extract_attributes(value) @@ -185,7 +185,7 @@ def _to_typed_values( ): return [ TypedValue( - expression=f"__getitem__({repr(k)})" if to_getitem else k, + expression=f"__getitem__({k!r})" if to_getitem else k, value_ref=repr(k) if ref_as_repr else k, value_str=repr(v), value_type=type(v).__name__, diff --git a/amber/src/main/python/pytexera/udf/udf_operator.py b/amber/src/main/python/pytexera/udf/udf_operator.py index a3ed76a4a36..40de47770ae 100644 --- a/amber/src/main/python/pytexera/udf/udf_operator.py +++ b/amber/src/main/python/pytexera/udf/udf_operator.py @@ -232,7 +232,6 @@ def open(self) -> None: Open a context of the operator. Usually can be used for loading/initiating some resources, such as a file, a model, or an API client. """ - pass @abstractmethod def process_tuple(self, tuple_: Tuple, port: int) -> Iterator[Optional[TupleLike]]: @@ -262,7 +261,6 @@ def close(self) -> None: """ Close the context of the operator. """ - pass class UDFSourceOperator(_UiParameterSupport, SourceOperator): @@ -271,7 +269,6 @@ def open(self) -> None: Open a context of the operator. Usually can be used for loading/initiating some resources, such as a file, a model, or an API client. """ - pass @abstractmethod def produce(self) -> Iterator[Optional[Union[TupleLike, TableLike]]]: @@ -287,7 +284,6 @@ def close(self) -> None: """ Close the context of the operator. """ - pass class UDFTableOperator(_UiParameterSupport, TableOperator): @@ -301,7 +297,6 @@ def open(self) -> None: Open a context of the operator. Usually can be used for loading/initiating some resources, such as a file, a model, or an API client. """ - pass @abstractmethod def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: @@ -320,7 +315,6 @@ def close(self) -> None: """ Close the context of the operator. """ - pass class UDFBatchOperator(_UiParameterSupport, BatchOperator): @@ -334,7 +328,6 @@ def open(self) -> None: Open a context of the operator. Usually can be used for loading/initiating some resources, such as a file, a model, or an API client. """ - pass @abstractmethod def process_batch(self, batch: Batch, port: int) -> Iterator[Optional[BatchLike]]: @@ -353,4 +346,3 @@ def close(self) -> None: """ Close the context of the operator. """ - pass diff --git a/amber/src/main/scala/org/apache/texera/web/resource/aiassistant/type_annotation_visitor.py b/amber/src/main/scala/org/apache/texera/web/resource/aiassistant/type_annotation_visitor.py index 9edf4f95bd3..00d32200fd0 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/aiassistant/type_annotation_visitor.py +++ b/amber/src/main/scala/org/apache/texera/web/resource/aiassistant/type_annotation_visitor.py @@ -52,9 +52,12 @@ def process_function(self, node): is_staticmethod = False if is_method and hasattr(node, 'decorator_list'): for decorator in node.decorator_list: - if isinstance(decorator, ast.Name) and decorator.id == 'staticmethod': - is_staticmethod = True - elif isinstance(decorator, ast.Attribute) and decorator.attr == 'staticmethod': + if ( + isinstance(decorator, ast.Name) and decorator.id == 'staticmethod' + ) or ( + isinstance(decorator, ast.Attribute) + and decorator.attr == 'staticmethod' + ): is_staticmethod = True args = node.args @@ -75,7 +78,7 @@ def process_function(self, node): # Skip the "self" or "cls" if is_method and not is_staticmethod: start_index = 1 - for i, arg in enumerate(all_args[start_index:]): + for arg in all_args[start_index:]: if not arg.annotation: self.add_untyped_arg(arg)