Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -59,5 +59,3 @@ def __call__(
),
)
)

return None
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
class ExceptionManager:
def __init__(self):
self.exc_info: Optional[ExceptionInfo] = None
self.exc_info_history: List[ExceptionInfo] = list()
self.exc_info_history: List[ExceptionInfo] = []

def set_exception_info(self, exc_info: ExceptionInfo) -> None:
self.exc_info = exc_info
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ def __init__(self, output_queue: InternalQueue, context: Context):
self._context = context
self._output_queue = output_queue
self._send_sequences: Dict[ActorVirtualIdentity, int] = defaultdict(int)
self._unfulfilled_promises: Dict[(ActorVirtualIdentity, int), Future] = dict()
self._unfulfilled_promises: Dict[(ActorVirtualIdentity, int), Future] = {}
# TODO: is this correct?
self._coordinator_service_stub = CoordinatorServiceStub("")
rpc_context = AsyncRpcContext(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ class OneToOnePartitioner(Partitioner):
def __init__(self, partitioning: OneToOnePartitioning, worker_id: str):
super().__init__(set_one_of(Partitioning, partitioning))
self.batch_size = partitioning.batch_size
self.batch: list[Tuple] = list()
self.batch: list[Tuple] = []
for channel in partitioning.channels:
if channel.from_worker_id.name == worker_id:
self.receiver = channel.to_worker_id
Expand Down Expand Up @@ -72,4 +72,4 @@ def flush_state(

@overrides
def reset(self) -> None:
self.batch = list()
self.batch = []
2 changes: 0 additions & 2 deletions amber/src/main/python/core/models/internal_marker.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,6 @@ class InternalMarker:
Markers preserve ordering and signal input lifecycle changes.
"""

pass


@dataclass(frozen=True)
class PortMarker(InternalMarker):
Expand Down
4 changes: 2 additions & 2 deletions amber/src/main/python/core/models/schema/attribute_type.py
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ def _local_wall_clock(moment):
# machine's standard offset. Before 1900 the engine reads every zone at
# its standard offset, where a named zone would give its local mean time.
zone = _named_local_zone()
utc = moment.astimezone(datetime.timezone.utc).replace(tzinfo=None)
utc = moment.astimezone(datetime.UTC).replace(tzinfo=None)
if utc < datetime.datetime(1900, 1, 1):
return utc + _standard_offset(zone)
if zone is not None:
Expand All @@ -146,7 +146,7 @@ def _named_local_zone():
return _zone_by_name(os.environ.get("TZ"))


@functools.lru_cache(maxsize=None)
@functools.cache
def _zone_by_name(tz):
# TZ names the zone when it is set, and /etc/localtime links to it when
# not. A zone neither of them names is left to localtime.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,7 @@ def detect_attribute_type_from_arrow_field(field: pa.Field) -> AttributeType:

if is_large_binary:
return AttributeType.LARGE_BINARY
else:
return FROM_ARROW_MAPPING[field.type.id]
return FROM_ARROW_MAPPING[field.type.id]


def create_arrow_field_with_metadata(
Expand Down
4 changes: 2 additions & 2 deletions amber/src/main/python/core/models/schema/schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ def add(self, attr_name: str, attr_type: AttributeType) -> None:
:return:
"""
if attr_name in self._name_type_mapping:
raise KeyError(f"Adding a duplicated attribute {repr(attr_name)}.")
raise KeyError(f"Adding a duplicated attribute {attr_name!r}.")
self._name_type_mapping[attr_name] = attr_type

def _from_raw_schema(self, raw_schema: Mapping[str, str]) -> None:
Expand Down Expand Up @@ -144,7 +144,7 @@ def __eq__(self, other: object) -> bool:

def __str__(self) -> str:
content = ",\n".join(
f"({index}){repr(attr_name)} -> {attr_type}"
f"({index}){attr_name!r} -> {attr_type}"
for index, (attr_name, attr_type) in enumerate(self.as_key_value_pairs(), 0)
)
return f"Schema[\n{content}\n]"
3 changes: 1 addition & 2 deletions amber/src/main/python/core/models/table.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,7 @@ def as_tuples(self) -> Iterator[Tuple]:
def __eq__(self, other: "Table") -> bool:
if isinstance(other, Table):
return all(a == b for a, b in zip(self.as_tuples(), other.as_tuples()))
else:
return super().__eq__(other).all()
return super().__eq__(other).all()


def all_output_to_tuple(output) -> Iterator[Tuple]:
Expand Down
2 changes: 1 addition & 1 deletion amber/src/main/python/core/proxy/proxy_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ def __init__(
def call_action(
self,
action_name: str,
payload: bytes = bytes(),
payload: bytes = b"",
options: Optional[FlightCallOptions] = None,
) -> bytes:
"""
Expand Down
15 changes: 6 additions & 9 deletions amber/src/main/python/core/proxy/proxy_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,14 +112,14 @@ def __init__(
if port is None:
port = get_free_local_port()
location = f"{scheme}://{host}:{port}"
super(ProxyServer, self).__init__(location)
super().__init__(location)
logger.debug(f"Serving on {location}")

self._port_number = port

# action name to callable map, will contain registered actions,
# identified by action name.
self._procedures: Dict[str, Tuple[Callable, str]] = dict()
self._procedures: Dict[str, Tuple[Callable, str]] = {}

# register heartbeat, this is the default action for the client to
# check the aliveness of the server.
Expand Down Expand Up @@ -213,7 +213,7 @@ def list_actions(self, context: ServerCallContext) -> Iterator[Tuple[str, str]]:
:param context: server context, containing information of middlewares.
:return: iterator of (action_name, action_description) pairs.
"""
return map(lambda x: (x[0], x[1][1]), self._procedures.items())
return ((k, v[1]) for k, v in self._procedures.items())

@overrides(check_signature=False)
def do_action(self, context: ServerCallContext, action: Action) -> Iterator[Result]:
Expand All @@ -234,14 +234,11 @@ def do_action(self, context: ServerCallContext, action: Action) -> Iterator[Resu
if action_name in self._procedures:
procedure, _ = self._procedures.get(action_name)
if not action:
raise KeyError("Unknown action {!r}".format(action_name))
raise KeyError(f"Unknown action {action_name!r}")

payload = action.body.to_pybytes()
# invoke the action
if payload:
result = procedure(payload)
else:
result = procedure()
result = procedure(payload) if payload else procedure()

# serialize the result
if isinstance(result, bytes):
Expand All @@ -254,7 +251,7 @@ def do_action(self, context: ServerCallContext, action: Action) -> Iterator[Resu
if action_name == "shutdown":
threading.Thread(target=self.graceful_shutdown).start()
else:
raise KeyError("Unknown action {!r}".format(action_name))
raise KeyError(f"Unknown action {action_name!r}")

@logger.catch(reraise=True)
def register(self, name: str, action: Callable, description: str = "") -> None:
Expand Down
3 changes: 1 addition & 2 deletions amber/src/main/python/core/runnables/heartbeat.py
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,6 @@ def stop(self):
os.kill(child.pid, signal.SIGKILL)
except Exception as e:
logger.warning(
"Exception during process termination "
f"PID {str(child.pid)}: {e} "
f"Exception during process termination PID {child.pid!s}: {e} "
)
os.kill(os.getpid(), signal.SIGTERM)
14 changes: 6 additions & 8 deletions amber/src/main/python/core/storage/document_factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,10 +104,9 @@ def create_document(uri: str, schema: Schema) -> VirtualDocument:
warehouse,
)

else:
raise NotImplementedError(
f"Unsupported URI scheme: {parsed_uri.scheme} for creating the document"
)
raise NotImplementedError(
f"Unsupported URI scheme: {parsed_uri.scheme} for creating the document"
)

@staticmethod
def document_exists(uri: str) -> bool:
Expand Down Expand Up @@ -166,7 +165,6 @@ def open_document(uri: str) -> typing.Tuple[VirtualDocument, Optional[Schema]]:
)
return document, amber_schema

else:
raise NotImplementedError(
f"Unsupported URI scheme: {parsed_uri.scheme} for opening the document"
)
raise NotImplementedError(
f"Unsupported URI scheme: {parsed_uri.scheme} for opening the document"
)
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ def get_instance(cls, warehouse: Optional[str] = None) -> Catalog:
StorageConfig.ICEBERG_POSTGRES_CATALOG_PASSWORD,
)
return cls._catalogs[cls._POSTGRES_KEY]
elif catalog_type == "rest":
if catalog_type == "rest":
key = warehouse or StorageConfig.ICEBERG_REST_CATALOG_WAREHOUSE_NAME
if key not in cls._catalogs:
cls._catalogs[key] = create_rest_catalog(
Expand All @@ -68,8 +68,7 @@ def get_instance(cls, warehouse: Optional[str] = None) -> Catalog:
StorageConfig.ICEBERG_REST_CATALOG_URI,
)
return cls._catalogs[key]
else:
raise ValueError(f"Unsupported catalog type: {catalog_type}")
raise ValueError(f"Unsupported catalog type: {catalog_type}")

@classmethod
def replace_instance(cls, catalog: Catalog, warehouse: Optional[str] = None):
Expand Down
13 changes: 4 additions & 9 deletions amber/src/main/python/core/storage/iceberg/iceberg_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@
from typing import Optional, Iterable
from pyiceberg import types as iceberg_types

import core
import core.models
from core.models import ArrowTableTupleProvider, Tuple
from core.models.schema.attribute_type import AttributeType, TO_ARROW_MAPPING
Expand Down Expand Up @@ -242,11 +241,9 @@ def create_rest_catalog(
# server, whose I/O goes through S3FileIO, not the local filesystem.
return load_catalog(
catalog_name,
**{
"type": "rest",
"uri": rest_uri,
"warehouse": warehouse_name,
},
type="rest",
uri=rest_uri,
warehouse=warehouse_name,
)


Expand Down Expand Up @@ -278,14 +275,12 @@ def create_table(
if catalog.table_exists(identifier) and override_if_exists:
catalog.drop_table(identifier)

table = catalog.create_table(
return catalog.create_table(
identifier=identifier,
schema=table_schema,
partition_spec=UNPARTITIONED_PARTITION_SPEC,
)

return table


def load_table_metadata(
catalog: Catalog, table_namespace: str, table_name: str
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,23 +37,20 @@ def buffer_size(self) -> int:
The size of the buffer.
:return: the buffer size.
"""
pass

@abstractmethod
def open(self) -> None:
"""
Open the writer, initializing any necessary resources.
This method should be called before any write operations.
"""
pass

@abstractmethod
def close(self) -> None:
"""
Close the writer, flushing any remaining items in the buffer
to the underlying storage and releasing any held resources.
"""
pass

@abstractmethod
def put_one(self, item: T) -> None:
Expand All @@ -62,4 +59,3 @@ def put_one(self, item: T) -> None:
the underlying storage.
:param item: the data item to be written.
"""
pass
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@ def get_uri(self) -> ParseResult:
Get the URI of the corresponding document.
:return: the URI of the document as a ParseResult object
"""
pass

@abstractmethod
def get_item(self, i: int) -> T:
Expand All @@ -46,15 +45,13 @@ def get_item(self, i: int) -> T:
:param i: index starting from 0
:return: data item of type T
"""
pass

@abstractmethod
def get(self) -> Iterator[T]:
"""
Get an iterator that iterates over all indexed items.
:return: an iterator that returns data items of type T
"""
pass

@abstractmethod
def get_range(self, from_index: int, until_index: int) -> Iterator[T]:
Expand All @@ -65,7 +62,6 @@ def get_range(self, from_index: int, until_index: int) -> Iterator[T]:
:param until_index: the ending index (exclusive)
:return: an iterator that returns data items of type T
"""
pass

@abstractmethod
def get_after(self, offset: int) -> Iterator[T]:
Expand All @@ -74,12 +70,10 @@ def get_after(self, offset: int) -> Iterator[T]:
:param offset: the starting index (exclusive)
:return: an iterator that returns data items of type T
"""
pass

@abstractmethod
def get_count(self) -> int:
"""
Get the count of items in the document.
:return: the count of items
"""
pass
4 changes: 1 addition & 3 deletions amber/src/main/python/core/util/console_message/timestamp.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,4 @@ def current_time_in_local_timezone():
local_timezone = tzlocal.get_localzone()

# Get the current time in the local timezone
local_time = datetime.datetime.now(local_timezone)

return local_time
return datetime.datetime.now(local_timezone)
4 changes: 2 additions & 2 deletions amber/src/main/python/core/util/expression_evaluator.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ def evaluate(
value_str = repr(value)
type_str = type(value).__name__

to_be_expanded = list()
to_be_expanded = []

if ExpressionEvaluator._has_attributes(value):
to_be_expanded += ExpressionEvaluator._extract_attributes(value)
Expand Down Expand Up @@ -185,7 +185,7 @@ def _to_typed_values(
):
return [
TypedValue(
expression=f"__getitem__({repr(k)})" if to_getitem else k,
expression=f"__getitem__({k!r})" if to_getitem else k,
value_ref=repr(k) if ref_as_repr else k,
value_str=repr(v),
value_type=type(v).__name__,
Expand Down
Loading
Loading