From 4609063a14a05889b34d6b2f352b9a005412bf11 Mon Sep 17 00:00:00 2001 From: hrodmn Date: Thu, 29 Jan 2026 11:30:50 -0600 Subject: [PATCH 01/23] feat: add first attempt at item table logic --- .env | 12 + .gitignore | 12 + .python-version | 1 + README.md | 215 +++++++++ docker-compose.yml | 52 ++ main.py | 37 ++ pyproject.toml | 39 ++ src/icestac/__init__.py | 2 + src/icestac/config.py | 181 +++++++ src/icestac/constants.py | 1 + src/icestac/item_table.py | 114 +++++ src/icestac/lambda_handler.py | 0 src/icestac/load.py | 27 ++ src/icestac/schema.py | 93 ++++ tests/conftest.py | 96 ++++ tests/test_config.py | 168 +++++++ tests/test_item_table.py | 105 ++++ tests/test_load.py | 193 ++++++++ tests/test_schema.py | 78 +++ uv.lock | 882 ++++++++++++++++++++++++++++++++++ 20 files changed, 2308 insertions(+) create mode 100644 .env create mode 100644 .gitignore create mode 100644 .python-version create mode 100644 docker-compose.yml create mode 100644 main.py create mode 100644 pyproject.toml create mode 100644 src/icestac/__init__.py create mode 100644 src/icestac/config.py create mode 100644 src/icestac/constants.py create mode 100644 src/icestac/item_table.py create mode 100644 src/icestac/lambda_handler.py create mode 100644 src/icestac/load.py create mode 100644 src/icestac/schema.py create mode 100644 tests/conftest.py create mode 100644 tests/test_config.py create mode 100644 tests/test_item_table.py create mode 100644 tests/test_load.py create mode 100644 tests/test_schema.py create mode 100644 uv.lock diff --git a/.env b/.env new file mode 100644 index 0000000..7ef3d3a --- /dev/null +++ b/.env @@ -0,0 +1,12 @@ +# Iceberg Catalog Configuration +ICESTAC_CATALOG_NAME=rest_catalog +ICESTAC_CATALOG_TYPE=rest +ICESTAC_CATALOG_URI=http://localhost:8181 +ICESTAC_WAREHOUSE_PATH=s3://warehouse/ + +# S3/MinIO Storage Configuration +# These settings allow PyIceberg to directly access MinIO for data file I/O +ICESTAC_S3_ENDPOINT=http://localhost:9000 +ICESTAC_S3_ACCESS_KEY_ID=admin +ICESTAC_S3_SECRET_ACCESS_KEY=password +ICESTAC_S3_PATH_STYLE_ACCESS=true diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f06d48f --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +# Python-generated files +__pycache__/ +*.py[oc] +build/ +dist/ +wheels/ +*.egg-info + +# Virtual environments +.venv + +minio-data/ diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..24ee5b1 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.13 diff --git a/README.md b/README.md index d49b10b..91d126a 100644 --- a/README.md +++ b/README.md @@ -1 +1,216 @@ # icestac + +## Overview + +This project creates a Python library (`icestac`) that uses rustac to convert STAC item collections to arrow tables and writes them to Apache Iceberg tables. + +The goal is a stac-geoparquet-backed system that can be used to maintain a **STAC Catalog** with many collections and support real-time ingestion. It will include an event-driven AWS pipeline for ingesting STAC items into an Iceberg catalog via SNS/SQS and Lambda. + +## Development + +### Tests + +```bash +# Run all tests +uv run pytest +``` + +### Local Instance + +**Environment Configuration** (`.env`): +```bash +# REST catalog endpoint +ICESTAC_CATALOG_NAME=rest_catalog +ICESTAC_CATALOG_TYPE=rest +ICESTAC_CATALOG_URI=http://localhost:8181 +ICESTAC_WAREHOUSE_PATH=s3://warehouse/ + +# S3/MinIO storage for PyIceberg data file I/O +ICESTAC_S3_ENDPOINT=http://localhost:9000 +ICESTAC_S3_ACCESS_KEY_ID=admin +ICESTAC_S3_SECRET_ACCESS_KEY=password +ICESTAC_S3_PATH_STYLE_ACCESS=true +``` + +**Starting the local environment:** +```bash +docker compose up +``` + +**Note:** [main.py](./main.py) currently uses an older API signature and needs to be updated to match the current `create_item_table` function signature. + +## Current Implementation Status + +### Core Library (`src/icestac/`) + +#### Item Table Module (`src/icestac/item_table.py`) - ✓ IMPLEMENTED + +Core functions for managing STAC item Iceberg tables: + +**`sanitize_collection_id(collection_id: str) -> str`** +- Converts STAC collection IDs to valid, deterministic Iceberg table names +- Uses lowercase + underscore normalization with 8-character hash suffix for uniqueness + +**`create_item_table(arrow_schema: ArrowSchema, collection_id: str, catalog: Catalog, namespace: str) -> Table`** +- Creates or loads Iceberg table from stac-geoparquet Arrow schema +- Converts Arrow schema to Iceberg schema with manual field ID assignment +- Creates table partitioned by datetime month using `MonthTransform` +- Validates schema for required STAC fields + +**Limitations:** +- Temporal partitioning is hardcoded to monthly (TODO: make configurable) +- No collection-level metadata management solution yet + +#### Schema Module (`src/icestac/schema.py`) - ✓ IMPLEMENTED + +Schema validation and enforcement: + +**`IcestacItem`** - Pydantic model extending stac-pydantic Item with required `collection` field + +**`get_schema_from_item(item: dict) -> Schema`** +- Validates STAC item and returns Arrow schema with enforced required fields + +**`enforce_required_fields(schema: Schema) -> Schema`** +- Marks required STAC fields as non-nullable in Arrow schema + +**`validate_schema(schema: Schema) -> None`** +- Validates Arrow schema contains all required STAC fields + +#### Config Module (`src/icestac/config.py`) - ✓ IMPLEMENTED + +**`IcebergCatalogConfig`** - Pydantic settings for catalog configuration +- Loads from environment variables with `ICESTAC_` prefix +- Supports catalog types: `rest`, `glue`, `hive`, `sql` +- Environment variables: + - Catalog: `CATALOG_NAME`, `CATALOG_TYPE`, `CATALOG_URI`, `WAREHOUSE_PATH`, `AWS_REGION` + - REST auth: `REST_TOKEN`, `REST_CREDENTIAL` + - SQL: `SQL_ECHO` + - S3/MinIO: `S3_ENDPOINT`, `S3_ACCESS_KEY_ID`, `S3_SECRET_ACCESS_KEY`, `S3_PATH_STYLE_ACCESS` + +**`get_catalog_properties() -> dict[str, str]`** +- Generates PyIceberg catalog properties from settings + +**`load_catalog() -> Catalog`** +- Factory method that creates PyIceberg Catalog instance + +#### Lambda Handler Module (`src/icestac/lambda_handler.py`) - NOT IMPLEMENTED + +Placeholder for AWS Lambda handler. + +### Testing Infrastructure (`tests/`) + +#### Test Coverage - ✓ IMPLEMENTED + +- **`tests/conftest.py`**: Pytest fixtures for test catalog, sample STAC items, and Arrow tables +- **`tests/test_item_table.py`**: Unit tests for `sanitize_collection_id` and `create_item_table` +- **`tests/test_config.py`**: Unit tests for catalog configuration and settings validation +- **`tests/test_schema.py`**: Unit tests for schema validation and enforcement + +### Dependencies + +**Core** (`pyproject.toml` dependencies): +- `pyarrow>=23.0.0` - Arrow table operations +- `pyiceberg[pyiceberg-core]>=0.10.0` - Iceberg table management +- `rustac[arrow]>=0.9.3` - STAC to Arrow conversion with arro3 schemas +- `stac-pydantic>=3.4.0` - STAC item validation +- `pydantic-settings>=2.12.0` - Environment-based configuration + +**Development** (dev dependency group): +- `pytest>=9.0.2` - Testing framework +- `sqlalchemy>=2.0.46` - SQL catalog backend for tests + +**Deployment** (deploy dependency group): +- `aws-cdk-lib>=2.236.0` - AWS infrastructure as code + +**Still needed for Lambda handler:** +- `boto3` - Lambda/SNS/SQS/S3 interactions +- `aws-lambda-powertools` - Structured logging and tracing +- `moto` - AWS service mocking for tests + +## Next Steps + +### Immediate Priorities + +1. **Design Collection Metadata Management** + - Determine approach for storing and managing collection-level metadata + - Options: Separate metadata table, catalog namespace properties, or external store + - Should track: collection description, temporal extent, spatial extent, schema versions + +2. **Complete Lambda Handler** (`src/icestac/lambda_handler.py`) + - Implement SNS event parsing + - Add collection grouping logic + - Integrate `create_item_table` function and config module + - Add error handling and structured logging + - Write integration tests + +3. **Enhance Item Table Module** + - Make partitioning strategy configurable (currently hardcoded to monthly) + - Add support for schema evolution + - Add write statistics/metadata + +### Future Work: AWS Infrastructure (CDK) + +**Planned Stack Structure**: +``` +infrastructure/ +├── app.py +├── stacks/ + ├── stac_ingestion_stack.py # SNS → SQS → Lambda pipeline + └── iceberg_catalog_stack.py # Optional: Glue/DynamoDB catalog +``` + +**STAC Ingestion Stack Components**: +- SNS Topic for incoming STAC items +- SQS Queue with batching and DLQ +- Lambda Function with icestac library +- CloudWatch Alarms for monitoring + +**Additional Dependencies Needed**: +- `constructs` +- `aws-cdk.aws-lambda-python-alpha` (Python Lambda bundling) + +## Development Workflow + +### Local Testing Strategy + +**Implemented:** +- PyIceberg with SQL catalog (SQLite) for unit tests +- Pytest for test framework +- Docker Compose with MinIO and Iceberg REST catalog for local development + +**Planned:** +- Moto for mocking AWS services in Lambda handler tests +- Helper script to simulate SNS events locally +- Optional: LocalStack for complete AWS simulation + +### Local Development Environment + +**Docker Compose Services**: +- **Iceberg REST Catalog** - `localhost:8181` for metadata operations +- **MinIO** - S3-compatible storage at `localhost:9000` (API) and `localhost:9001` (Console) + - Credentials: `admin` / `password` + - Warehouse bucket: `s3://warehouse/` +- **MinIO Client (mc)** - Initializes warehouse bucket on startup + +## Key Design Decisions + +### Decided + +1. **Table naming**: Sanitized collection ID with 8-character hash suffix for uniqueness +2. **Partitioning**: Monthly partitioning by datetime field (hardcoded, to be made configurable) +3. **Schema conversion**: Manual field ID assignment to avoid pyiceberg limitations +4. **Schema validation**: Required STAC fields marked as non-nullable using Pydantic models +5. **Configuration**: Environment variables with `ICESTAC_` prefix using Pydantic settings +6. **Testing catalog**: In-memory SQL catalog with SQLite for unit tests +7. **STAC to Arrow conversion**: Use rustac library with arro3 schemas + +### To Be Decided + +1. **Collection metadata management**: How to store and query collection-level metadata (description, extents, schema versions)? +2. **Iceberg catalog type for production**: AWS Glue (managed AWS) vs REST (self-hosted/managed) vs SQL (RDS)? +3. **Configurable partitioning strategies**: Support daily, monthly, yearly, or custom partitioning? +4. **Schema evolution policy**: Strict or flexible? How to handle schema changes across items in same collection? +5. **Batch size**: How many STAC items per SQS batch for optimal performance? +6. **Error handling**: Retry strategy for failed items? DLQ processing? +7. **S3 bucket structure**: How to organize Iceberg table data and metadata? +>>>>>>> a2b56ec (initial commit) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..7cebaf0 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,52 @@ +services: + rest: + image: apache/iceberg-rest-fixture + container_name: iceberg-rest + networks: + iceberg_net: + ports: + - 8181:8181 + environment: + - AWS_ACCESS_KEY_ID=admin + - AWS_SECRET_ACCESS_KEY=password + - AWS_REGION=us-east-1 + - CATALOG_WAREHOUSE=s3://warehouse/ + - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO + - CATALOG_S3_ENDPOINT=http://minio:9000 + minio: + image: minio/minio + container_name: minio + environment: + - MINIO_ROOT_USER=admin + - MINIO_ROOT_PASSWORD=password + - MINIO_DOMAIN=minio + networks: + iceberg_net: + aliases: + - warehouse.minio + ports: + - 9001:9001 + - 9000:9000 + volumes: + - ./minio-data:/data + command: ["server", "/data", "--console-address", ":9001"] + mc: + depends_on: + - minio + image: minio/mc + container_name: mc + networks: + iceberg_net: + environment: + - AWS_ACCESS_KEY_ID=admin + - AWS_SECRET_ACCESS_KEY=password + - AWS_REGION=us-east-1 + entrypoint: | + /bin/sh -c " + until (/usr/bin/mc alias set minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done; + /usr/bin/mc mb -p minio/warehouse; + /usr/bin/mc policy set public minio/warehouse; + tail -f /dev/null + " +networks: + iceberg_net: diff --git a/main.py b/main.py new file mode 100644 index 0000000..baaf765 --- /dev/null +++ b/main.py @@ -0,0 +1,37 @@ +import asyncio + +import rustac + +from icestac.config import IcebergCatalogConfig +from icestac.constants import DEFAULT_NAMESPACE +from icestac.item_table import create_item_table +from icestac.load import load_items +from icestac.schema import get_schema_from_item + + +async def run(): + config = IcebergCatalogConfig() + catalog = config.load_catalog() + + items = await rustac.search( + "https://stac.maap-project.org", + collections="icesat2-boreal-v3.1-agb", + max_items=5, + ) + schema = get_schema_from_item(items[0]) + table = create_item_table( + arrow_schema=schema, + collection_id=items[0]["collection"], + catalog=catalog, + namespace=DEFAULT_NAMESPACE, + ) + + load_items( + items=items, + table=table, + method="upsert", + ) + + +if __name__ == "__main__": + asyncio.run(run()) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..187f5f4 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,39 @@ +[project] +name = "icestac" +version = "0.1.0" +description = "Add your description here" +readme = "README.md" +authors = [ + { name = "hrodmn", email = "henry.rodman@gmail.com" } +] +requires-python = ">=3.13" +dependencies = [ + "pyarrow>=23.0.0", + "pydantic-settings>=2.12.0", + "pyiceberg[pyiceberg-core]>=0.10.0", + "rustac[arrow]>=0.9.3", + "stac-pydantic>=3.4.0", +] + +[project.scripts] +icestac = "icestac:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[dependency-groups] +deploy = [ + "aws-cdk-lib>=2.236.0", +] +dev = [ + "pytest>=9.0.2", + "sqlalchemy>=2.0.46", +] + +[tool.pytest.ini_options] +filterwarnings = [ + "ignore::DeprecationWarning:pyiceberg.*", + "ignore::pydantic.PydanticDeprecatedSince20", + "ignore::DeprecationWarning:pydantic.*", +] diff --git a/src/icestac/__init__.py b/src/icestac/__init__.py new file mode 100644 index 0000000..a570042 --- /dev/null +++ b/src/icestac/__init__.py @@ -0,0 +1,2 @@ +def main() -> None: + print("Hello from icestac!") diff --git a/src/icestac/config.py b/src/icestac/config.py new file mode 100644 index 0000000..45b92aa --- /dev/null +++ b/src/icestac/config.py @@ -0,0 +1,181 @@ +"""Settings module for icestac Iceberg catalog configuration.""" + +from typing import Literal + +from pydantic import Field, model_validator +from pydantic_settings import BaseSettings, SettingsConfigDict +from pyiceberg.catalog import Catalog, load_catalog + + +class IcebergCatalogConfig(BaseSettings): + """ + Pydantic settings for configuring an Apache Iceberg catalog. + + Settings are loaded from environment variables with the ICESTAC_ prefix. + Supports multiple catalog types: rest, glue, hive, and sql. + + Examples: + REST Catalog: + ICESTAC_CATALOG_NAME=my_catalog + ICESTAC_CATALOG_TYPE=rest + ICESTAC_CATALOG_URI=https://iceberg-rest.example.com + + AWS Glue Catalog: + ICESTAC_CATALOG_NAME=glue_catalog + ICESTAC_CATALOG_TYPE=glue + ICESTAC_WAREHOUSE_PATH=s3://my-bucket/warehouse/ + + SQL Catalog (SQLite): + ICESTAC_CATALOG_NAME=local_catalog + ICESTAC_CATALOG_TYPE=sql + ICESTAC_CATALOG_URI=sqlite:///path/to/catalog.db + ICESTAC_WAREHOUSE_PATH=/path/to/warehouse + """ + + model_config = SettingsConfigDict( + env_prefix="ICESTAC_", + case_sensitive=False, + env_file=".env", + env_file_encoding="utf-8", + extra="allow", # Allow extra fields for catalog-specific properties + ) + + # Core catalog settings + catalog_name: str = Field( + default="default", description="Name of the Iceberg catalog" + ) + + catalog_type: Literal["rest", "glue", "hive", "sql"] = Field( + default="sql", description="Type of Iceberg catalog backend" + ) + + catalog_uri: str | None = Field( + default=None, + description="URI for the catalog (required for rest, hive, and sql catalogs)", + ) + + warehouse_path: str | None = Field( + default=None, + description="Base path for the data warehouse (required for most catalog types)", + ) + + # AWS-specific settings + aws_region: str | None = Field( + default=None, description="AWS region for Glue catalog" + ) + + # REST catalog authentication + rest_token: str | None = Field( + default=None, description="Bearer token for REST catalog authentication" + ) + + rest_credential: str | None = Field( + default=None, description="Credential for REST catalog authentication" + ) + + # SQL catalog settings + sql_echo: bool = Field( + default=False, description="Enable SQL query logging (for sql catalog type)" + ) + + # S3/MinIO storage settings + s3_endpoint: str | None = Field( + default=None, + description="S3 endpoint URL (required for MinIO or custom S3-compatible storage)", + ) + + s3_access_key_id: str | None = Field(default=None, description="S3 access key ID") + + s3_secret_access_key: str | None = Field( + default=None, description="S3 secret access key" + ) + + s3_path_style_access: bool = Field( + default=True, description="Use path-style access for S3 (required for MinIO)" + ) + + @model_validator(mode="after") + def validate_required_fields(self): + """Ensure required fields are provided based on catalog type.""" + # Validate catalog_uri requirement + if self.catalog_type in ["rest", "hive", "sql"] and not self.catalog_uri: + raise ValueError( + f"catalog_uri is required for catalog_type='{self.catalog_type}'" + ) + + # Validate warehouse_path requirement + if self.catalog_type in ["sql", "hive"] and not self.warehouse_path: + raise ValueError( + f"warehouse_path is required for catalog_type='{self.catalog_type}'" + ) + + return self + + def get_catalog_properties(self) -> dict[str, str]: + """ + Generate the properties dict for PyIceberg catalog initialization. + + Returns: + Dictionary of catalog properties suitable for pyiceberg.catalog.load_catalog() + """ + properties: dict[str, str] = { + "type": self.catalog_type, + } + + # Add URI if provided + if self.catalog_uri: + properties["uri"] = self.catalog_uri + + # Add warehouse path if provided + if self.warehouse_path: + properties["warehouse"] = self.warehouse_path + + # Add AWS region for Glue + if self.catalog_type == "glue" and self.aws_region: + properties["region"] = self.aws_region + + # Add REST authentication + if self.catalog_type == "rest": + if self.rest_token: + properties["token"] = self.rest_token + if self.rest_credential: + properties["credential"] = self.rest_credential + + # Add SQL-specific settings + if self.catalog_type == "sql": + properties["echo"] = str(self.sql_echo).lower() + + # Add S3/MinIO storage settings + if self.s3_endpoint: + properties["s3.endpoint"] = self.s3_endpoint + if self.s3_access_key_id: + properties["s3.access-key-id"] = self.s3_access_key_id + if self.s3_secret_access_key: + properties["s3.secret-access-key"] = self.s3_secret_access_key + # Always set path-style-access when S3 endpoint is configured + if self.s3_endpoint: + properties["s3.path-style-access"] = str(self.s3_path_style_access).lower() + + # Include any extra fields from environment (for catalog-specific properties) + for key, value in ( + self.model_extra.items() if hasattr(self, "model_extra") else [] + ): + if value is not None: + properties[key] = str(value) + + return properties + + def load_catalog(self) -> Catalog: + """ + Create and return a PyIceberg Catalog instance using the configured settings. + + Returns: + PyIceberg Catalog instance configured with the specified properties + + Example: + >>> settings = IcebergCatalogSettings() + >>> catalog = settings.load_catalog() + >>> catalog.list_namespaces() + """ + properties = self.get_catalog_properties() + return load_catalog(self.catalog_name, **properties) diff --git a/src/icestac/constants.py b/src/icestac/constants.py new file mode 100644 index 0000000..1012ecf --- /dev/null +++ b/src/icestac/constants.py @@ -0,0 +1 @@ +DEFAULT_NAMESPACE = "icestac" diff --git a/src/icestac/item_table.py b/src/icestac/item_table.py new file mode 100644 index 0000000..b6e19ca --- /dev/null +++ b/src/icestac/item_table.py @@ -0,0 +1,114 @@ +import hashlib +import re + +import pyarrow +from arro3.core import Schema as ArrowSchema +from pyiceberg.catalog import Catalog +from pyiceberg.io.pyarrow import _pyarrow_to_schema_without_ids +from pyiceberg.partitioning import PartitionField, PartitionSpec +from pyiceberg.schema import Schema as IcebergSchema +from pyiceberg.table import Table +from pyiceberg.transforms import MonthTransform +from pyiceberg.types import NestedField + +from icestac.constants import DEFAULT_NAMESPACE +from icestac.schema import enforce_required_fields, validate_schema + + +def sanitize_collection_id(collection_id: str) -> str: + """ + Sanitize a STAC collection ID to a valid Iceberg table name. + + Creates a deterministic, unique table identifier by: + 1. Converting to lowercase + 2. Replacing non-alphanumeric characters with underscores + 3. Collapsing consecutive underscores + 4. Ensuring it starts with a letter or underscore + 5. Appending an 8-character hash suffix to guarantee uniqueness + + This prevents collisions where different collection IDs might otherwise + map to the same table name (e.g., "my.collection" vs "my_collection"). + + Args: + collection_id: STAC collection identifier + + Returns: + Sanitized table name that is valid for Iceberg and guaranteed unique + + Examples: + >>> sanitize_collection_id("sentinel-2-l2a") + 'sentinel_2_l2a_a1b2c3d4' + >>> sanitize_collection_id("my.collection") + 'my_collection_e5f6g7h8' + >>> sanitize_collection_id("my_collection") + 'my_collection_i9j0k1l2' + """ + # Convert to lowercase and replace non-alphanumeric chars with underscores + sanitized = re.sub(r"[^a-z0-9_]", "_", collection_id.lower()) + sanitized = re.sub(r"_+", "_", sanitized) + sanitized = sanitized.strip("_") + + if sanitized and sanitized[0].isdigit(): + sanitized = f"c_{sanitized}" + + # Generate a short hash of the original collection_id for uniqueness + hash_suffix = hashlib.sha256(collection_id.encode()).hexdigest()[:8] + + # Combine sanitized name with hash suffix + return f"{sanitized}_{hash_suffix}" + + +def create_item_table( + arrow_schema: ArrowSchema, + collection_id: str, + catalog: Catalog, + namespace: str = DEFAULT_NAMESPACE, +) -> Table: + """ + Create an Iceberg table from a stac-geoparquet Arrow schema + + Converts the Arrow schema to an Iceberg schema with manually assigned field IDs, + then creates or loads the Iceberg table partitioned by datetime month. + + Args: + schema: arro3.core.Schema for the items in this collection + collection_id: the collection id for the items in this table + catalog: PyIceberg catalog instance + namespace: Namespace for the Iceberg table + + Returns: + PyIceberg Table instance + + """ + validate_schema(arrow_schema) + + # Ensure required STAC fields are marked as non-nullable + pa_schema = pyarrow.schema(enforce_required_fields(arrow_schema)) + _schema = _pyarrow_to_schema_without_ids(pa_schema) + + # assign iceberg field ids manually + fields = [] + for i, _field in enumerate(_schema.fields, start=1): + field_dict = _field.model_dump() + field_dict["id"] = i + fields.append(NestedField(**field_dict)) + + table_id = f"{namespace}.{sanitize_collection_id(collection_id)}" + + iceberg_schema = IcebergSchema(*fields) + + catalog.create_namespace_if_not_exists(namespace) + + return catalog.create_table_if_not_exists( + identifier=table_id, + schema=iceberg_schema, + partition_spec=PartitionSpec( + # TODO: make temporal partitioning configurable + PartitionField( + source_id=iceberg_schema.find_field("datetime").field_id, + field_id=1000, + transform=MonthTransform(), + name="datetime_month", + ) + ), + ) diff --git a/src/icestac/lambda_handler.py b/src/icestac/lambda_handler.py new file mode 100644 index 0000000..e69de29 diff --git a/src/icestac/load.py b/src/icestac/load.py new file mode 100644 index 0000000..7202f87 --- /dev/null +++ b/src/icestac/load.py @@ -0,0 +1,27 @@ +from typing import Any, Literal + +import pyarrow +from pyiceberg.table import Table +from rustac import to_arrow + +from icestac.schema import enforce_required_fields + +Method = Literal["append", "upsert"] + + +def load_items( + items: list[dict[str, Any]], + table: Table, + method: Method = "upsert", +) -> None: + arrow_data = to_arrow(items) + enforced_schema = enforce_required_fields(arrow_data.schema) + arrow_table = pyarrow.table(arrow_data).cast(pyarrow.schema(enforced_schema)) + + if method == "upsert": + table.upsert( + df=arrow_table, + join_cols=["id"], + ) + elif method == "append": + table.append(df=arrow_table) diff --git a/src/icestac/schema.py b/src/icestac/schema.py new file mode 100644 index 0000000..386ca99 --- /dev/null +++ b/src/icestac/schema.py @@ -0,0 +1,93 @@ +from typing import Any + +import pyarrow as pa +from arro3.core import Schema +from rustac import to_arrow +from stac_pydantic.item import Item + + +class IcestacItem(Item): + collection: str + + +def get_schema_from_item(item: dict[str, Any]) -> Schema: + # validate stac item + _ = IcestacItem(**item) + + return enforce_required_fields(to_arrow([item]).schema) + + +def get_required_fields() -> set[str]: + """ + Get the set of required field names from IcestacItem. + + Returns: + Set of required field names, with special handling for flattened properties + """ + required_fields = set() + + for field_name, field_info in IcestacItem.model_fields.items(): + if field_info.is_required(): + required_fields.add(field_name) + + # Special handling: rustac flattens properties.datetime to just "datetime" + if "properties" in required_fields: + required_fields.remove("properties") + required_fields.add("datetime") + + return required_fields + + +def enforce_required_fields(schema: Schema) -> Schema: + """ + Ensure required STAC fields are marked as non-nullable in the Arrow schema. + + Takes an arro3 Schema and returns a pyarrow Schema with required fields + marked as nullable=False. This ensures the Iceberg table will enforce + these fields as required. + + Args: + schema: arro3.core.Schema from rustac + + Returns: + pyarrow.Schema with required fields marked as non-nullable + """ + required_fields = get_required_fields() + + # Convert arro3 schema to pyarrow schema and rebuild with correct nullable flags + pa_schema = pa.schema(schema) + new_fields = [] + + for field in pa_schema: + if field.name in required_fields: + # Mark as non-nullable (required) + new_fields.append(pa.field(field.name, field.type, nullable=False)) + else: + # Keep original nullable setting + new_fields.append(field) + + return Schema.from_arrow(pa.schema(new_fields)) + + +def validate_schema(schema: Schema) -> None: + """ + Validate that an Arrow schema contains required STAC item fields. + + Checks for top-level required fields from IcestacItem. + Note: rustac flattens nested properties, so 'properties.datetime' + becomes 'datetime' in the Arrow schema. + + Args: + schema: arro3.core.Schema to validate + + Raises: + ValueError: If required STAC fields are missing from the schema + """ + schema_fields = set(schema.names) + required_fields = get_required_fields() + missing_fields = required_fields - schema_fields + + if missing_fields: + raise ValueError( + f"Arrow schema is missing required STAC fields: {sorted(missing_fields)}" + ) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..9e15541 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,96 @@ +import tempfile +from pathlib import Path +from typing import Any + +import pyarrow +import pytest +from pyarrow import Table +from pyiceberg.catalog.sql import SqlCatalog +from rustac import to_arrow + + +@pytest.fixture +def temp_warehouse(): + """Create a temporary warehouse directory for testing.""" + with tempfile.TemporaryDirectory() as tmpdir: + yield Path(tmpdir) + + +@pytest.fixture +def test_catalog(temp_warehouse): + """Create an in-memory SQL catalog for testing.""" + catalog = SqlCatalog( + "test_catalog", + **{ + "uri": f"sqlite:///{temp_warehouse}/catalog.db", + "warehouse": f"file://{temp_warehouse}", + }, + ) + + # Create test namespace + catalog.create_namespace("test_namespace") + + return catalog + + +@pytest.fixture +def test_namespace(): + """Provide a test namespace name.""" + return "test_namespace" + + +@pytest.fixture +def test_collection_id(): + """Provide a test collection ID.""" + return "test-collection" + + +@pytest.fixture +def sample_stac_item(): + """Provide a sample STAC item for testing.""" + return { + "type": "Feature", + "stac_version": "1.0.0", + "id": "test-item-001", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [-180.0, -90.0], + [180.0, -90.0], + [180.0, 90.0], + [-180.0, 90.0], + [-180.0, -90.0], + ] + ], + }, + "bbox": [-180.0, -90.0, 180.0, 90.0], + "properties": { + "datetime": "2024-01-01T00:00:00Z", + "title": "Test Item", + }, + "collection": "test-collection", + "links": [], + "assets": { + "data": {"href": "https://example.com/data.tif", "type": "image/tiff"} + }, + } + + +@pytest.fixture +def sample_stac_items(sample_stac_item) -> list[dict[str, Any]]: + """Provide a list of sample STAC items for testing.""" + items = [] + for i in range(3): + item = sample_stac_item.copy() + item["id"] = f"test-item-{i:03d}" + item["properties"] = sample_stac_item["properties"].copy() + item["properties"]["datetime"] = f"2024-01-{i + 1:02d}T00:00:00Z" + items.append(item) + + return items + + +@pytest.fixture +def sample_item_arrow_table(sample_stac_items) -> Table: + return pyarrow.table(to_arrow(sample_stac_items)) diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..46fef78 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,168 @@ +"""Tests for icestac config module.""" + +import os + +import pytest +from pydantic import ValidationError + +from icestac.config import IcebergCatalogConfig + + +class TestIcebergCatalogConfig: + """Test Iceberg catalog settings configuration.""" + + def test_default_settings(self, monkeypatch, tmp_path): + """Test default settings values with minimal valid configuration.""" + # Clear any existing environment variables + for key in os.environ.copy(): + if key.startswith("ICESTAC_"): + monkeypatch.delenv(key, raising=False) + + # Set minimal required configuration for SQL catalog (the default) + catalog_db = tmp_path / "catalog.db" + warehouse_path = tmp_path / "warehouse" + warehouse_path.mkdir() + + monkeypatch.setenv("ICESTAC_CATALOG_URI", f"sqlite:///{catalog_db}") + monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", str(warehouse_path)) + + # Disable .env file reading to avoid pollution from project .env file + settings = IcebergCatalogConfig(_env_file=None) + + # Verify defaults are applied + assert settings.catalog_name == "default" + assert settings.catalog_type == "sql" + assert settings.sql_echo is False + assert settings.aws_region is None + assert settings.rest_token is None + + def test_sql_catalog_properties(self, monkeypatch): + """Test SQL catalog configuration.""" + monkeypatch.setenv("ICESTAC_CATALOG_NAME", "test_catalog") + monkeypatch.setenv("ICESTAC_CATALOG_TYPE", "sql") + monkeypatch.setenv("ICESTAC_CATALOG_URI", "sqlite:///catalog.db") + monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", "/tmp/warehouse") + + settings = IcebergCatalogConfig() + + assert settings.catalog_name == "test_catalog" + assert settings.catalog_type == "sql" + assert settings.catalog_uri == "sqlite:///catalog.db" + assert settings.warehouse_path == "/tmp/warehouse" + + properties = settings.get_catalog_properties() + assert properties["type"] == "sql" + assert properties["uri"] == "sqlite:///catalog.db" + assert properties["warehouse"] == "/tmp/warehouse" + assert properties["echo"] == "false" + + def test_rest_catalog_properties(self, monkeypatch): + """Test REST catalog configuration.""" + monkeypatch.setenv("ICESTAC_CATALOG_NAME", "rest_catalog") + monkeypatch.setenv("ICESTAC_CATALOG_TYPE", "rest") + monkeypatch.setenv("ICESTAC_CATALOG_URI", "https://rest.example.com") + monkeypatch.setenv("ICESTAC_REST_TOKEN", "my-token") + + settings = IcebergCatalogConfig() + + assert settings.catalog_type == "rest" + assert settings.rest_token == "my-token" + + properties = settings.get_catalog_properties() + assert properties["type"] == "rest" + assert properties["uri"] == "https://rest.example.com" + assert properties["token"] == "my-token" + + def test_glue_catalog_properties(self, monkeypatch): + """Test AWS Glue catalog configuration.""" + monkeypatch.setenv("ICESTAC_CATALOG_NAME", "glue_catalog") + monkeypatch.setenv("ICESTAC_CATALOG_TYPE", "glue") + monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", "s3://bucket/warehouse") + monkeypatch.setenv("ICESTAC_AWS_REGION", "us-west-2") + + settings = IcebergCatalogConfig() + + assert settings.catalog_type == "glue" + assert settings.aws_region == "us-west-2" + + properties = settings.get_catalog_properties() + assert properties["type"] == "glue" + assert properties["warehouse"] == "s3://bucket/warehouse" + assert properties["region"] == "us-west-2" + + def test_sql_catalog_requires_uri(self, monkeypatch): + """Test that SQL catalog requires a URI.""" + monkeypatch.setenv("ICESTAC_CATALOG_TYPE", "sql") + monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", "/tmp/warehouse") + + with pytest.raises(ValidationError) as exc_info: + IcebergCatalogConfig(_env_file=None) + + assert "catalog_uri is required" in str(exc_info.value) + + def test_sql_catalog_requires_warehouse_path(self, monkeypatch): + """Test that SQL catalog requires a warehouse path.""" + monkeypatch.setenv("ICESTAC_CATALOG_TYPE", "sql") + monkeypatch.setenv("ICESTAC_CATALOG_URI", "sqlite:///catalog.db") + + with pytest.raises(ValidationError) as exc_info: + IcebergCatalogConfig(_env_file=None) + + assert "warehouse_path is required" in str(exc_info.value) + + def test_rest_catalog_requires_uri(self, monkeypatch): + """Test that REST catalog requires a URI.""" + monkeypatch.setenv("ICESTAC_CATALOG_TYPE", "rest") + + with pytest.raises(ValidationError) as exc_info: + IcebergCatalogConfig(_env_file=None) + + assert "catalog_uri is required" in str(exc_info.value) + + def test_case_insensitive_env_vars(self, monkeypatch): + """Test that environment variables are case-insensitive.""" + monkeypatch.setenv("icestac_catalog_name", "test") + monkeypatch.setenv("ICESTAC_CATALOG_TYPE", "sql") + monkeypatch.setenv("icestac_catalog_uri", "sqlite:///catalog.db") + monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", "/tmp/warehouse") + + settings = IcebergCatalogConfig() + + assert settings.catalog_name == "test" + + def test_sql_echo_enabled(self, monkeypatch): + """Test SQL echo setting.""" + monkeypatch.setenv("ICESTAC_CATALOG_TYPE", "sql") + monkeypatch.setenv("ICESTAC_CATALOG_URI", "sqlite:///catalog.db") + monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", "/tmp/warehouse") + monkeypatch.setenv("ICESTAC_SQL_ECHO", "true") + + settings = IcebergCatalogConfig() + + assert settings.sql_echo is True + + properties = settings.get_catalog_properties() + assert properties["echo"] == "true" + + def test_load_catalog(self, tmp_path, monkeypatch): + """Test loading a PyIceberg catalog from settings.""" + catalog_db = tmp_path / "catalog.db" + warehouse_path = tmp_path / "warehouse" + warehouse_path.mkdir() + + monkeypatch.setenv("ICESTAC_CATALOG_NAME", "test_catalog") + monkeypatch.setenv("ICESTAC_CATALOG_TYPE", "sql") + monkeypatch.setenv("ICESTAC_CATALOG_URI", f"sqlite:///{catalog_db}") + monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", str(warehouse_path)) + + settings = IcebergCatalogConfig() + catalog = settings.load_catalog() + + # Verify catalog is created successfully + assert catalog is not None + assert catalog.name == "test_catalog" + + # Test basic catalog operations + catalog.create_namespace("test") + namespaces = catalog.list_namespaces() + assert ("test",) in namespaces diff --git a/tests/test_item_table.py b/tests/test_item_table.py new file mode 100644 index 0000000..6af1640 --- /dev/null +++ b/tests/test_item_table.py @@ -0,0 +1,105 @@ +from typing import Any + +import pyarrow +from pyiceberg.catalog import Catalog +from rustac import to_arrow + +from icestac.item_table import create_item_table, sanitize_collection_id +from icestac.schema import enforce_required_fields, get_schema_from_item + + +def test_create_item_table( + test_catalog: Catalog, + test_namespace: str, + sample_stac_items: list[dict[str, Any]], +) -> None: + arrow_schema = get_schema_from_item(sample_stac_items[0]) + table = create_item_table( + arrow_schema=arrow_schema, + collection_id=sample_stac_items[0]["collection"], + catalog=test_catalog, + namespace=test_namespace, + ) + + assert table.schema().find_field("datetime") + + # Ensure data has required fields marked as non-nullable to match table schema + arrow_data = to_arrow(sample_stac_items) + enforced_schema = enforce_required_fields(arrow_data.schema) + arrow_table = pyarrow.table(arrow_data).cast(pyarrow.schema(enforced_schema)) + + table.upsert( + df=arrow_table, + join_cols=["id"], + ) + + # Verify records were inserted + result = table.scan().to_arrow() + assert len(result) == len(sample_stac_items) + assert result.column("id").to_pylist() == [item["id"] for item in sample_stac_items] + + +def test_sanitize_collection_id_basic(): + """Test basic sanitization of collection IDs.""" + result = sanitize_collection_id("sentinel-2-l2a") + # Should be lowercase with underscores and have 8-char hash suffix + assert result.startswith("sentinel_2_l2a_") + assert len(result.split("_")[-1]) == 8 + assert result.islower() or "_" in result + + +def test_sanitize_collection_id_dots(): + """Test that dots are replaced with underscores.""" + result = sanitize_collection_id("my.collection.id") + assert result.startswith("my_collection_id_") + assert ".." not in result + + +def test_sanitize_collection_id_uniqueness(): + """Test that different collection IDs produce different sanitized names.""" + # These would collide with simple character replacement + id1 = sanitize_collection_id("my.collection") + id2 = sanitize_collection_id("my_collection") + id3 = sanitize_collection_id("my-collection") + + # All should be different due to hash suffix + assert id1 != id2 + assert id2 != id3 + assert id1 != id3 + + +def test_sanitize_collection_id_deterministic(): + """Test that sanitization is deterministic.""" + collection_id = "test-collection-123" + result1 = sanitize_collection_id(collection_id) + result2 = sanitize_collection_id(collection_id) + + assert result1 == result2 + + +def test_sanitize_collection_id_special_chars(): + """Test handling of various special characters.""" + result = sanitize_collection_id("my@collection#with$special%chars!") + # Should only contain lowercase alphanumeric and underscores + assert all(c.islower() or c.isdigit() or c == "_" for c in result) + + +def test_sanitize_collection_id_consecutive_underscores(): + """Test that consecutive underscores are collapsed.""" + result = sanitize_collection_id("my___collection___id") + # Should not have triple underscores in the sanitized portion + base_name = "_".join(result.split("_")[:-1]) # exclude hash suffix + assert "___" not in base_name + + +def test_sanitize_collection_id_starts_with_digit(): + """Test handling of collection IDs that start with a digit.""" + result = sanitize_collection_id("3dep-lidar") + # Should be prepended with 'c_' to make it valid + assert result.startswith("c_3") + + +def test_sanitize_collection_id_uppercase(): + """Test that uppercase letters are converted to lowercase.""" + result = sanitize_collection_id("MyCollection-ID") + assert result == result.lower() diff --git a/tests/test_load.py b/tests/test_load.py new file mode 100644 index 0000000..29f4fc0 --- /dev/null +++ b/tests/test_load.py @@ -0,0 +1,193 @@ +from typing import Any + +from pyiceberg.catalog import Catalog + +from icestac.item_table import create_item_table +from icestac.load import load_items +from icestac.schema import get_schema_from_item + + +def test_load_items_upsert_default( + test_catalog: Catalog, + test_namespace: str, + sample_stac_items: list[dict[str, Any]], +) -> None: + """Test loading items with default upsert method.""" + # Create the table + arrow_schema = get_schema_from_item(sample_stac_items[0]) + table = create_item_table( + arrow_schema=arrow_schema, + collection_id=sample_stac_items[0]["collection"], + catalog=test_catalog, + namespace=test_namespace, + ) + + # Load items (default method is upsert) + load_items(sample_stac_items, table) + + # Verify records were inserted + result = table.scan().to_arrow() + assert len(result) == len(sample_stac_items) + assert sorted(result.column("id").to_pylist()) == sorted( + [item["id"] for item in sample_stac_items] + ) + + +def test_load_items_upsert_explicit( + test_catalog: Catalog, + test_namespace: str, + sample_stac_items: list[dict[str, Any]], +) -> None: + """Test loading items with explicit upsert method.""" + # Create the table + arrow_schema = get_schema_from_item(sample_stac_items[0]) + table = create_item_table( + arrow_schema=arrow_schema, + collection_id=sample_stac_items[0]["collection"], + catalog=test_catalog, + namespace=test_namespace, + ) + + # Load items with explicit upsert method + load_items(sample_stac_items, table, method="upsert") + + # Verify records were inserted + result = table.scan().to_arrow() + assert len(result) == len(sample_stac_items) + + +def test_load_items_upsert_updates_existing( + test_catalog: Catalog, + test_namespace: str, + sample_stac_items: list[dict[str, Any]], +) -> None: + """Test that upsert updates existing records with same ID.""" + # Create the table + arrow_schema = get_schema_from_item(sample_stac_items[0]) + table = create_item_table( + arrow_schema=arrow_schema, + collection_id=sample_stac_items[0]["collection"], + catalog=test_catalog, + namespace=test_namespace, + ) + + # Load initial items + load_items(sample_stac_items, table, method="upsert") + + # Modify items (same IDs but different data) + modified_items = [] + for item in sample_stac_items: + modified_item = item.copy() + modified_item["properties"] = item["properties"].copy() + modified_item["properties"]["title"] = f"Updated {item['properties']['title']}" + modified_items.append(modified_item) + + # Load modified items with upsert + load_items(modified_items, table, method="upsert") + + # Verify only 3 records exist (not 6) and they have updated titles + result = table.scan().to_arrow() + assert len(result) == len(sample_stac_items) + + # Check that titles were updated + titles = result.column("title").to_pylist() + assert all(title.startswith("Updated") for title in titles) + + +def test_load_items_append( + test_catalog: Catalog, + test_namespace: str, + sample_stac_items: list[dict[str, Any]], +) -> None: + """Test loading items with append method.""" + # Create the table + arrow_schema = get_schema_from_item(sample_stac_items[0]) + table = create_item_table( + arrow_schema=arrow_schema, + collection_id=sample_stac_items[0]["collection"], + catalog=test_catalog, + namespace=test_namespace, + ) + + # Load items with append method + load_items(sample_stac_items, table, method="append") + + # Verify records were inserted + result = table.scan().to_arrow() + assert len(result) == len(sample_stac_items) + + +def test_load_items_append_creates_duplicates( + test_catalog: Catalog, + test_namespace: str, + sample_stac_items: list[dict[str, Any]], +) -> None: + """Test that append creates duplicate records when IDs overlap.""" + # Create the table + arrow_schema = get_schema_from_item(sample_stac_items[0]) + table = create_item_table( + arrow_schema=arrow_schema, + collection_id=sample_stac_items[0]["collection"], + catalog=test_catalog, + namespace=test_namespace, + ) + + # Load items twice with append + load_items(sample_stac_items, table, method="append") + load_items(sample_stac_items, table, method="append") + + # Verify we have double the records (append doesn't deduplicate) + result = table.scan().to_arrow() + assert len(result) == len(sample_stac_items) * 2 + + +def test_load_items_multiple_batches( + test_catalog: Catalog, + test_namespace: str, + sample_stac_items: list[dict[str, Any]], +) -> None: + """Test loading items in multiple batches with different methods.""" + # Create the table + arrow_schema = get_schema_from_item(sample_stac_items[0]) + table = create_item_table( + arrow_schema=arrow_schema, + collection_id=sample_stac_items[0]["collection"], + catalog=test_catalog, + namespace=test_namespace, + ) + + # Load first batch + load_items(sample_stac_items[:2], table, method="upsert") + + result = table.scan().to_arrow() + assert len(result) == 2 + + # Load second batch + load_items(sample_stac_items[2:], table, method="upsert") + + result = table.scan().to_arrow() + assert len(result) == 3 + + +def test_load_items_single_item( + test_catalog: Catalog, + test_namespace: str, + sample_stac_item: dict[str, Any], +) -> None: + """Test loading a single item.""" + # Create the table + arrow_schema = get_schema_from_item(sample_stac_item) + table = create_item_table( + arrow_schema=arrow_schema, + collection_id=sample_stac_item["collection"], + catalog=test_catalog, + namespace=test_namespace, + ) + + # Load single item as a list + load_items([sample_stac_item], table) + + # Verify record was inserted + result = table.scan().to_arrow() + assert len(result) == 1 + assert result.column("id").to_pylist()[0] == sample_stac_item["id"] diff --git a/tests/test_schema.py b/tests/test_schema.py new file mode 100644 index 0000000..ea47a96 --- /dev/null +++ b/tests/test_schema.py @@ -0,0 +1,78 @@ +from typing import Any + +import pyarrow as pa +import pytest +from arro3.core import Schema +from pydantic import ValidationError + +from icestac.schema import get_schema_from_item, validate_schema + + +def test_get_schema_from_item(sample_stac_item: dict[str, Any]) -> None: + """Test that we can extract an Arrow schema from a STAC item.""" + schema = get_schema_from_item(sample_stac_item) + + assert isinstance(schema, Schema) + assert "id" in schema.names + assert "datetime" in schema.names + assert "collection" in schema.names + + +def test_get_schema_from_item_validates(sample_stac_item: dict[str, Any]) -> None: + """Test that get_schema_from_item validates the STAC item.""" + invalid_item = {"not": "a stac item"} + + with pytest.raises(ValidationError): + get_schema_from_item(invalid_item) + + +def test_get_schema_from_item_no_collection(sample_stac_item: dict[str, Any]) -> None: + """Test that missing collection field raises ValueError.""" + _ = sample_stac_item.pop("collection") + + with pytest.raises(ValidationError): + _ = get_schema_from_item(sample_stac_item) + + +def test_validate_schema_valid(sample_stac_item: dict[str, Any]) -> None: + """Test that a valid STAC schema passes validation.""" + schema = get_schema_from_item(sample_stac_item) + + # Should not raise + validate_schema(schema) + + +def test_validate_schema_missing_required_field() -> None: + """Test that a schema missing required fields raises ValueError.""" + # Create a schema missing the required 'id' field + schema = pa.schema( + [ + ("type", pa.string()), + ("geometry", pa.string()), + ("collection", pa.string()), + ("datetime", pa.string()), + ] + ) + + with pytest.raises(ValueError, match="missing required STAC fields.*'id'"): + validate_schema(schema) + + +def test_validate_schema_missing_datetime() -> None: + """Test that a schema missing datetime field raises ValueError.""" + # Create a schema with all required fields except datetime + schema = pa.schema( + [ + ("type", pa.string()), + ("id", pa.string()), + ("geometry", pa.string()), + ("collection", pa.string()), + ("stac_version", pa.string()), + ("links", pa.string()), + ("assets", pa.string()), + ("bbox", pa.list_(pa.float64())), + ] + ) + + with pytest.raises(ValueError, match="missing required STAC fields.*'datetime'"): + validate_schema(schema) diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..675af43 --- /dev/null +++ b/uv.lock @@ -0,0 +1,882 @@ +version = 1 +revision = 2 +requires-python = ">=3.13" + +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, +] + +[[package]] +name = "arro3-core" +version = "0.6.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2a/01/f06342d2eb822153f63d188153e41fbeabb29b48247f7a11ce76c538f7d1/arro3_core-0.6.5.tar.gz", hash = "sha256:768078887cd7ac82de4736f94bbd91f6d660f10779848bd5b019f511badd9d75", size = 107522, upload-time = "2025-10-13T23:12:38.872Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/85/20e46d3ed59d2f93be4a4d1abea4f6bef3e96acd59bf5a50726f84303c51/arro3_core-0.6.5-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:9d5999506daec1ab31096b3deb1e3573041d6ecadb4ca99c96f7ab26720c592c", size = 2685615, upload-time = "2025-10-13T23:09:41.793Z" }, + { url = "https://files.pythonhosted.org/packages/d0/9c/427d578f7d2bf3149515a8b75217e7189e7b1d74e5c5609e1a7e7f0f8d3c/arro3_core-0.6.5-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:bd3e251184c2dd6ade81c5613256b6d85ab3ddbd5af838b1de657e0ddec017f8", size = 2391944, upload-time = "2025-10-13T23:09:45.266Z" }, + { url = "https://files.pythonhosted.org/packages/90/24/7e4af478eb889bfa401e1c1b8868048ca692e6205affbf81cf3666347852/arro3_core-0.6.5-cp311-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7cadb29349960d3821b0515d9df80f2725cea155ad966c699f6084de32e313cb", size = 2888376, upload-time = "2025-10-13T23:09:48.737Z" }, + { url = "https://files.pythonhosted.org/packages/70/3b/01006a96bc980275aa4d2eb759c5f10afb7c85fcdce3c36ddb18635ad23b/arro3_core-0.6.5-cp311-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a922e560ed2ccee3293d51b39e013b51cc233895d25ddafcacfb83c540a19e6f", size = 2916568, upload-time = "2025-10-13T23:09:51.95Z" }, + { url = "https://files.pythonhosted.org/packages/a2/2f/4e04c7f5687de6fb6f88aa7590b16bcf507ba17ddbd268525f27b70b7a68/arro3_core-0.6.5-cp311-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:68fe6672bf51f039b12046a209cba0a9405e10ae44e5a0d557f091b356a62051", size = 3144223, upload-time = "2025-10-13T23:09:55.387Z" }, + { url = "https://files.pythonhosted.org/packages/31/4a/72dc383d1a0d14f1d453e334e3461e229762edb1bf3f75b3ab977e9386ed/arro3_core-0.6.5-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c3ee95603e375401a58ff763ce2c8aa858e0c4f757c1fb719f48fb070f540b2", size = 2781862, upload-time = "2025-10-13T23:09:59.035Z" }, + { url = "https://files.pythonhosted.org/packages/14/dc/0df7684b683114eaf8e57989b4230edb359cbfb6e98b8770d69128b27572/arro3_core-0.6.5-cp311-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:fbaf6b65213630007b798b565e0701c2092a330deeba16bd3d896d401f7e9f28", size = 2522442, upload-time = "2025-10-13T23:10:02.134Z" }, + { url = "https://files.pythonhosted.org/packages/c9/04/75f8627cd7fe4d103eca51760d50269cfbc0bf6beaf83a3cdefb4ebd37c7/arro3_core-0.6.5-cp311-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:20679f874558bb2113e96325522625ec64a72687000b7a9578031a4d082c6ef5", size = 3033454, upload-time = "2025-10-13T23:10:05.192Z" }, + { url = "https://files.pythonhosted.org/packages/ea/19/f2d54985da65bf6d3da76218bee56383285035541c8d0cadb53095845b3e/arro3_core-0.6.5-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:d82d6ec32d5c7c73057fb9c528390289fd5bc94b8d8f28fca9c56fc8e41c412c", size = 2705984, upload-time = "2025-10-13T23:10:08.518Z" }, + { url = "https://files.pythonhosted.org/packages/6c/53/b1d7742d6db7b4aa44d3785956955d651b3ac36db321625fd15466be1aca/arro3_core-0.6.5-cp311-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:4cba4db0a4203a3ccf131c3fb7804d77f0740d6165ec9efa3aa3acbca87c43a3", size = 3157472, upload-time = "2025-10-13T23:10:11.976Z" }, + { url = "https://files.pythonhosted.org/packages/05/31/68711327dbdd480aed54158fc1c46ab245e860ab0286e0916ce788f9889e/arro3_core-0.6.5-cp311-abi3-musllinux_1_2_i686.whl", hash = "sha256:e358affc4a0fe5c1b5dccf4f92c43a836aaa4c4eab0906c83b00b60275de3b6d", size = 3117099, upload-time = "2025-10-13T23:10:15.374Z" }, + { url = "https://files.pythonhosted.org/packages/31/e3/15ffca0797d9500b23759ae4477cf052fde8dd47a3890f4e4e1d04639016/arro3_core-0.6.5-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:324e43f07b7681846d00a8995b78bdc4b4a719047aa0d34426b462b8f208ee98", size = 2963677, upload-time = "2025-10-13T23:10:18.828Z" }, + { url = "https://files.pythonhosted.org/packages/bc/02/69e60dbe3bbe2bfc8b6dfa4f4bfcb8d1dd240a137bf2a5f7bcc84703f05c/arro3_core-0.6.5-cp311-abi3-win_amd64.whl", hash = "sha256:285f802c8a42fe29ecb84584d1700bc4c4f974552b75f805e1f4362d28b97080", size = 2850445, upload-time = "2025-10-13T23:10:22.345Z" }, + { url = "https://files.pythonhosted.org/packages/b1/29/2e5b091f6b5cffb6489dbe7ed353841568dde8ac4d1232c77321da1d0925/arro3_core-0.6.5-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:8c20e69c3b3411fd6ed56091f388e699072651e880e682be5bd14f3a392ed3e8", size = 2671985, upload-time = "2025-10-13T23:10:25.515Z" }, + { url = "https://files.pythonhosted.org/packages/30/74/764ac4b58fef3fdfc655416c42349206156db5c687fa24a0674acaeaadbb/arro3_core-0.6.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:92211f1d03221ff74d0b535a576b39601083d8e98e9d47228314573f9d4f9ae2", size = 2382931, upload-time = "2025-10-13T23:10:29.893Z" }, + { url = "https://files.pythonhosted.org/packages/6a/07/bd8c92e218240ae8a30150a5d7a2dab359b452ab54a8bb7b90effe806e3d/arro3_core-0.6.5-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:280d933b75f2649779d76e32a07f91d2352a952f2c97ddf7b320e267f440cd42", size = 2879900, upload-time = "2025-10-13T23:10:33.238Z" }, + { url = "https://files.pythonhosted.org/packages/0f/d4/253725019fe2ae5f5fde87928118ffa568cc59f07b2d6a0e90620938c537/arro3_core-0.6.5-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bfc3f6b93b924f43fb7985b06202343c30b43da6bd5055ba8b84eda431e494d4", size = 2904149, upload-time = "2025-10-13T23:10:36.547Z" }, + { url = "https://files.pythonhosted.org/packages/f0/b0/7a3dea641ac8de041c1a34859a2f2a82d3cdf3c3360872101c1d198a1e24/arro3_core-0.6.5-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a5963635eb698ebc7da689e641f68b3998864bab894cf0ca84bd058b8c60d97f", size = 3143477, upload-time = "2025-10-13T23:10:40.232Z" }, + { url = "https://files.pythonhosted.org/packages/a7/05/1a50575be33fe9240898a1b5a8574658a905b5675865285585e070dcf7e2/arro3_core-0.6.5-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac291b3e74b57e56e03373d57530540cbbbfd92e4219fe2778ea531006673fe9", size = 2776522, upload-time = "2025-10-13T23:10:43.413Z" }, + { url = "https://files.pythonhosted.org/packages/2e/bd/e7b03207e7906e94e327cd4190fdb2d26ae52bc4ee1edeb057fed760796b/arro3_core-0.6.5-cp313-cp313t-manylinux_2_24_aarch64.whl", hash = "sha256:5d3f4cc58a654037d61f61ba230419da2c8f88a0ac82b9d41fe307f7cf9fda97", size = 2515426, upload-time = "2025-10-13T23:10:46.926Z" }, + { url = "https://files.pythonhosted.org/packages/f9/ed/82d1febd5c104eccdfb82434e3619125c328c36da143e19dfa3c86de4a81/arro3_core-0.6.5-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:93cddac90238d64451f5e66c630ded89d0b5fd6d2c099bf3a5151dde2c1ddf1d", size = 3024759, upload-time = "2025-10-13T23:10:50.281Z" }, + { url = "https://files.pythonhosted.org/packages/da/cd/00e06907e42e404c21eb08282dee94ac7a1961facfa9a96d116829031721/arro3_core-0.6.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:1fa7ac10db5846c33f4e8b66a6eaa705d84998e38575a835acac9a6a6649933d", size = 2700191, upload-time = "2025-10-13T23:10:53.776Z" }, + { url = "https://files.pythonhosted.org/packages/a3/11/a4bb9a900f456a6905d481bd2289f7a2371dcde024de56779621fd6a92c3/arro3_core-0.6.5-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:ca69f698a065cdbf845d59d412bc204e8f8af12f93737d82e6a18f3cff812349", size = 3149963, upload-time = "2025-10-13T23:10:57.163Z" }, + { url = "https://files.pythonhosted.org/packages/28/8a/79c76ad88b16f2fac25684f7313593738f353355eb1af2307e43efd7b1ca/arro3_core-0.6.5-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:de74a2512e2e2366d4b064c498c38672bf6ddea38acec8b1999b4e66182dd001", size = 3104663, upload-time = "2025-10-13T23:11:00.582Z" }, + { url = "https://files.pythonhosted.org/packages/20/66/9152feaa87f851a37c1a2bd74fb89d7e82e4c76447ee590bf8e6fff5e9d8/arro3_core-0.6.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:806ca8e20507675b2de68b3d009f76e898cc3c3e441c834ea5220866f68aac50", size = 2956440, upload-time = "2025-10-13T23:11:03.769Z" }, + { url = "https://files.pythonhosted.org/packages/ad/66/f4179ef64d5c18fe76ec93cfbff42c0f401438ef771c6766b880044d7e13/arro3_core-0.6.5-cp313-cp313t-win_amd64.whl", hash = "sha256:8f6f0cc78877ade7ad6e678a4671b191406547e7b407bc9637436869c017ed47", size = 2845345, upload-time = "2025-10-13T23:11:07.447Z" }, +] + +[[package]] +name = "attrs" +version = "25.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6b/5c/685e6633917e101e5dcb62b9dd76946cbb57c26e133bae9e0cd36033c0a9/attrs-25.4.0.tar.gz", hash = "sha256:16d5969b87f0859ef33a48b35d55ac1be6e42ae49d5e853b597db70c35c57e11", size = 934251, upload-time = "2025-10-06T13:54:44.725Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/2a/7cc015f5b9f5db42b7d48157e23356022889fc354a2813c15934b7cb5c0e/attrs-25.4.0-py3-none-any.whl", hash = "sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373", size = 67615, upload-time = "2025-10-06T13:54:43.17Z" }, +] + +[[package]] +name = "aws-cdk-asset-awscli-v1" +version = "2.2.263" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jsii" }, + { name = "publication" }, + { name = "typeguard" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ad/03/1bf33967bf1e2a196907266152743ec2dff96b842176084646f910dae732/aws_cdk_asset_awscli_v1-2.2.263.tar.gz", hash = "sha256:657605260ace055fac4ae30a6fb84a80504a6e24ce0b1d278913d4db4bafa266", size = 20229495, upload-time = "2026-01-19T17:25:28.242Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/af/02481a85fdd0b8a2a1fd9b84925591e1113a7a61b014dabe9d3f92dd6a67/aws_cdk_asset_awscli_v1-2.2.263-py3-none-any.whl", hash = "sha256:185150757d4216ea982d7b35596de5b3d767776be00cd78cacce02eaa08f8851", size = 20227990, upload-time = "2026-01-19T17:25:25.692Z" }, +] + +[[package]] +name = "aws-cdk-asset-node-proxy-agent-v6" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jsii" }, + { name = "publication" }, + { name = "typeguard" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d5/ab/09ac3ecc0067988d02398328e088d66cbe8555c991563c8ddfa1db5296ae/aws_cdk_asset_node_proxy_agent_v6-2.1.0.tar.gz", hash = "sha256:1f292c0631f86708ba4ee328b3a2b229f7e46ea1c79fbde567ee9eb119c2b0e2", size = 1540231, upload-time = "2024-09-03T09:36:51.634Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/86/1817a6da223aa80aeb94a504f07f930170284694b18f6053729e9930cc6a/aws_cdk.asset_node_proxy_agent_v6-2.1.0-py3-none-any.whl", hash = "sha256:24a388b69a44d03bae6dbf864c4e25ba650d4b61c008b4568b94ffbb9a69e40e", size = 1538724, upload-time = "2024-09-03T09:36:49.8Z" }, +] + +[[package]] +name = "aws-cdk-cloud-assembly-schema" +version = "48.20.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jsii" }, + { name = "publication" }, + { name = "typeguard" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a7/b5/1ce2f6bff913ca8c94a001b84290ec4ce3729f54a3af0e3ff0edb303ac20/aws_cdk_cloud_assembly_schema-48.20.0.tar.gz", hash = "sha256:229aa136c26b71b0a82b5a32658eabcd30e344f7e136315fdb6e3de8ef523bfa", size = 208109, upload-time = "2025-11-19T12:19:48.206Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/08/17a35f0b668451484f2254f5e50a0105958bffe90da11c41b7629972e6a9/aws_cdk_cloud_assembly_schema-48.20.0-py3-none-any.whl", hash = "sha256:f5b6cf661cac8690add9461de13aeae3f3742eec71c066032bd045b08d0b7c3e", size = 207669, upload-time = "2025-11-19T12:19:46.614Z" }, +] + +[[package]] +name = "aws-cdk-lib" +version = "2.236.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aws-cdk-asset-awscli-v1" }, + { name = "aws-cdk-asset-node-proxy-agent-v6" }, + { name = "aws-cdk-cloud-assembly-schema" }, + { name = "constructs" }, + { name = "jsii" }, + { name = "publication" }, + { name = "typeguard" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ce/6d/2b54697444a806257f19ed603ef34ff71b6c3beb6f916587087ffb016ff5/aws_cdk_lib-2.236.0.tar.gz", hash = "sha256:1ed9f3798101d3271fd219bc101b9eab41dead3a25dde516a3c16b8274e0e77a", size = 47209293, upload-time = "2026-01-23T17:39:34.624Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/53/82/6c47d68033aab1e778e226477d2863663ee59bac4345560a83f34f737e77/aws_cdk_lib-2.236.0-py3-none-any.whl", hash = "sha256:b724c1313a184ce5d62ff63f0594a1a4e12d098f7036fe07e7557a73229d0037", size = 47857717, upload-time = "2026-01-23T17:38:45.327Z" }, +] + +[[package]] +name = "cachetools" +version = "6.2.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/86/e7/18ea2907d2ca91e9c0697596b8e60cd485b091152eb4109fad1e468e457d/cachetools-6.2.5.tar.gz", hash = "sha256:6d8bfbba1ba94412fb9d9196c4da7a87e9d4928fffc5e93542965dca4740c77f", size = 32168, upload-time = "2026-01-25T14:57:40.349Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/a6/24169d70ec5264b65ba54ba49b3d10f46d6b1ad97e185c94556539b3dfc8/cachetools-6.2.5-py3-none-any.whl", hash = "sha256:db3ae5465e90befb7c74720dd9308d77a09b7cf13433570e07caa0845c30d5fe", size = 11553, upload-time = "2026-01-25T14:57:39.112Z" }, +] + +[[package]] +name = "cattrs" +version = "25.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6e/00/2432bb2d445b39b5407f0a90e01b9a271475eea7caf913d7a86bcb956385/cattrs-25.3.0.tar.gz", hash = "sha256:1ac88d9e5eda10436c4517e390a4142d88638fe682c436c93db7ce4a277b884a", size = 509321, upload-time = "2025-10-07T12:26:08.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d8/2b/a40e1488fdfa02d3f9a653a61a5935ea08b3c2225ee818db6a76c7ba9695/cattrs-25.3.0-py3-none-any.whl", hash = "sha256:9896e84e0a5bf723bc7b4b68f4481785367ce07a8a02e7e9ee6eb2819bc306ff", size = 70738, upload-time = "2025-10-07T12:26:06.603Z" }, +] + +[[package]] +name = "certifi" +version = "2026.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/2d/a891ca51311197f6ad14a7ef42e2399f36cf2f9bd44752b3dc4eab60fdc5/certifi-2026.1.4.tar.gz", hash = "sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120", size = 154268, upload-time = "2026-01-04T02:42:41.825Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e6/ad/3cc14f097111b4de0040c83a525973216457bbeeb63739ef1ed275c1c021/certifi-2026.1.4-py3-none-any.whl", hash = "sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c", size = 152900, upload-time = "2026-01-04T02:42:40.15Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/13/69/33ddede1939fdd074bce5434295f38fae7136463422fe4fd3e0e89b98062/charset_normalizer-3.4.4.tar.gz", hash = "sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a", size = 129418, upload-time = "2025-10-14T04:42:32.879Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/97/45/4b3a1239bbacd321068ea6e7ac28875b03ab8bc0aa0966452db17cd36714/charset_normalizer-3.4.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794", size = 208091, upload-time = "2025-10-14T04:41:13.346Z" }, + { url = "https://files.pythonhosted.org/packages/7d/62/73a6d7450829655a35bb88a88fca7d736f9882a27eacdca2c6d505b57e2e/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed", size = 147936, upload-time = "2025-10-14T04:41:14.461Z" }, + { url = "https://files.pythonhosted.org/packages/89/c5/adb8c8b3d6625bef6d88b251bbb0d95f8205831b987631ab0c8bb5d937c2/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72", size = 144180, upload-time = "2025-10-14T04:41:15.588Z" }, + { url = "https://files.pythonhosted.org/packages/91/ed/9706e4070682d1cc219050b6048bfd293ccf67b3d4f5a4f39207453d4b99/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:81d5eb2a312700f4ecaa977a8235b634ce853200e828fbadf3a9c50bab278328", size = 161346, upload-time = "2025-10-14T04:41:16.738Z" }, + { url = "https://files.pythonhosted.org/packages/d5/0d/031f0d95e4972901a2f6f09ef055751805ff541511dc1252ba3ca1f80cf5/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5bd2293095d766545ec1a8f612559f6b40abc0eb18bb2f5d1171872d34036ede", size = 158874, upload-time = "2025-10-14T04:41:17.923Z" }, + { url = "https://files.pythonhosted.org/packages/f5/83/6ab5883f57c9c801ce5e5677242328aa45592be8a00644310a008d04f922/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8a8b89589086a25749f471e6a900d3f662d1d3b6e2e59dcecf787b1cc3a1894", size = 153076, upload-time = "2025-10-14T04:41:19.106Z" }, + { url = "https://files.pythonhosted.org/packages/75/1e/5ff781ddf5260e387d6419959ee89ef13878229732732ee73cdae01800f2/charset_normalizer-3.4.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc7637e2f80d8530ee4a78e878bce464f70087ce73cf7c1caf142416923b98f1", size = 150601, upload-time = "2025-10-14T04:41:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/d7/57/71be810965493d3510a6ca79b90c19e48696fb1ff964da319334b12677f0/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f8bf04158c6b607d747e93949aa60618b61312fe647a6369f88ce2ff16043490", size = 150376, upload-time = "2025-10-14T04:41:21.398Z" }, + { url = "https://files.pythonhosted.org/packages/e5/d5/c3d057a78c181d007014feb7e9f2e65905a6c4ef182c0ddf0de2924edd65/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:554af85e960429cf30784dd47447d5125aaa3b99a6f0683589dbd27e2f45da44", size = 144825, upload-time = "2025-10-14T04:41:22.583Z" }, + { url = "https://files.pythonhosted.org/packages/e6/8c/d0406294828d4976f275ffbe66f00266c4b3136b7506941d87c00cab5272/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:74018750915ee7ad843a774364e13a3db91682f26142baddf775342c3f5b1133", size = 162583, upload-time = "2025-10-14T04:41:23.754Z" }, + { url = "https://files.pythonhosted.org/packages/d7/24/e2aa1f18c8f15c4c0e932d9287b8609dd30ad56dbe41d926bd846e22fb8d/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c0463276121fdee9c49b98908b3a89c39be45d86d1dbaa22957e38f6321d4ce3", size = 150366, upload-time = "2025-10-14T04:41:25.27Z" }, + { url = "https://files.pythonhosted.org/packages/e4/5b/1e6160c7739aad1e2df054300cc618b06bf784a7a164b0f238360721ab86/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:362d61fd13843997c1c446760ef36f240cf81d3ebf74ac62652aebaf7838561e", size = 160300, upload-time = "2025-10-14T04:41:26.725Z" }, + { url = "https://files.pythonhosted.org/packages/7a/10/f882167cd207fbdd743e55534d5d9620e095089d176d55cb22d5322f2afd/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9a26f18905b8dd5d685d6d07b0cdf98a79f3c7a918906af7cc143ea2e164c8bc", size = 154465, upload-time = "2025-10-14T04:41:28.322Z" }, + { url = "https://files.pythonhosted.org/packages/89/66/c7a9e1b7429be72123441bfdbaf2bc13faab3f90b933f664db506dea5915/charset_normalizer-3.4.4-cp313-cp313-win32.whl", hash = "sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac", size = 99404, upload-time = "2025-10-14T04:41:29.95Z" }, + { url = "https://files.pythonhosted.org/packages/c4/26/b9924fa27db384bdcd97ab83b4f0a8058d96ad9626ead570674d5e737d90/charset_normalizer-3.4.4-cp313-cp313-win_amd64.whl", hash = "sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14", size = 107092, upload-time = "2025-10-14T04:41:31.188Z" }, + { url = "https://files.pythonhosted.org/packages/af/8f/3ed4bfa0c0c72a7ca17f0380cd9e4dd842b09f664e780c13cff1dcf2ef1b/charset_normalizer-3.4.4-cp313-cp313-win_arm64.whl", hash = "sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2", size = 100408, upload-time = "2025-10-14T04:41:32.624Z" }, + { url = "https://files.pythonhosted.org/packages/2a/35/7051599bd493e62411d6ede36fd5af83a38f37c4767b92884df7301db25d/charset_normalizer-3.4.4-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd", size = 207746, upload-time = "2025-10-14T04:41:33.773Z" }, + { url = "https://files.pythonhosted.org/packages/10/9a/97c8d48ef10d6cd4fcead2415523221624bf58bcf68a802721a6bc807c8f/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb", size = 147889, upload-time = "2025-10-14T04:41:34.897Z" }, + { url = "https://files.pythonhosted.org/packages/10/bf/979224a919a1b606c82bd2c5fa49b5c6d5727aa47b4312bb27b1734f53cd/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e", size = 143641, upload-time = "2025-10-14T04:41:36.116Z" }, + { url = "https://files.pythonhosted.org/packages/ba/33/0ad65587441fc730dc7bd90e9716b30b4702dc7b617e6ba4997dc8651495/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14", size = 160779, upload-time = "2025-10-14T04:41:37.229Z" }, + { url = "https://files.pythonhosted.org/packages/67/ed/331d6b249259ee71ddea93f6f2f0a56cfebd46938bde6fcc6f7b9a3d0e09/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191", size = 159035, upload-time = "2025-10-14T04:41:38.368Z" }, + { url = "https://files.pythonhosted.org/packages/67/ff/f6b948ca32e4f2a4576aa129d8bed61f2e0543bf9f5f2b7fc3758ed005c9/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838", size = 152542, upload-time = "2025-10-14T04:41:39.862Z" }, + { url = "https://files.pythonhosted.org/packages/16/85/276033dcbcc369eb176594de22728541a925b2632f9716428c851b149e83/charset_normalizer-3.4.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6", size = 149524, upload-time = "2025-10-14T04:41:41.319Z" }, + { url = "https://files.pythonhosted.org/packages/9e/f2/6a2a1f722b6aba37050e626530a46a68f74e63683947a8acff92569f979a/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e", size = 150395, upload-time = "2025-10-14T04:41:42.539Z" }, + { url = "https://files.pythonhosted.org/packages/60/bb/2186cb2f2bbaea6338cad15ce23a67f9b0672929744381e28b0592676824/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c", size = 143680, upload-time = "2025-10-14T04:41:43.661Z" }, + { url = "https://files.pythonhosted.org/packages/7d/a5/bf6f13b772fbb2a90360eb620d52ed8f796f3c5caee8398c3b2eb7b1c60d/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090", size = 162045, upload-time = "2025-10-14T04:41:44.821Z" }, + { url = "https://files.pythonhosted.org/packages/df/c5/d1be898bf0dc3ef9030c3825e5d3b83f2c528d207d246cbabe245966808d/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152", size = 149687, upload-time = "2025-10-14T04:41:46.442Z" }, + { url = "https://files.pythonhosted.org/packages/a5/42/90c1f7b9341eef50c8a1cb3f098ac43b0508413f33affd762855f67a410e/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828", size = 160014, upload-time = "2025-10-14T04:41:47.631Z" }, + { url = "https://files.pythonhosted.org/packages/76/be/4d3ee471e8145d12795ab655ece37baed0929462a86e72372fd25859047c/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec", size = 154044, upload-time = "2025-10-14T04:41:48.81Z" }, + { url = "https://files.pythonhosted.org/packages/b0/6f/8f7af07237c34a1defe7defc565a9bc1807762f672c0fde711a4b22bf9c0/charset_normalizer-3.4.4-cp314-cp314-win32.whl", hash = "sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9", size = 99940, upload-time = "2025-10-14T04:41:49.946Z" }, + { url = "https://files.pythonhosted.org/packages/4b/51/8ade005e5ca5b0d80fb4aff72a3775b325bdc3d27408c8113811a7cbe640/charset_normalizer-3.4.4-cp314-cp314-win_amd64.whl", hash = "sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c", size = 107104, upload-time = "2025-10-14T04:41:51.051Z" }, + { url = "https://files.pythonhosted.org/packages/da/5f/6b8f83a55bb8278772c5ae54a577f3099025f9ade59d0136ac24a0df4bde/charset_normalizer-3.4.4-cp314-cp314-win_arm64.whl", hash = "sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2", size = 100743, upload-time = "2025-10-14T04:41:52.122Z" }, + { url = "https://files.pythonhosted.org/packages/0a/4c/925909008ed5a988ccbb72dcc897407e5d6d3bd72410d69e051fc0c14647/charset_normalizer-3.4.4-py3-none-any.whl", hash = "sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f", size = 53402, upload-time = "2025-10-14T04:42:31.76Z" }, +] + +[[package]] +name = "click" +version = "8.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/fa/656b739db8587d7b5dfa22e22ed02566950fbfbcdc20311993483657a5c0/click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a", size = 295065, upload-time = "2025-11-15T20:45:42.706Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/78/01c019cdb5d6498122777c1a43056ebb3ebfeef2076d9d026bfe15583b2b/click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6", size = 108274, upload-time = "2025-11-15T20:45:41.139Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "constructs" +version = "10.4.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jsii" }, + { name = "publication" }, + { name = "typeguard" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/55/ff7f5095a070af7db51c18e8d4a77aa92008db1fccc022b16c3444f80701/constructs-10.4.5.tar.gz", hash = "sha256:efa0f9e4fe65bd334bad719d00e0c8d9e3e4c6d873da8e874e67517a029c190b", size = 64803, upload-time = "2026-01-16T16:09:10.852Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cd/a1/0cbba03ce3c1377a788192163f2614f396a8614cf5022ac54bdc2085c078/constructs-10.4.5-py3-none-any.whl", hash = "sha256:e63d6675ba2e8a9076db8df1d4c7af78efdb96182ba28abe938384ba321e4b81", size = 63037, upload-time = "2026-01-16T16:09:09.178Z" }, +] + +[[package]] +name = "fsspec" +version = "2026.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d5/7d/5df2650c57d47c57232af5ef4b4fdbff182070421e405e0d62c6cdbfaa87/fsspec-2026.1.0.tar.gz", hash = "sha256:e987cb0496a0d81bba3a9d1cee62922fb395e7d4c3b575e57f547953334fe07b", size = 310496, upload-time = "2026-01-09T15:21:35.562Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/01/c9/97cc5aae1648dcb851958a3ddf73ccd7dbe5650d95203ecb4d7720b4cdbf/fsspec-2026.1.0-py3-none-any.whl", hash = "sha256:cb76aa913c2285a3b49bdd5fc55b1d7c708d7208126b60f2eb8194fe1b4cbdcc", size = 201838, upload-time = "2026-01-09T15:21:34.041Z" }, +] + +[[package]] +name = "geojson-pydantic" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5c/52/961c8f7c51067f5d853a732cd4abc09b4d15c742384406dda8348b98071e/geojson_pydantic-2.1.0.tar.gz", hash = "sha256:78a52b2a7cd9c113bac4898a81ce00c146c7927dd2804f1c7e9fd05c2515073f", size = 9398, upload-time = "2025-10-08T13:31:12.474Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/18/8a9dca353e605b344408114f6b045b11d14082d19f4668b073259d3ed1a9/geojson_pydantic-2.1.0-py3-none-any.whl", hash = "sha256:f9091bed334ab9fbb1bef113674edc1212a3737f374a0b13b1aa493f57964c1d", size = 8819, upload-time = "2025-10-08T13:31:11.646Z" }, +] + +[[package]] +name = "greenlet" +version = "3.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8a/99/1cd3411c56a410994669062bd73dd58270c00cc074cac15f385a1fd91f8a/greenlet-3.3.1.tar.gz", hash = "sha256:41848f3230b58c08bb43dee542e74a2a2e34d3c59dc3076cec9151aeeedcae98", size = 184690, upload-time = "2026-01-23T15:31:02.076Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/ab/d26750f2b7242c2b90ea2ad71de70cfcd73a948a49513188a0fc0d6fc15a/greenlet-3.3.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:7ab327905cabb0622adca5971e488064e35115430cec2c35a50fd36e72a315b3", size = 275205, upload-time = "2026-01-23T15:30:24.556Z" }, + { url = "https://files.pythonhosted.org/packages/10/d3/be7d19e8fad7c5a78eeefb2d896a08cd4643e1e90c605c4be3b46264998f/greenlet-3.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:65be2f026ca6a176f88fb935ee23c18333ccea97048076aef4db1ef5bc0713ac", size = 599284, upload-time = "2026-01-23T16:00:58.584Z" }, + { url = "https://files.pythonhosted.org/packages/ae/21/fe703aaa056fdb0f17e5afd4b5c80195bbdab701208918938bd15b00d39b/greenlet-3.3.1-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7a3ae05b3d225b4155bda56b072ceb09d05e974bc74be6c3fc15463cf69f33fd", size = 610274, upload-time = "2026-01-23T16:05:29.312Z" }, + { url = "https://files.pythonhosted.org/packages/06/00/95df0b6a935103c0452dad2203f5be8377e551b8466a29650c4c5a5af6cc/greenlet-3.3.1-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:12184c61e5d64268a160226fb4818af4df02cfead8379d7f8b99a56c3a54ff3e", size = 624375, upload-time = "2026-01-23T16:15:55.915Z" }, + { url = "https://files.pythonhosted.org/packages/cb/86/5c6ab23bb3c28c21ed6bebad006515cfe08b04613eb105ca0041fecca852/greenlet-3.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6423481193bbbe871313de5fd06a082f2649e7ce6e08015d2a76c1e9186ca5b3", size = 612904, upload-time = "2026-01-23T15:32:52.317Z" }, + { url = "https://files.pythonhosted.org/packages/c2/f3/7949994264e22639e40718c2daf6f6df5169bf48fb038c008a489ec53a50/greenlet-3.3.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:33a956fe78bbbda82bfc95e128d61129b32d66bcf0a20a1f0c08aa4839ffa951", size = 1567316, upload-time = "2026-01-23T16:04:23.316Z" }, + { url = "https://files.pythonhosted.org/packages/8d/6e/d73c94d13b6465e9f7cd6231c68abde838bb22408596c05d9059830b7872/greenlet-3.3.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4b065d3284be43728dd280f6f9a13990b56470b81be20375a207cdc814a983f2", size = 1636549, upload-time = "2026-01-23T15:33:48.643Z" }, + { url = "https://files.pythonhosted.org/packages/5e/b3/c9c23a6478b3bcc91f979ce4ca50879e4d0b2bd7b9a53d8ecded719b92e2/greenlet-3.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:27289986f4e5b0edec7b5a91063c109f0276abb09a7e9bdab08437525977c946", size = 227042, upload-time = "2026-01-23T15:33:58.216Z" }, + { url = "https://files.pythonhosted.org/packages/90/e7/824beda656097edee36ab15809fd063447b200cc03a7f6a24c34d520bc88/greenlet-3.3.1-cp313-cp313-win_arm64.whl", hash = "sha256:2f080e028001c5273e0b42690eaf359aeef9cb1389da0f171ea51a5dc3c7608d", size = 226294, upload-time = "2026-01-23T15:30:52.73Z" }, + { url = "https://files.pythonhosted.org/packages/ae/fb/011c7c717213182caf78084a9bea51c8590b0afda98001f69d9f853a495b/greenlet-3.3.1-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:bd59acd8529b372775cd0fcbc5f420ae20681c5b045ce25bd453ed8455ab99b5", size = 275737, upload-time = "2026-01-23T15:32:16.889Z" }, + { url = "https://files.pythonhosted.org/packages/41/2e/a3a417d620363fdbb08a48b1dd582956a46a61bf8fd27ee8164f9dfe87c2/greenlet-3.3.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b31c05dd84ef6871dd47120386aed35323c944d86c3d91a17c4b8d23df62f15b", size = 646422, upload-time = "2026-01-23T16:01:00.354Z" }, + { url = "https://files.pythonhosted.org/packages/b4/09/c6c4a0db47defafd2d6bab8ddfe47ad19963b4e30f5bed84d75328059f8c/greenlet-3.3.1-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:02925a0bfffc41e542c70aa14c7eda3593e4d7e274bfcccca1827e6c0875902e", size = 658219, upload-time = "2026-01-23T16:05:30.956Z" }, + { url = "https://files.pythonhosted.org/packages/e2/89/b95f2ddcc5f3c2bc09c8ee8d77be312df7f9e7175703ab780f2014a0e781/greenlet-3.3.1-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3e0f3878ca3a3ff63ab4ea478585942b53df66ddde327b59ecb191b19dbbd62d", size = 671455, upload-time = "2026-01-23T16:15:57.232Z" }, + { url = "https://files.pythonhosted.org/packages/80/38/9d42d60dffb04b45f03dbab9430898352dba277758640751dc5cc316c521/greenlet-3.3.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:34a729e2e4e4ffe9ae2408d5ecaf12f944853f40ad724929b7585bca808a9d6f", size = 660237, upload-time = "2026-01-23T15:32:53.967Z" }, + { url = "https://files.pythonhosted.org/packages/96/61/373c30b7197f9e756e4c81ae90a8d55dc3598c17673f91f4d31c3c689c3f/greenlet-3.3.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:aec9ab04e82918e623415947921dea15851b152b822661cce3f8e4393c3df683", size = 1615261, upload-time = "2026-01-23T16:04:25.066Z" }, + { url = "https://files.pythonhosted.org/packages/fd/d3/ca534310343f5945316f9451e953dcd89b36fe7a19de652a1dc5a0eeef3f/greenlet-3.3.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:71c767cf281a80d02b6c1bdc41c9468e1f5a494fb11bc8688c360524e273d7b1", size = 1683719, upload-time = "2026-01-23T15:33:50.61Z" }, + { url = "https://files.pythonhosted.org/packages/52/cb/c21a3fd5d2c9c8b622e7bede6d6d00e00551a5ee474ea6d831b5f567a8b4/greenlet-3.3.1-cp314-cp314-win_amd64.whl", hash = "sha256:96aff77af063b607f2489473484e39a0bbae730f2ea90c9e5606c9b73c44174a", size = 228125, upload-time = "2026-01-23T15:32:45.265Z" }, + { url = "https://files.pythonhosted.org/packages/6a/8e/8a2db6d11491837af1de64b8aff23707c6e85241be13c60ed399a72e2ef8/greenlet-3.3.1-cp314-cp314-win_arm64.whl", hash = "sha256:b066e8b50e28b503f604fa538adc764a638b38cf8e81e025011d26e8a627fa79", size = 227519, upload-time = "2026-01-23T15:31:47.284Z" }, + { url = "https://files.pythonhosted.org/packages/28/24/cbbec49bacdcc9ec652a81d3efef7b59f326697e7edf6ed775a5e08e54c2/greenlet-3.3.1-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:3e63252943c921b90abb035ebe9de832c436401d9c45f262d80e2d06cc659242", size = 282706, upload-time = "2026-01-23T15:33:05.525Z" }, + { url = "https://files.pythonhosted.org/packages/86/2e/4f2b9323c144c4fe8842a4e0d92121465485c3c2c5b9e9b30a52e80f523f/greenlet-3.3.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:76e39058e68eb125de10c92524573924e827927df5d3891fbc97bd55764a8774", size = 651209, upload-time = "2026-01-23T16:01:01.517Z" }, + { url = "https://files.pythonhosted.org/packages/d9/87/50ca60e515f5bb55a2fbc5f0c9b5b156de7d2fc51a0a69abc9d23914a237/greenlet-3.3.1-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9f9d5e7a9310b7a2f416dd13d2e3fd8b42d803968ea580b7c0f322ccb389b97", size = 654300, upload-time = "2026-01-23T16:05:32.199Z" }, + { url = "https://files.pythonhosted.org/packages/7c/25/c51a63f3f463171e09cb586eb64db0861eb06667ab01a7968371a24c4f3b/greenlet-3.3.1-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4b9721549a95db96689458a1e0ae32412ca18776ed004463df3a9299c1b257ab", size = 662574, upload-time = "2026-01-23T16:15:58.364Z" }, + { url = "https://files.pythonhosted.org/packages/1d/94/74310866dfa2b73dd08659a3d18762f83985ad3281901ba0ee9a815194fb/greenlet-3.3.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:92497c78adf3ac703b57f1e3813c2d874f27f71a178f9ea5887855da413cd6d2", size = 653842, upload-time = "2026-01-23T15:32:55.671Z" }, + { url = "https://files.pythonhosted.org/packages/97/43/8bf0ffa3d498eeee4c58c212a3905dd6146c01c8dc0b0a046481ca29b18c/greenlet-3.3.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ed6b402bc74d6557a705e197d47f9063733091ed6357b3de33619d8a8d93ac53", size = 1614917, upload-time = "2026-01-23T16:04:26.276Z" }, + { url = "https://files.pythonhosted.org/packages/89/90/a3be7a5f378fc6e84abe4dcfb2ba32b07786861172e502388b4c90000d1b/greenlet-3.3.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:59913f1e5ada20fde795ba906916aea25d442abcc0593fba7e26c92b7ad76249", size = 1676092, upload-time = "2026-01-23T15:33:52.176Z" }, + { url = "https://files.pythonhosted.org/packages/e1/2b/98c7f93e6db9977aaee07eb1e51ca63bd5f779b900d362791d3252e60558/greenlet-3.3.1-cp314-cp314t-win_amd64.whl", hash = "sha256:301860987846c24cb8964bdec0e31a96ad4a2a801b41b4ef40963c1b44f33451", size = 233181, upload-time = "2026-01-23T15:33:00.29Z" }, +] + +[[package]] +name = "icestac" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "pyarrow" }, + { name = "pydantic-settings" }, + { name = "pyiceberg", extra = ["pyiceberg-core"] }, + { name = "rustac", extra = ["arrow"] }, + { name = "stac-pydantic" }, +] + +[package.dev-dependencies] +deploy = [ + { name = "aws-cdk-lib" }, +] +dev = [ + { name = "pytest" }, + { name = "sqlalchemy" }, +] + +[package.metadata] +requires-dist = [ + { name = "pyarrow", specifier = ">=23.0.0" }, + { name = "pydantic-settings", specifier = ">=2.12.0" }, + { name = "pyiceberg", extras = ["pyiceberg-core"], specifier = ">=0.10.0" }, + { name = "rustac", extras = ["arrow"], specifier = ">=0.9.3" }, + { name = "stac-pydantic", specifier = ">=3.4.0" }, +] + +[package.metadata.requires-dev] +deploy = [{ name = "aws-cdk-lib", specifier = ">=2.236.0" }] +dev = [ + { name = "pytest", specifier = ">=9.0.2" }, + { name = "sqlalchemy", specifier = ">=2.0.46" }, +] + +[[package]] +name = "idna" +version = "3.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, +] + +[[package]] +name = "importlib-resources" +version = "6.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cf/8c/f834fbf984f691b4f7ff60f50b514cc3de5cc08abfc3295564dd89c5e2e7/importlib_resources-6.5.2.tar.gz", hash = "sha256:185f87adef5bcc288449d98fb4fba07cea78bc036455dd44c5fc4a2fe78fed2c", size = 44693, upload-time = "2025-01-03T18:51:56.698Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/ed/1f1afb2e9e7f38a545d628f864d562a5ae64fe6f7a10e28ffb9b185b4e89/importlib_resources-6.5.2-py3-none-any.whl", hash = "sha256:789cfdc3ed28c78b67a06acb8126751ced69a3d5f79c095a98298cd8a760ccec", size = 37461, upload-time = "2025-01-03T18:51:54.306Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "jsii" +version = "1.126.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "cattrs" }, + { name = "importlib-resources" }, + { name = "publication" }, + { name = "python-dateutil" }, + { name = "typeguard" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/22/a3c8a8bfc6bd25b761553380d91d9b5a772a3796e78b7f6d4eecc003bced/jsii-1.126.0.tar.gz", hash = "sha256:5e4739843aab3af25472490a05a271cf7d53f01a6d46167ab0f1f2cff3a8df95", size = 626608, upload-time = "2026-01-26T10:43:16.152Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/7d/f058cfdc20e1536b09d165303b1996d24504fbe105b0aba62c865e60bdc9/jsii-1.126.0-py3-none-any.whl", hash = "sha256:0bb3d5423fd62a499f9ce83e98668b48424ac6ef39472bff90cdf4650aa41b41", size = 602728, upload-time = "2026-01-26T10:43:14.68Z" }, +] + +[[package]] +name = "markdown-it-py" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" }, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + +[[package]] +name = "mmh3" +version = "5.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a7/af/f28c2c2f51f31abb4725f9a64bc7863d5f491f6539bd26aee2a1d21a649e/mmh3-5.2.0.tar.gz", hash = "sha256:1efc8fec8478e9243a78bb993422cf79f8ff85cb4cf6b79647480a31e0d950a8", size = 33582, upload-time = "2025-07-29T07:43:48.49Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d8/fa/27f6ab93995ef6ad9f940e96593c5dd24744d61a7389532b0fec03745607/mmh3-5.2.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:e79c00eba78f7258e5b354eccd4d7907d60317ced924ea4a5f2e9d83f5453065", size = 40874, upload-time = "2025-07-29T07:42:30.662Z" }, + { url = "https://files.pythonhosted.org/packages/11/9c/03d13bcb6a03438bc8cac3d2e50f80908d159b31a4367c2e1a7a077ded32/mmh3-5.2.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:956127e663d05edbeec54df38885d943dfa27406594c411139690485128525de", size = 42012, upload-time = "2025-07-29T07:42:31.539Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/0865d9765408a7d504f1789944e678f74e0888b96a766d578cb80b040999/mmh3-5.2.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:c3dca4cb5b946ee91b3d6bb700d137b1cd85c20827f89fdf9c16258253489044", size = 39197, upload-time = "2025-07-29T07:42:32.374Z" }, + { url = "https://files.pythonhosted.org/packages/3e/12/76c3207bd186f98b908b6706c2317abb73756d23a4e68ea2bc94825b9015/mmh3-5.2.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:e651e17bfde5840e9e4174b01e9e080ce49277b70d424308b36a7969d0d1af73", size = 39840, upload-time = "2025-07-29T07:42:33.227Z" }, + { url = "https://files.pythonhosted.org/packages/5d/0d/574b6cce5555c9f2b31ea189ad44986755eb14e8862db28c8b834b8b64dc/mmh3-5.2.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:9f64bf06f4bf623325fda3a6d02d36cd69199b9ace99b04bb2d7fd9f89688504", size = 40644, upload-time = "2025-07-29T07:42:34.099Z" }, + { url = "https://files.pythonhosted.org/packages/52/82/3731f8640b79c46707f53ed72034a58baad400be908c87b0088f1f89f986/mmh3-5.2.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ddc63328889bcaee77b743309e5c7d2d52cee0d7d577837c91b6e7cc9e755e0b", size = 56153, upload-time = "2025-07-29T07:42:35.031Z" }, + { url = "https://files.pythonhosted.org/packages/4f/34/e02dca1d4727fd9fdeaff9e2ad6983e1552804ce1d92cc796e5b052159bb/mmh3-5.2.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:bb0fdc451fb6d86d81ab8f23d881b8d6e37fc373a2deae1c02d27002d2ad7a05", size = 40684, upload-time = "2025-07-29T07:42:35.914Z" }, + { url = "https://files.pythonhosted.org/packages/8f/36/3dee40767356e104967e6ed6d102ba47b0b1ce2a89432239b95a94de1b89/mmh3-5.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b29044e1ffdb84fe164d0a7ea05c7316afea93c00f8ed9449cf357c36fc4f814", size = 40057, upload-time = "2025-07-29T07:42:36.755Z" }, + { url = "https://files.pythonhosted.org/packages/31/58/228c402fccf76eb39a0a01b8fc470fecf21965584e66453b477050ee0e99/mmh3-5.2.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:58981d6ea9646dbbf9e59a30890cbf9f610df0e4a57dbfe09215116fd90b0093", size = 97344, upload-time = "2025-07-29T07:42:37.675Z" }, + { url = "https://files.pythonhosted.org/packages/34/82/fc5ce89006389a6426ef28e326fc065b0fbaaed230373b62d14c889f47ea/mmh3-5.2.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7e5634565367b6d98dc4aa2983703526ef556b3688ba3065edb4b9b90ede1c54", size = 103325, upload-time = "2025-07-29T07:42:38.591Z" }, + { url = "https://files.pythonhosted.org/packages/09/8c/261e85777c6aee1ebd53f2f17e210e7481d5b0846cd0b4a5c45f1e3761b8/mmh3-5.2.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0271ac12415afd3171ab9a3c7cbfc71dee2c68760a7dc9d05bf8ed6ddfa3a7a", size = 106240, upload-time = "2025-07-29T07:42:39.563Z" }, + { url = "https://files.pythonhosted.org/packages/70/73/2f76b3ad8a3d431824e9934403df36c0ddacc7831acf82114bce3c4309c8/mmh3-5.2.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:45b590e31bc552c6f8e2150ff1ad0c28dd151e9f87589e7eaf508fbdd8e8e908", size = 113060, upload-time = "2025-07-29T07:42:40.585Z" }, + { url = "https://files.pythonhosted.org/packages/9f/b9/7ea61a34e90e50a79a9d87aa1c0b8139a7eaf4125782b34b7d7383472633/mmh3-5.2.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bdde97310d59604f2a9119322f61b31546748499a21b44f6715e8ced9308a6c5", size = 120781, upload-time = "2025-07-29T07:42:41.618Z" }, + { url = "https://files.pythonhosted.org/packages/0f/5b/ae1a717db98c7894a37aeedbd94b3f99e6472a836488f36b6849d003485b/mmh3-5.2.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:fc9c5f280438cf1c1a8f9abb87dc8ce9630a964120cfb5dd50d1e7ce79690c7a", size = 99174, upload-time = "2025-07-29T07:42:42.587Z" }, + { url = "https://files.pythonhosted.org/packages/e3/de/000cce1d799fceebb6d4487ae29175dd8e81b48e314cba7b4da90bcf55d7/mmh3-5.2.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:c903e71fd8debb35ad2a4184c1316b3cb22f64ce517b4e6747f25b0a34e41266", size = 98734, upload-time = "2025-07-29T07:42:43.996Z" }, + { url = "https://files.pythonhosted.org/packages/79/19/0dc364391a792b72fbb22becfdeacc5add85cc043cd16986e82152141883/mmh3-5.2.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:eed4bba7ff8a0d37106ba931ab03bdd3915fbb025bcf4e1f0aa02bc8114960c5", size = 106493, upload-time = "2025-07-29T07:42:45.07Z" }, + { url = "https://files.pythonhosted.org/packages/3c/b1/bc8c28e4d6e807bbb051fefe78e1156d7f104b89948742ad310612ce240d/mmh3-5.2.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:1fdb36b940e9261aff0b5177c5b74a36936b902f473180f6c15bde26143681a9", size = 110089, upload-time = "2025-07-29T07:42:46.122Z" }, + { url = "https://files.pythonhosted.org/packages/3b/a2/d20f3f5c95e9c511806686c70d0a15479cc3941c5f322061697af1c1ff70/mmh3-5.2.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7303aab41e97adcf010a09efd8f1403e719e59b7705d5e3cfed3dd7571589290", size = 97571, upload-time = "2025-07-29T07:42:47.18Z" }, + { url = "https://files.pythonhosted.org/packages/7b/23/665296fce4f33488deec39a750ffd245cfc07aafb0e3ef37835f91775d14/mmh3-5.2.0-cp313-cp313-win32.whl", hash = "sha256:03e08c6ebaf666ec1e3d6ea657a2d363bb01effd1a9acfe41f9197decaef0051", size = 40806, upload-time = "2025-07-29T07:42:48.166Z" }, + { url = "https://files.pythonhosted.org/packages/59/b0/92e7103f3b20646e255b699e2d0327ce53a3f250e44367a99dc8be0b7c7a/mmh3-5.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:7fddccd4113e7b736706e17a239a696332360cbaddf25ae75b57ba1acce65081", size = 41600, upload-time = "2025-07-29T07:42:49.371Z" }, + { url = "https://files.pythonhosted.org/packages/99/22/0b2bd679a84574647de538c5b07ccaa435dbccc37815067fe15b90fe8dad/mmh3-5.2.0-cp313-cp313-win_arm64.whl", hash = "sha256:fa0c966ee727aad5406d516375593c5f058c766b21236ab8985693934bb5085b", size = 39349, upload-time = "2025-07-29T07:42:50.268Z" }, + { url = "https://files.pythonhosted.org/packages/f7/ca/a20db059a8a47048aaf550da14a145b56e9c7386fb8280d3ce2962dcebf7/mmh3-5.2.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:e5015f0bb6eb50008bed2d4b1ce0f2a294698a926111e4bb202c0987b4f89078", size = 39209, upload-time = "2025-07-29T07:42:51.559Z" }, + { url = "https://files.pythonhosted.org/packages/98/dd/e5094799d55c7482d814b979a0fd608027d0af1b274bfb4c3ea3e950bfd5/mmh3-5.2.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:e0f3ed828d709f5b82d8bfe14f8856120718ec4bd44a5b26102c3030a1e12501", size = 39843, upload-time = "2025-07-29T07:42:52.536Z" }, + { url = "https://files.pythonhosted.org/packages/f4/6b/7844d7f832c85400e7cc89a1348e4e1fdd38c5a38415bb5726bbb8fcdb6c/mmh3-5.2.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:f35727c5118aba95f0397e18a1a5b8405425581bfe53e821f0fb444cbdc2bc9b", size = 40648, upload-time = "2025-07-29T07:42:53.392Z" }, + { url = "https://files.pythonhosted.org/packages/1f/bf/71f791f48a21ff3190ba5225807cbe4f7223360e96862c376e6e3fb7efa7/mmh3-5.2.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3bc244802ccab5220008cb712ca1508cb6a12f0eb64ad62997156410579a1770", size = 56164, upload-time = "2025-07-29T07:42:54.267Z" }, + { url = "https://files.pythonhosted.org/packages/70/1f/f87e3d34d83032b4f3f0f528c6d95a98290fcacf019da61343a49dccfd51/mmh3-5.2.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ff3d50dc3fe8a98059f99b445dfb62792b5d006c5e0b8f03c6de2813b8376110", size = 40692, upload-time = "2025-07-29T07:42:55.234Z" }, + { url = "https://files.pythonhosted.org/packages/a6/e2/db849eaed07117086f3452feca8c839d30d38b830ac59fe1ce65af8be5ad/mmh3-5.2.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:37a358cc881fe796e099c1db6ce07ff757f088827b4e8467ac52b7a7ffdca647", size = 40068, upload-time = "2025-07-29T07:42:56.158Z" }, + { url = "https://files.pythonhosted.org/packages/df/6b/209af927207af77425b044e32f77f49105a0b05d82ff88af6971d8da4e19/mmh3-5.2.0-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:b9a87025121d1c448f24f27ff53a5fe7b6ef980574b4a4f11acaabe702420d63", size = 97367, upload-time = "2025-07-29T07:42:57.037Z" }, + { url = "https://files.pythonhosted.org/packages/ca/e0/78adf4104c425606a9ce33fb351f790c76a6c2314969c4a517d1ffc92196/mmh3-5.2.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1ba55d6ca32eeef8b2625e1e4bfc3b3db52bc63014bd7e5df8cc11bf2b036b12", size = 103306, upload-time = "2025-07-29T07:42:58.522Z" }, + { url = "https://files.pythonhosted.org/packages/a3/79/c2b89f91b962658b890104745b1b6c9ce38d50a889f000b469b91eeb1b9e/mmh3-5.2.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c9ff37ba9f15637e424c2ab57a1a590c52897c845b768e4e0a4958084ec87f22", size = 106312, upload-time = "2025-07-29T07:42:59.552Z" }, + { url = "https://files.pythonhosted.org/packages/4b/14/659d4095528b1a209be90934778c5ffe312177d51e365ddcbca2cac2ec7c/mmh3-5.2.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a094319ec0db52a04af9fdc391b4d39a1bc72bc8424b47c4411afb05413a44b5", size = 113135, upload-time = "2025-07-29T07:43:00.745Z" }, + { url = "https://files.pythonhosted.org/packages/8d/6f/cd7734a779389a8a467b5c89a48ff476d6f2576e78216a37551a97e9e42a/mmh3-5.2.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c5584061fd3da584659b13587f26c6cad25a096246a481636d64375d0c1f6c07", size = 120775, upload-time = "2025-07-29T07:43:02.124Z" }, + { url = "https://files.pythonhosted.org/packages/1d/ca/8256e3b96944408940de3f9291d7e38a283b5761fe9614d4808fcf27bd62/mmh3-5.2.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ecbfc0437ddfdced5e7822d1ce4855c9c64f46819d0fdc4482c53f56c707b935", size = 99178, upload-time = "2025-07-29T07:43:03.182Z" }, + { url = "https://files.pythonhosted.org/packages/8a/32/39e2b3cf06b6e2eb042c984dab8680841ac2a0d3ca6e0bea30db1f27b565/mmh3-5.2.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:7b986d506a8e8ea345791897ba5d8ba0d9d8820cd4fc3e52dbe6de19388de2e7", size = 98738, upload-time = "2025-07-29T07:43:04.207Z" }, + { url = "https://files.pythonhosted.org/packages/61/d3/7bbc8e0e8cf65ebbe1b893ffa0467b7ecd1bd07c3bbf6c9db4308ada22ec/mmh3-5.2.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:38d899a156549da8ef6a9f1d6f7ef231228d29f8f69bce2ee12f5fba6d6fd7c5", size = 106510, upload-time = "2025-07-29T07:43:05.656Z" }, + { url = "https://files.pythonhosted.org/packages/10/99/b97e53724b52374e2f3859046f0eb2425192da356cb19784d64bc17bb1cf/mmh3-5.2.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:d86651fa45799530885ba4dab3d21144486ed15285e8784181a0ab37a4552384", size = 110053, upload-time = "2025-07-29T07:43:07.204Z" }, + { url = "https://files.pythonhosted.org/packages/ac/62/3688c7d975ed195155671df68788c83fed6f7909b6ec4951724c6860cb97/mmh3-5.2.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c463d7c1c4cfc9d751efeaadd936bbba07b5b0ed81a012b3a9f5a12f0872bd6e", size = 97546, upload-time = "2025-07-29T07:43:08.226Z" }, + { url = "https://files.pythonhosted.org/packages/ca/3b/c6153250f03f71a8b7634cded82939546cdfba02e32f124ff51d52c6f991/mmh3-5.2.0-cp314-cp314-win32.whl", hash = "sha256:bb4fe46bdc6104fbc28db7a6bacb115ee6368ff993366bbd8a2a7f0076e6f0c0", size = 41422, upload-time = "2025-07-29T07:43:09.216Z" }, + { url = "https://files.pythonhosted.org/packages/74/01/a27d98bab083a435c4c07e9d1d720d4c8a578bf4c270bae373760b1022be/mmh3-5.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:7c7f0b342fd06044bedd0b6e72177ddc0076f54fd89ee239447f8b271d919d9b", size = 42135, upload-time = "2025-07-29T07:43:10.183Z" }, + { url = "https://files.pythonhosted.org/packages/cb/c9/dbba5507e95429b8b380e2ba091eff5c20a70a59560934dff0ad8392b8c8/mmh3-5.2.0-cp314-cp314-win_arm64.whl", hash = "sha256:3193752fc05ea72366c2b63ff24b9a190f422e32d75fdeae71087c08fff26115", size = 39879, upload-time = "2025-07-29T07:43:11.106Z" }, + { url = "https://files.pythonhosted.org/packages/b5/d1/c8c0ef839c17258b9de41b84f663574fabcf8ac2007b7416575e0f65ff6e/mmh3-5.2.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:69fc339d7202bea69ef9bd7c39bfdf9fdabc8e6822a01eba62fb43233c1b3932", size = 57696, upload-time = "2025-07-29T07:43:11.989Z" }, + { url = "https://files.pythonhosted.org/packages/2f/55/95e2b9ff201e89f9fe37036037ab61a6c941942b25cdb7b6a9df9b931993/mmh3-5.2.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:12da42c0a55c9d86ab566395324213c319c73ecb0c239fad4726324212b9441c", size = 41421, upload-time = "2025-07-29T07:43:13.269Z" }, + { url = "https://files.pythonhosted.org/packages/77/79/9be23ad0b7001a4b22752e7693be232428ecc0a35068a4ff5c2f14ef8b20/mmh3-5.2.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f7f9034c7cf05ddfaac8d7a2e63a3c97a840d4615d0a0e65ba8bdf6f8576e3be", size = 40853, upload-time = "2025-07-29T07:43:14.888Z" }, + { url = "https://files.pythonhosted.org/packages/ac/1b/96b32058eda1c1dee8264900c37c359a7325c1f11f5ff14fd2be8e24eff9/mmh3-5.2.0-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:11730eeb16dfcf9674fdea9bb6b8e6dd9b40813b7eb839bc35113649eef38aeb", size = 109694, upload-time = "2025-07-29T07:43:15.816Z" }, + { url = "https://files.pythonhosted.org/packages/8d/6f/a2ae44cd7dad697b6dea48390cbc977b1e5ca58fda09628cbcb2275af064/mmh3-5.2.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:932a6eec1d2e2c3c9e630d10f7128d80e70e2d47fe6b8c7ea5e1afbd98733e65", size = 117438, upload-time = "2025-07-29T07:43:16.865Z" }, + { url = "https://files.pythonhosted.org/packages/a0/08/bfb75451c83f05224a28afeaf3950c7b793c0b71440d571f8e819cfb149a/mmh3-5.2.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ca975c51c5028947bbcfc24966517aac06a01d6c921e30f7c5383c195f87991", size = 120409, upload-time = "2025-07-29T07:43:18.207Z" }, + { url = "https://files.pythonhosted.org/packages/9f/ea/8b118b69b2ff8df568f742387d1a159bc654a0f78741b31437dd047ea28e/mmh3-5.2.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5b0b58215befe0f0e120b828f7645e97719bbba9f23b69e268ed0ac7adde8645", size = 125909, upload-time = "2025-07-29T07:43:19.39Z" }, + { url = "https://files.pythonhosted.org/packages/3e/11/168cc0b6a30650032e351a3b89b8a47382da541993a03af91e1ba2501234/mmh3-5.2.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29c2b9ce61886809d0492a274a5a53047742dea0f703f9c4d5d223c3ea6377d3", size = 135331, upload-time = "2025-07-29T07:43:20.435Z" }, + { url = "https://files.pythonhosted.org/packages/31/05/e3a9849b1c18a7934c64e831492c99e67daebe84a8c2f2c39a7096a830e3/mmh3-5.2.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:a367d4741ac0103f8198c82f429bccb9359f543ca542b06a51f4f0332e8de279", size = 110085, upload-time = "2025-07-29T07:43:21.92Z" }, + { url = "https://files.pythonhosted.org/packages/d9/d5/a96bcc306e3404601418b2a9a370baec92af84204528ba659fdfe34c242f/mmh3-5.2.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:5a5dba98e514fb26241868f6eb90a7f7ca0e039aed779342965ce24ea32ba513", size = 111195, upload-time = "2025-07-29T07:43:23.066Z" }, + { url = "https://files.pythonhosted.org/packages/af/29/0fd49801fec5bff37198684e0849b58e0dab3a2a68382a357cfffb0fafc3/mmh3-5.2.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:941603bfd75a46023807511c1ac2f1b0f39cccc393c15039969806063b27e6db", size = 116919, upload-time = "2025-07-29T07:43:24.178Z" }, + { url = "https://files.pythonhosted.org/packages/2d/04/4f3c32b0a2ed762edca45d8b46568fc3668e34f00fb1e0a3b5451ec1281c/mmh3-5.2.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:132dd943451a7c7546978863d2f5a64977928410782e1a87d583cb60eb89e667", size = 123160, upload-time = "2025-07-29T07:43:25.26Z" }, + { url = "https://files.pythonhosted.org/packages/91/76/3d29eaa38821730633d6a240d36fa8ad2807e9dfd432c12e1a472ed211eb/mmh3-5.2.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f698733a8a494466432d611a8f0d1e026f5286dee051beea4b3c3146817e35d5", size = 110206, upload-time = "2025-07-29T07:43:26.699Z" }, + { url = "https://files.pythonhosted.org/packages/44/1c/ccf35892684d3a408202e296e56843743e0b4fb1629e59432ea88cdb3909/mmh3-5.2.0-cp314-cp314t-win32.whl", hash = "sha256:6d541038b3fc360ec538fc116de87462627944765a6750308118f8b509a8eec7", size = 41970, upload-time = "2025-07-29T07:43:27.666Z" }, + { url = "https://files.pythonhosted.org/packages/75/b2/b9e4f1e5adb5e21eb104588fcee2cd1eaa8308255173481427d5ecc4284e/mmh3-5.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:e912b19cf2378f2967d0c08e86ff4c6c360129887f678e27e4dde970d21b3f4d", size = 43063, upload-time = "2025-07-29T07:43:28.582Z" }, + { url = "https://files.pythonhosted.org/packages/6a/fc/0e61d9a4e29c8679356795a40e48f647b4aad58d71bfc969f0f8f56fb912/mmh3-5.2.0-cp314-cp314t-win_arm64.whl", hash = "sha256:e7884931fe5e788163e7b3c511614130c2c59feffdc21112290a194487efb2e9", size = 40455, upload-time = "2025-07-29T07:43:29.563Z" }, +] + +[[package]] +name = "packaging" +version = "26.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", size = 143416, upload-time = "2026-01-21T20:50:39.064Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "publication" +version = "0.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6b/8e/8c9fe7e32fdf9c386f83d59610cc819a25dadb874b5920f2d0ef7d35f46d/publication-0.0.3.tar.gz", hash = "sha256:68416a0de76dddcdd2930d1c8ef853a743cc96c82416c4e4d3b5d901c6276dc4", size = 5484, upload-time = "2019-01-15T07:52:23.914Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/d3/6308debad7afcdb3ea5f50b4b3d852f41eb566a311fbcb4da23755a28155/publication-0.0.3-py2.py3-none-any.whl", hash = "sha256:0248885351febc11d8a1098d5c8e3ab2dabcf3e8c0c96db1e17ecd12b53afbe6", size = 7687, upload-time = "2019-01-15T07:52:22.151Z" }, +] + +[[package]] +name = "pyarrow" +version = "23.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/33/ffd9c3eb087fa41dd79c3cf20c4c0ae3cdb877c4f8e1107a446006344924/pyarrow-23.0.0.tar.gz", hash = "sha256:180e3150e7edfcd182d3d9afba72f7cf19839a497cc76555a8dce998a8f67615", size = 1167185, upload-time = "2026-01-18T16:19:42.218Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/66/34/564db447d083ec7ff93e0a883a597d2f214e552823bfc178a2d0b1f2c257/pyarrow-23.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:ad96a597547af7827342ffb3c503c8316e5043bb09b47a84885ce39394c96e00", size = 34184630, upload-time = "2026-01-18T16:16:22.141Z" }, + { url = "https://files.pythonhosted.org/packages/aa/3a/3999daebcb5e6119690c92a621c4d78eef2ffba7a0a1b56386d2875fcd77/pyarrow-23.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:b9edf990df77c2901e79608f08c13fbde60202334a4fcadb15c1f57bf7afee43", size = 35796820, upload-time = "2026-01-18T16:16:29.441Z" }, + { url = "https://files.pythonhosted.org/packages/ec/ee/39195233056c6a8d0976d7d1ac1cd4fe21fb0ec534eca76bc23ef3f60e11/pyarrow-23.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:36d1b5bc6ddcaff0083ceec7e2561ed61a51f49cce8be079ee8ed406acb6fdef", size = 44438735, upload-time = "2026-01-18T16:16:38.79Z" }, + { url = "https://files.pythonhosted.org/packages/2c/41/6a7328ee493527e7afc0c88d105ecca69a3580e29f2faaeac29308369fd7/pyarrow-23.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:4292b889cd224f403304ddda8b63a36e60f92911f89927ec8d98021845ea21be", size = 47557263, upload-time = "2026-01-18T16:16:46.248Z" }, + { url = "https://files.pythonhosted.org/packages/c6/ee/34e95b21ee84db494eae60083ddb4383477b31fb1fd19fd866d794881696/pyarrow-23.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:dfd9e133e60eaa847fd80530a1b89a052f09f695d0b9c34c235ea6b2e0924cf7", size = 48153529, upload-time = "2026-01-18T16:16:53.412Z" }, + { url = "https://files.pythonhosted.org/packages/52/88/8a8d83cea30f4563efa1b7bf51d241331ee5cd1b185a7e063f5634eca415/pyarrow-23.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:832141cc09fac6aab1cd3719951d23301396968de87080c57c9a7634e0ecd068", size = 50598851, upload-time = "2026-01-18T16:17:01.133Z" }, + { url = "https://files.pythonhosted.org/packages/c6/4c/2929c4be88723ba025e7b3453047dc67e491c9422965c141d24bab6b5962/pyarrow-23.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:7a7d067c9a88faca655c71bcc30ee2782038d59c802d57950826a07f60d83c4c", size = 27577747, upload-time = "2026-01-18T16:18:02.413Z" }, + { url = "https://files.pythonhosted.org/packages/64/52/564a61b0b82d72bd68ec3aef1adda1e3eba776f89134b9ebcb5af4b13cb6/pyarrow-23.0.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:ce9486e0535a843cf85d990e2ec5820a47918235183a5c7b8b97ed7e92c2d47d", size = 34446038, upload-time = "2026-01-18T16:17:07.861Z" }, + { url = "https://files.pythonhosted.org/packages/cc/c9/232d4f9855fd1de0067c8a7808a363230d223c83aeee75e0fe6eab851ba9/pyarrow-23.0.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:075c29aeaa685fd1182992a9ed2499c66f084ee54eea47da3eb76e125e06064c", size = 35921142, upload-time = "2026-01-18T16:17:15.401Z" }, + { url = "https://files.pythonhosted.org/packages/96/f2/60af606a3748367b906bb82d41f0032e059f075444445d47e32a7ff1df62/pyarrow-23.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:799965a5379589510d888be3094c2296efd186a17ca1cef5b77703d4d5121f53", size = 44490374, upload-time = "2026-01-18T16:17:23.93Z" }, + { url = "https://files.pythonhosted.org/packages/ff/2d/7731543050a678ea3a413955a2d5d80d2a642f270aa57a3cb7d5a86e3f46/pyarrow-23.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:ef7cac8fe6fccd8b9e7617bfac785b0371a7fe26af59463074e4882747145d40", size = 47527896, upload-time = "2026-01-18T16:17:33.393Z" }, + { url = "https://files.pythonhosted.org/packages/5a/90/f3342553b7ac9879413aed46500f1637296f3c8222107523a43a1c08b42a/pyarrow-23.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:15a414f710dc927132dd67c361f78c194447479555af57317066ee5116b90e9e", size = 48210401, upload-time = "2026-01-18T16:17:42.012Z" }, + { url = "https://files.pythonhosted.org/packages/f3/da/9862ade205ecc46c172b6ce5038a74b5151c7401e36255f15975a45878b2/pyarrow-23.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e0d2e6915eca7d786be6a77bf227fbc06d825a75b5b5fe9bcbef121dec32685", size = 50579677, upload-time = "2026-01-18T16:17:50.241Z" }, + { url = "https://files.pythonhosted.org/packages/c2/4c/f11f371f5d4740a5dafc2e11c76bcf42d03dfdb2d68696da97de420b6963/pyarrow-23.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:4b317ea6e800b5704e5e5929acb6e2dc13e9276b708ea97a39eb8b345aa2658b", size = 27631889, upload-time = "2026-01-18T16:17:56.55Z" }, + { url = "https://files.pythonhosted.org/packages/97/bb/15aec78bcf43a0c004067bd33eb5352836a29a49db8581fc56f2b6ca88b7/pyarrow-23.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:20b187ed9550d233a872074159f765f52f9d92973191cd4b93f293a19efbe377", size = 34213265, upload-time = "2026-01-18T16:18:07.904Z" }, + { url = "https://files.pythonhosted.org/packages/f6/6c/deb2c594bbba41c37c5d9aa82f510376998352aa69dfcb886cb4b18ad80f/pyarrow-23.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:18ec84e839b493c3886b9b5e06861962ab4adfaeb79b81c76afbd8d84c7d5fda", size = 35819211, upload-time = "2026-01-18T16:18:13.94Z" }, + { url = "https://files.pythonhosted.org/packages/e0/e5/ee82af693cb7b5b2b74f6524cdfede0e6ace779d7720ebca24d68b57c36b/pyarrow-23.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:e438dd3f33894e34fd02b26bd12a32d30d006f5852315f611aa4add6c7fab4bc", size = 44502313, upload-time = "2026-01-18T16:18:20.367Z" }, + { url = "https://files.pythonhosted.org/packages/9c/86/95c61ad82236495f3c31987e85135926ba3ec7f3819296b70a68d8066b49/pyarrow-23.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:a244279f240c81f135631be91146d7fa0e9e840e1dfed2aba8483eba25cd98e6", size = 47585886, upload-time = "2026-01-18T16:18:27.544Z" }, + { url = "https://files.pythonhosted.org/packages/bb/6e/a72d901f305201802f016d015de1e05def7706fff68a1dedefef5dc7eff7/pyarrow-23.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c4692e83e42438dba512a570c6eaa42be2f8b6c0f492aea27dec54bdc495103a", size = 48207055, upload-time = "2026-01-18T16:18:35.425Z" }, + { url = "https://files.pythonhosted.org/packages/f9/e5/5de029c537630ca18828db45c30e2a78da03675a70ac6c3528203c416fe3/pyarrow-23.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae7f30f898dfe44ea69654a35c93e8da4cef6606dc4c72394068fd95f8e9f54a", size = 50619812, upload-time = "2026-01-18T16:18:43.553Z" }, + { url = "https://files.pythonhosted.org/packages/59/8d/2af846cd2412e67a087f5bda4a8e23dfd4ebd570f777db2e8686615dafc1/pyarrow-23.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:5b86bb649e4112fb0614294b7d0a175c7513738876b89655605ebb87c804f861", size = 28263851, upload-time = "2026-01-18T16:19:38.567Z" }, + { url = "https://files.pythonhosted.org/packages/7b/7f/caab863e587041156f6786c52e64151b7386742c8c27140f637176e9230e/pyarrow-23.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:ebc017d765d71d80a3f8584ca0566b53e40464586585ac64176115baa0ada7d3", size = 34463240, upload-time = "2026-01-18T16:18:49.755Z" }, + { url = "https://files.pythonhosted.org/packages/c9/fa/3a5b8c86c958e83622b40865e11af0857c48ec763c11d472c87cd518283d/pyarrow-23.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:0800cc58a6d17d159df823f87ad66cefebf105b982493d4bad03ee7fab84b993", size = 35935712, upload-time = "2026-01-18T16:18:55.626Z" }, + { url = "https://files.pythonhosted.org/packages/c5/08/17a62078fc1a53decb34a9aa79cf9009efc74d63d2422e5ade9fed2f99e3/pyarrow-23.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:3a7c68c722da9bb5b0f8c10e3eae71d9825a4b429b40b32709df5d1fa55beb3d", size = 44503523, upload-time = "2026-01-18T16:19:03.958Z" }, + { url = "https://files.pythonhosted.org/packages/cc/70/84d45c74341e798aae0323d33b7c39194e23b1abc439ceaf60a68a7a969a/pyarrow-23.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:bd5556c24622df90551063ea41f559b714aa63ca953db884cfb958559087a14e", size = 47542490, upload-time = "2026-01-18T16:19:11.208Z" }, + { url = "https://files.pythonhosted.org/packages/61/d9/d1274b0e6f19e235de17441e53224f4716574b2ca837022d55702f24d71d/pyarrow-23.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:54810f6e6afc4ffee7c2e0051b61722fbea9a4961b46192dcfae8ea12fa09059", size = 48233605, upload-time = "2026-01-18T16:19:19.544Z" }, + { url = "https://files.pythonhosted.org/packages/39/07/e4e2d568cb57543d84482f61e510732820cddb0f47c4bb7df629abfed852/pyarrow-23.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:14de7d48052cf4b0ed174533eafa3cfe0711b8076ad70bede32cf59f744f0d7c", size = 50603979, upload-time = "2026-01-18T16:19:26.717Z" }, + { url = "https://files.pythonhosted.org/packages/72/9c/47693463894b610f8439b2e970b82ef81e9599c757bf2049365e40ff963c/pyarrow-23.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:427deac1f535830a744a4f04a6ac183a64fcac4341b3f618e693c41b7b98d2b0", size = 28338905, upload-time = "2026-01-18T16:19:32.93Z" }, +] + +[[package]] +name = "pydantic" +version = "2.12.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/69/44/36f1a6e523abc58ae5f928898e4aca2e0ea509b5aa6f6f392a5d882be928/pydantic-2.12.5.tar.gz", hash = "sha256:4d351024c75c0f085a9febbb665ce8c0c6ec5d30e903bdb6394b7ede26aebb49", size = 821591, upload-time = "2025-11-26T15:11:46.471Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl", hash = "sha256:e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d", size = 463580, upload-time = "2025-11-26T15:11:44.605Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.41.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/70/23b021c950c2addd24ec408e9ab05d59b035b39d97cdc1130e1bce647bb6/pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e", size = 460952, upload-time = "2025-11-04T13:43:49.098Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/06/8806241ff1f70d9939f9af039c6c35f2360cf16e93c2ca76f184e76b1564/pydantic_core-2.41.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9", size = 2120403, upload-time = "2025-11-04T13:40:25.248Z" }, + { url = "https://files.pythonhosted.org/packages/94/02/abfa0e0bda67faa65fef1c84971c7e45928e108fe24333c81f3bfe35d5f5/pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34", size = 1896206, upload-time = "2025-11-04T13:40:27.099Z" }, + { url = "https://files.pythonhosted.org/packages/15/df/a4c740c0943e93e6500f9eb23f4ca7ec9bf71b19e608ae5b579678c8d02f/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0", size = 1919307, upload-time = "2025-11-04T13:40:29.806Z" }, + { url = "https://files.pythonhosted.org/packages/9a/e3/6324802931ae1d123528988e0e86587c2072ac2e5394b4bc2bc34b61ff6e/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33", size = 2063258, upload-time = "2025-11-04T13:40:33.544Z" }, + { url = "https://files.pythonhosted.org/packages/c9/d4/2230d7151d4957dd79c3044ea26346c148c98fbf0ee6ebd41056f2d62ab5/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e", size = 2214917, upload-time = "2025-11-04T13:40:35.479Z" }, + { url = "https://files.pythonhosted.org/packages/e6/9f/eaac5df17a3672fef0081b6c1bb0b82b33ee89aa5cec0d7b05f52fd4a1fa/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2", size = 2332186, upload-time = "2025-11-04T13:40:37.436Z" }, + { url = "https://files.pythonhosted.org/packages/cf/4e/35a80cae583a37cf15604b44240e45c05e04e86f9cfd766623149297e971/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586", size = 2073164, upload-time = "2025-11-04T13:40:40.289Z" }, + { url = "https://files.pythonhosted.org/packages/bf/e3/f6e262673c6140dd3305d144d032f7bd5f7497d3871c1428521f19f9efa2/pydantic_core-2.41.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d", size = 2179146, upload-time = "2025-11-04T13:40:42.809Z" }, + { url = "https://files.pythonhosted.org/packages/75/c7/20bd7fc05f0c6ea2056a4565c6f36f8968c0924f19b7d97bbfea55780e73/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740", size = 2137788, upload-time = "2025-11-04T13:40:44.752Z" }, + { url = "https://files.pythonhosted.org/packages/3a/8d/34318ef985c45196e004bc46c6eab2eda437e744c124ef0dbe1ff2c9d06b/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e", size = 2340133, upload-time = "2025-11-04T13:40:46.66Z" }, + { url = "https://files.pythonhosted.org/packages/9c/59/013626bf8c78a5a5d9350d12e7697d3d4de951a75565496abd40ccd46bee/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858", size = 2324852, upload-time = "2025-11-04T13:40:48.575Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d9/c248c103856f807ef70c18a4f986693a46a8ffe1602e5d361485da502d20/pydantic_core-2.41.5-cp313-cp313-win32.whl", hash = "sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36", size = 1994679, upload-time = "2025-11-04T13:40:50.619Z" }, + { url = "https://files.pythonhosted.org/packages/9e/8b/341991b158ddab181cff136acd2552c9f35bd30380422a639c0671e99a91/pydantic_core-2.41.5-cp313-cp313-win_amd64.whl", hash = "sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11", size = 2019766, upload-time = "2025-11-04T13:40:52.631Z" }, + { url = "https://files.pythonhosted.org/packages/73/7d/f2f9db34af103bea3e09735bb40b021788a5e834c81eedb541991badf8f5/pydantic_core-2.41.5-cp313-cp313-win_arm64.whl", hash = "sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd", size = 1981005, upload-time = "2025-11-04T13:40:54.734Z" }, + { url = "https://files.pythonhosted.org/packages/ea/28/46b7c5c9635ae96ea0fbb779e271a38129df2550f763937659ee6c5dbc65/pydantic_core-2.41.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a", size = 2119622, upload-time = "2025-11-04T13:40:56.68Z" }, + { url = "https://files.pythonhosted.org/packages/74/1a/145646e5687e8d9a1e8d09acb278c8535ebe9e972e1f162ed338a622f193/pydantic_core-2.41.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14", size = 1891725, upload-time = "2025-11-04T13:40:58.807Z" }, + { url = "https://files.pythonhosted.org/packages/23/04/e89c29e267b8060b40dca97bfc64a19b2a3cf99018167ea1677d96368273/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1", size = 1915040, upload-time = "2025-11-04T13:41:00.853Z" }, + { url = "https://files.pythonhosted.org/packages/84/a3/15a82ac7bd97992a82257f777b3583d3e84bdb06ba6858f745daa2ec8a85/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66", size = 2063691, upload-time = "2025-11-04T13:41:03.504Z" }, + { url = "https://files.pythonhosted.org/packages/74/9b/0046701313c6ef08c0c1cf0e028c67c770a4e1275ca73131563c5f2a310a/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869", size = 2213897, upload-time = "2025-11-04T13:41:05.804Z" }, + { url = "https://files.pythonhosted.org/packages/8a/cd/6bac76ecd1b27e75a95ca3a9a559c643b3afcd2dd62086d4b7a32a18b169/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2", size = 2333302, upload-time = "2025-11-04T13:41:07.809Z" }, + { url = "https://files.pythonhosted.org/packages/4c/d2/ef2074dc020dd6e109611a8be4449b98cd25e1b9b8a303c2f0fca2f2bcf7/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375", size = 2064877, upload-time = "2025-11-04T13:41:09.827Z" }, + { url = "https://files.pythonhosted.org/packages/18/66/e9db17a9a763d72f03de903883c057b2592c09509ccfe468187f2a2eef29/pydantic_core-2.41.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553", size = 2180680, upload-time = "2025-11-04T13:41:12.379Z" }, + { url = "https://files.pythonhosted.org/packages/d3/9e/3ce66cebb929f3ced22be85d4c2399b8e85b622db77dad36b73c5387f8f8/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90", size = 2138960, upload-time = "2025-11-04T13:41:14.627Z" }, + { url = "https://files.pythonhosted.org/packages/a6/62/205a998f4327d2079326b01abee48e502ea739d174f0a89295c481a2272e/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07", size = 2339102, upload-time = "2025-11-04T13:41:16.868Z" }, + { url = "https://files.pythonhosted.org/packages/3c/0d/f05e79471e889d74d3d88f5bd20d0ed189ad94c2423d81ff8d0000aab4ff/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb", size = 2326039, upload-time = "2025-11-04T13:41:18.934Z" }, + { url = "https://files.pythonhosted.org/packages/ec/e1/e08a6208bb100da7e0c4b288eed624a703f4d129bde2da475721a80cab32/pydantic_core-2.41.5-cp314-cp314-win32.whl", hash = "sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23", size = 1995126, upload-time = "2025-11-04T13:41:21.418Z" }, + { url = "https://files.pythonhosted.org/packages/48/5d/56ba7b24e9557f99c9237e29f5c09913c81eeb2f3217e40e922353668092/pydantic_core-2.41.5-cp314-cp314-win_amd64.whl", hash = "sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf", size = 2015489, upload-time = "2025-11-04T13:41:24.076Z" }, + { url = "https://files.pythonhosted.org/packages/4e/bb/f7a190991ec9e3e0ba22e4993d8755bbc4a32925c0b5b42775c03e8148f9/pydantic_core-2.41.5-cp314-cp314-win_arm64.whl", hash = "sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0", size = 1977288, upload-time = "2025-11-04T13:41:26.33Z" }, + { url = "https://files.pythonhosted.org/packages/92/ed/77542d0c51538e32e15afe7899d79efce4b81eee631d99850edc2f5e9349/pydantic_core-2.41.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a", size = 2120255, upload-time = "2025-11-04T13:41:28.569Z" }, + { url = "https://files.pythonhosted.org/packages/bb/3d/6913dde84d5be21e284439676168b28d8bbba5600d838b9dca99de0fad71/pydantic_core-2.41.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3", size = 1863760, upload-time = "2025-11-04T13:41:31.055Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f0/e5e6b99d4191da102f2b0eb9687aaa7f5bea5d9964071a84effc3e40f997/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c", size = 1878092, upload-time = "2025-11-04T13:41:33.21Z" }, + { url = "https://files.pythonhosted.org/packages/71/48/36fb760642d568925953bcc8116455513d6e34c4beaa37544118c36aba6d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612", size = 2053385, upload-time = "2025-11-04T13:41:35.508Z" }, + { url = "https://files.pythonhosted.org/packages/20/25/92dc684dd8eb75a234bc1c764b4210cf2646479d54b47bf46061657292a8/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d", size = 2218832, upload-time = "2025-11-04T13:41:37.732Z" }, + { url = "https://files.pythonhosted.org/packages/e2/09/f53e0b05023d3e30357d82eb35835d0f6340ca344720a4599cd663dca599/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9", size = 2327585, upload-time = "2025-11-04T13:41:40Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4e/2ae1aa85d6af35a39b236b1b1641de73f5a6ac4d5a7509f77b814885760c/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660", size = 2041078, upload-time = "2025-11-04T13:41:42.323Z" }, + { url = "https://files.pythonhosted.org/packages/cd/13/2e215f17f0ef326fc72afe94776edb77525142c693767fc347ed6288728d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9", size = 2173914, upload-time = "2025-11-04T13:41:45.221Z" }, + { url = "https://files.pythonhosted.org/packages/02/7a/f999a6dcbcd0e5660bc348a3991c8915ce6599f4f2c6ac22f01d7a10816c/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3", size = 2129560, upload-time = "2025-11-04T13:41:47.474Z" }, + { url = "https://files.pythonhosted.org/packages/3a/b1/6c990ac65e3b4c079a4fb9f5b05f5b013afa0f4ed6780a3dd236d2cbdc64/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf", size = 2329244, upload-time = "2025-11-04T13:41:49.992Z" }, + { url = "https://files.pythonhosted.org/packages/d9/02/3c562f3a51afd4d88fff8dffb1771b30cfdfd79befd9883ee094f5b6c0d8/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470", size = 2331955, upload-time = "2025-11-04T13:41:54.079Z" }, + { url = "https://files.pythonhosted.org/packages/5c/96/5fb7d8c3c17bc8c62fdb031c47d77a1af698f1d7a406b0f79aaa1338f9ad/pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa", size = 1988906, upload-time = "2025-11-04T13:41:56.606Z" }, + { url = "https://files.pythonhosted.org/packages/22/ed/182129d83032702912c2e2d8bbe33c036f342cc735737064668585dac28f/pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c", size = 1981607, upload-time = "2025-11-04T13:41:58.889Z" }, + { url = "https://files.pythonhosted.org/packages/9f/ed/068e41660b832bb0b1aa5b58011dea2a3fe0ba7861ff38c4d4904c1c1a99/pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008", size = 1974769, upload-time = "2025-11-04T13:42:01.186Z" }, +] + +[[package]] +name = "pydantic-settings" +version = "2.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/4b/ac7e0aae12027748076d72a8764ff1c9d82ca75a7a52622e67ed3f765c54/pydantic_settings-2.12.0.tar.gz", hash = "sha256:005538ef951e3c2a68e1c08b292b5f2e71490def8589d4221b95dab00dafcfd0", size = 194184, upload-time = "2025-11-10T14:25:47.013Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/60/5d4751ba3f4a40a6891f24eec885f51afd78d208498268c734e256fb13c4/pydantic_settings-2.12.0-py3-none-any.whl", hash = "sha256:fddb9fd99a5b18da837b29710391e945b1e30c135477f484084ee513adb93809", size = 51880, upload-time = "2025-11-10T14:25:45.546Z" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, +] + +[[package]] +name = "pyiceberg" +version = "0.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cachetools" }, + { name = "click" }, + { name = "fsspec" }, + { name = "mmh3" }, + { name = "pydantic" }, + { name = "pyparsing" }, + { name = "pyroaring" }, + { name = "requests" }, + { name = "rich" }, + { name = "sortedcontainers" }, + { name = "strictyaml" }, + { name = "tenacity" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/0e/90e61c38504f4fbd5ed79631f85da7d5ea5e5bf997bdeaa65b28ebf04cab/pyiceberg-0.10.0.tar.gz", hash = "sha256:2525afa5e7e5fc4e72b291f8e1cc219e982d2bda5ff17e62cd05b8d91c4139f5", size = 842633, upload-time = "2025-09-11T14:59:34.044Z" } + +[package.optional-dependencies] +pyiceberg-core = [ + { name = "pyiceberg-core" }, +] + +[[package]] +name = "pyiceberg-core" +version = "0.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/60/80/06bd9159cacd80797122a88d65e8d3377fb76f00f1b23eeefe4ab0d85f4d/pyiceberg_core-0.6.0.tar.gz", hash = "sha256:ce2cac8cf8a85da6e682cec032165fcf387256257971f0f84bc6d50c0941f261", size = 457209, upload-time = "2025-07-30T09:20:23.447Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/8b/7ff908f6f18bc3d6351d9f4334d6a299eb7f1975b0bacb061d73b3292c1c/pyiceberg_core-0.6.0-cp39-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:2f228a54a2a69912378be18f98ea866bb4a08d265c875856f99cd81f2f7299ba", size = 55132736, upload-time = "2025-07-30T09:20:07.91Z" }, + { url = "https://files.pythonhosted.org/packages/57/09/ee720f4811fd4323a45d9d7bebcfd5d99283cf45092bccea87787a06bdff/pyiceberg_core-0.6.0-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:edb41a1f182774085b11352a1f44955d561e21453f00973021244471873fbbd7", size = 30041729, upload-time = "2025-07-30T09:20:13.056Z" }, + { url = "https://files.pythonhosted.org/packages/37/cd/94095aa2282ebe716e0a12130760b51076b1c921285574b1f88e5f63e234/pyiceberg_core-0.6.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5cf869d225d57254a54bc3778841cffea4193319bc0a849767a15e05e75c9b36", size = 30566511, upload-time = "2025-07-30T09:20:15.596Z" }, + { url = "https://files.pythonhosted.org/packages/87/62/7971cc8b090e51448da8d59e411be7b752a3a2abb1365e760871f27611e7/pyiceberg_core-0.6.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:18c12fe1ac5b4725b673cf0d1d0ab3e9475644ac0dae871a2e9a293c2622f0a8", size = 29570254, upload-time = "2025-07-30T09:20:18.371Z" }, + { url = "https://files.pythonhosted.org/packages/29/40/96bd273520075ee10718eeb609e92d44ee0b7701b5c225eae505a38fb22d/pyiceberg_core-0.6.0-cp39-abi3-win_amd64.whl", hash = "sha256:d3249eeae5e1d1f1d2c8bd8d6eced98da002afa7c48c751cb22d8dbd4b091a1e", size = 25815921, upload-time = "2025-07-30T09:20:20.781Z" }, +] + +[[package]] +name = "pyparsing" +version = "3.3.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/91/9c6ee907786a473bf81c5f53cf703ba0957b23ab84c264080fb5a450416f/pyparsing-3.3.2.tar.gz", hash = "sha256:c777f4d763f140633dcb6d8a3eda953bf7a214dc4eff598413c070bcdc117cbc", size = 6851574, upload-time = "2026-01-21T03:57:59.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781, upload-time = "2026-01-21T03:57:55.912Z" }, +] + +[[package]] +name = "pyroaring" +version = "1.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/e4/975f0fa77fc3590820b4a3ac49704644b389795409bc12eb91729f845812/pyroaring-1.0.3.tar.gz", hash = "sha256:cd7392d1c010c9e41c11c62cd0610c8852e7e9698b1f7f6c2fcdefe50e7ef6da", size = 188688, upload-time = "2025-10-09T09:08:22.448Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1f/95/97142ee32587ddda9e2cd614b865eeb5c0ee91006a51928f4074cd6e8e5f/pyroaring-1.0.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:20bc947054b197d1baa76cd05d70b8e04f95b82e698266e2f8f2f4b36d764477", size = 678813, upload-time = "2025-10-09T09:07:29.936Z" }, + { url = "https://files.pythonhosted.org/packages/70/5e/cff22be3a76a80024bdf00a9decdffedc6e80f037328a58b58c1b521442d/pyroaring-1.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ba5909b4c66bb85cab345e2f3a87e5ce671509c94b8c9823d8db64e107cbe854", size = 373661, upload-time = "2025-10-09T09:07:30.983Z" }, + { url = "https://files.pythonhosted.org/packages/86/73/fc406a67cd49e1707d1c3d08214458959dd579eff88c28587b356dfa068b/pyroaring-1.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b744746ba5da27fad760067f12633f5d384db6a1e65648d00244ceacbbd87731", size = 313559, upload-time = "2025-10-09T09:07:32.099Z" }, + { url = "https://files.pythonhosted.org/packages/f9/64/c7fe510523445f27e2cb04de6ffd3137f9d72db438b62db2bfa3dafcf4fc/pyroaring-1.0.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5b16c2a2791a5a09c4b59c0e1069ac1c877d0df25cae3155579c7eac8844676e", size = 1875926, upload-time = "2025-10-09T09:07:33.701Z" }, + { url = "https://files.pythonhosted.org/packages/47/74/da9b8ad2ca9ce6af1377f2cffdad6582a51a5f5df4f26df5c41810c9de5b/pyroaring-1.0.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7f68dfcf8d01177267f4bc06c4960fe8e39577470d1b52c9af8b61a72ca8767", size = 2064377, upload-time = "2025-10-09T09:07:35.273Z" }, + { url = "https://files.pythonhosted.org/packages/99/e3/8a70c5a5f7821c63709e2769aeccda8ae87a192198374bc475cbee543a22/pyroaring-1.0.3-cp313-cp313-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:dba4e4700030182a981a3c887aa73887697145fc9ffb192f908aa59b718fbbdd", size = 1778320, upload-time = "2025-10-09T09:07:36.782Z" }, + { url = "https://files.pythonhosted.org/packages/04/4c/08159a07c3723a2775064887543766b6115b4975e7baaa4d51e5580701a4/pyroaring-1.0.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e26dd1dc1edba02288902914bdb559e53e346e9155defa43c31fcab831b55342", size = 1786569, upload-time = "2025-10-09T09:07:38.473Z" }, + { url = "https://files.pythonhosted.org/packages/e5/ff/55a18d0e7e0dc4cd9f43988b746e788234a8d660fa17367c5ed9fa799348/pyroaring-1.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6eb98d2cacfc6d51c6a69893f04075e07b3df761eac71ba162c43b9b4c4452ad", size = 2852766, upload-time = "2025-10-09T09:07:39.633Z" }, + { url = "https://files.pythonhosted.org/packages/24/3c/419e25c51843dd40975ae37d67dea4f2f256554b5bec32237f607ec8ef21/pyroaring-1.0.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:a967e9eddb9485cbdd95d6371e3dada67880844d836c0283d3b11efe9225d1b7", size = 2683904, upload-time = "2025-10-09T09:07:41.139Z" }, + { url = "https://files.pythonhosted.org/packages/75/64/8d91f1b85b42925af632fc2c1047bb314be622dce890a4181a0a8d6e498d/pyroaring-1.0.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b12ef7f992ba7be865f91c7c098fd8ac6c413563aaa14d5b1e2bcb8cb43a4614", size = 2973884, upload-time = "2025-10-09T09:07:42.34Z" }, + { url = "https://files.pythonhosted.org/packages/61/6d/c867625549df0dc9ad675424ecf989fa2f08f0571bd46dfc4f7218737dd2/pyroaring-1.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:82ca5be174b85c40be7b00bc6bf39b2931a1b4a465f3af17ec6b9c48e9aa6fe0", size = 3103671, upload-time = "2025-10-09T09:07:44.055Z" }, + { url = "https://files.pythonhosted.org/packages/59/b1/d47c5ec2b2580d0b94f42575be8f49907a0f4aa396fdc18660f3b5060d54/pyroaring-1.0.3-cp313-cp313-win32.whl", hash = "sha256:f758c681e63ffe74b20423695e71f0410920f41b075cee679ffb5bc2bf38440b", size = 205153, upload-time = "2025-10-09T09:07:45.496Z" }, + { url = "https://files.pythonhosted.org/packages/c4/92/3600486936eebab747ae1462d231d7f87d234da24a04e82e1915c00f4427/pyroaring-1.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:428c3bb384fe4c483feb5cf7aa3aef1621fb0a5c4f3d391da67b2c4a43f08a10", size = 260349, upload-time = "2025-10-09T09:07:46.524Z" }, + { url = "https://files.pythonhosted.org/packages/77/96/8dde074f1ad2a1c3d2091b22de80d1b3007824e649e06eeeebded83f4d48/pyroaring-1.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:9c0c856e8aa5606e8aed5f30201286e404fdc9093f81fefe82d2e79e67472bb2", size = 218775, upload-time = "2025-10-09T09:07:47.558Z" }, +] + +[[package]] +name = "pytest" +version = "9.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d1/db/7ef3487e0fb0049ddb5ce41d3a49c235bf9ad299b6a25d5780a89f19230f/pytest-9.0.2.tar.gz", hash = "sha256:75186651a92bd89611d1d9fc20f0b4345fd827c41ccd5c299a868a05d70edf11", size = 1568901, upload-time = "2025-12-06T21:30:51.014Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/ab/b3226f0bd7cdcf710fbede2b3548584366da3b19b5021e74f5bde2a8fa3f/pytest-9.0.2-py3-none-any.whl", hash = "sha256:711ffd45bf766d5264d487b917733b453d917afd2b0ad65223959f59089f875b", size = 374801, upload-time = "2025-12-06T21:30:49.154Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "python-dotenv" +version = "1.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f0/26/19cadc79a718c5edbec86fd4919a6b6d3f681039a2f6d66d14be94e75fb9/python_dotenv-1.2.1.tar.gz", hash = "sha256:42667e897e16ab0d66954af0e60a9caa94f0fd4ecf3aaf6d2d260eec1aa36ad6", size = 44221, upload-time = "2025-10-26T15:12:10.434Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/1b/a298b06749107c305e1fe0f814c6c74aea7b2f1e10989cb30f544a1b3253/python_dotenv-1.2.1-py3-none-any.whl", hash = "sha256:b81ee9561e9ca4004139c6cbba3a238c32b03e4894671e181b671e8cb8425d61", size = 21230, upload-time = "2025-10-26T15:12:09.109Z" }, +] + +[[package]] +name = "requests" +version = "2.32.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c9/74/b3ff8e6c8446842c3f5c837e9c3dfcfe2018ea6ecef224c710c85ef728f4/requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf", size = 134517, upload-time = "2025-08-18T20:46:02.573Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" }, +] + +[[package]] +name = "rich" +version = "14.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a1/84/4831f881aa6ff3c976f6d6809b58cdfa350593ffc0dc3c58f5f6586780fb/rich-14.3.1.tar.gz", hash = "sha256:b8c5f568a3a749f9290ec6bddedf835cec33696bfc1e48bcfecb276c7386e4b8", size = 230125, upload-time = "2026-01-24T21:40:44.847Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/2a/a1810c8627b9ec8c57ec5ec325d306701ae7be50235e8fd81266e002a3cc/rich-14.3.1-py3-none-any.whl", hash = "sha256:da750b1aebbff0b372557426fb3f35ba56de8ef954b3190315eb64076d6fb54e", size = 309952, upload-time = "2026-01-24T21:40:42.969Z" }, +] + +[[package]] +name = "rustac" +version = "0.9.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b1/dc/0c0618f576119fe1ac7b5b03a968a4a825411b0460f748039210e98c1dcd/rustac-0.9.3.tar.gz", hash = "sha256:427dc5325617d6c57f504318bc9f703763dc40df66d35a888d407d0ebc0f8c9f", size = 737820, upload-time = "2026-01-06T12:50:32.929Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/ef/d9698e162ffcbc47f172162d8f37a82f56c8afb7e3d48d10583952c6c29a/rustac-0.9.3-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:d3cc8bed80370b54450377a53ebc33980ae59ec4e5fc37aea01ff29dc9133400", size = 25960779, upload-time = "2026-01-06T12:50:19.666Z" }, + { url = "https://files.pythonhosted.org/packages/c2/62/6bebac90e854f008cf4b5788698d8c568fa4184fd77cb3b3f9f45cdac546/rustac-0.9.3-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:2f9cf5f1fe536bd88c1fe2ac26d9d22a55e5143ac76a8af6b1ee8534b99dd9b7", size = 24090602, upload-time = "2026-01-06T12:50:17.183Z" }, + { url = "https://files.pythonhosted.org/packages/0f/2f/07edacb1b1a82b08cdaf5691442076f5852e40979859b116eae8802acbc3/rustac-0.9.3-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e1f6f94eaf0fd93d5f5f0687b73b43a9ada0e176eb6b4c8d279829892ccf9109", size = 28102224, upload-time = "2026-01-06T12:50:05.228Z" }, + { url = "https://files.pythonhosted.org/packages/ec/46/555e3fefe7f5775885e0d3765f10e035d0e540cc4253a493f29fda9da14b/rustac-0.9.3-cp311-abi3-manylinux_2_28_armv7l.whl", hash = "sha256:be326379c3e2599e1e02e02ff2c56fc3b324a9afd30bed1f0abfb03b33f3e6d1", size = 26388084, upload-time = "2026-01-06T12:50:08.089Z" }, + { url = "https://files.pythonhosted.org/packages/82/1e/1c6e8ab14ef6066991232e9338ae42cb824376d22291e3d3c9741274eaca/rustac-0.9.3-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:288216ca0f96136afb8422e24728e19be535e6e34192b25763bfc374c98d72a9", size = 34572191, upload-time = "2026-01-06T12:50:10.682Z" }, + { url = "https://files.pythonhosted.org/packages/46/0f/e41b599cd4f81e62b14b06b5c45b68ea52fb644f9e99d7791df5bd55bfc7/rustac-0.9.3-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:df941bdbaedaf6bf0b51976c923642c9bab55ed5a8e25a56bc49b02a11887f28", size = 31255045, upload-time = "2026-01-06T12:50:13.232Z" }, + { url = "https://files.pythonhosted.org/packages/a3/08/479539289c0e0de5095a8fd42308aa0669c3413f7590c5907137fd1a1f48/rustac-0.9.3-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:8e8355fa226d109be2be7aed485034eb69851fb6b59a9ed47af9379356b75825", size = 34678348, upload-time = "2026-01-06T12:50:22.632Z" }, + { url = "https://files.pythonhosted.org/packages/14/19/53c5cbb4b7daa46abbb4f3db01f9fba619b73c20600eab9fc10ddcaed19b/rustac-0.9.3-cp311-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:1a02d268580e7c6595563a2c3b3c8f12ee15411071ebcbd09f61f0325b64d46e", size = 34008454, upload-time = "2026-01-06T12:50:25.328Z" }, + { url = "https://files.pythonhosted.org/packages/cc/28/6606216351812a1bd9043dd6284216b3065af0a0f3d7deb00333ad4561db/rustac-0.9.3-cp311-abi3-musllinux_1_2_i686.whl", hash = "sha256:4b162fc0051dd440deb1bdfa4a9d3ea6726d0b3355f11c630a7528780178679a", size = 39073017, upload-time = "2026-01-06T12:50:27.863Z" }, + { url = "https://files.pythonhosted.org/packages/56/7e/e3390934aa0a85fb7044fe8ca9c53868b55c0c2e996236e074b7c51ff429/rustac-0.9.3-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d3f3bb74f49acfbbce42be113dab300e98226b763974f7bbe94835bb90e7dcd6", size = 37084098, upload-time = "2026-01-06T12:50:30.696Z" }, +] + +[package.optional-dependencies] +arrow = [ + { name = "arro3-core" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] + +[[package]] +name = "sqlalchemy" +version = "2.0.46" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/aa/9ce0f3e7a9829ead5c8ce549392f33a12c4555a6c0609bb27d882e9c7ddf/sqlalchemy-2.0.46.tar.gz", hash = "sha256:cf36851ee7219c170bb0793dbc3da3e80c582e04a5437bc601bfe8c85c9216d7", size = 9865393, upload-time = "2026-01-21T18:03:45.119Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/4b/fa7838fe20bb752810feed60e45625a9a8b0102c0c09971e2d1d95362992/sqlalchemy-2.0.46-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:93a12da97cca70cea10d4b4fc602589c4511f96c1f8f6c11817620c021d21d00", size = 2150268, upload-time = "2026-01-21T19:05:56.621Z" }, + { url = "https://files.pythonhosted.org/packages/46/c1/b34dccd712e8ea846edf396e00973dda82d598cb93762e55e43e6835eba9/sqlalchemy-2.0.46-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:af865c18752d416798dae13f83f38927c52f085c52e2f32b8ab0fef46fdd02c2", size = 3276511, upload-time = "2026-01-21T18:46:49.022Z" }, + { url = "https://files.pythonhosted.org/packages/96/48/a04d9c94753e5d5d096c628c82a98c4793b9c08ca0e7155c3eb7d7db9f24/sqlalchemy-2.0.46-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8d679b5f318423eacb61f933a9a0f75535bfca7056daeadbf6bd5bcee6183aee", size = 3292881, upload-time = "2026-01-21T18:40:13.089Z" }, + { url = "https://files.pythonhosted.org/packages/be/f4/06eda6e91476f90a7d8058f74311cb65a2fb68d988171aced81707189131/sqlalchemy-2.0.46-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:64901e08c33462acc9ec3bad27fc7a5c2b6491665f2aa57564e57a4f5d7c52ad", size = 3224559, upload-time = "2026-01-21T18:46:50.974Z" }, + { url = "https://files.pythonhosted.org/packages/ab/a2/d2af04095412ca6345ac22b33b89fe8d6f32a481e613ffcb2377d931d8d0/sqlalchemy-2.0.46-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e8ac45e8f4eaac0f9f8043ea0e224158855c6a4329fd4ee37c45c61e3beb518e", size = 3262728, upload-time = "2026-01-21T18:40:14.883Z" }, + { url = "https://files.pythonhosted.org/packages/31/48/1980c7caa5978a3b8225b4d230e69a2a6538a3562b8b31cea679b6933c83/sqlalchemy-2.0.46-cp313-cp313-win32.whl", hash = "sha256:8d3b44b3d0ab2f1319d71d9863d76eeb46766f8cf9e921ac293511804d39813f", size = 2111295, upload-time = "2026-01-21T18:42:52.366Z" }, + { url = "https://files.pythonhosted.org/packages/2d/54/f8d65bbde3d877617c4720f3c9f60e99bb7266df0d5d78b6e25e7c149f35/sqlalchemy-2.0.46-cp313-cp313-win_amd64.whl", hash = "sha256:77f8071d8fbcbb2dd11b7fd40dedd04e8ebe2eb80497916efedba844298065ef", size = 2137076, upload-time = "2026-01-21T18:42:53.924Z" }, + { url = "https://files.pythonhosted.org/packages/56/ba/9be4f97c7eb2b9d5544f2624adfc2853e796ed51d2bb8aec90bc94b7137e/sqlalchemy-2.0.46-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a1e8cc6cc01da346dc92d9509a63033b9b1bda4fed7a7a7807ed385c7dccdc10", size = 3556533, upload-time = "2026-01-21T18:33:06.636Z" }, + { url = "https://files.pythonhosted.org/packages/20/a6/b1fc6634564dbb4415b7ed6419cdfeaadefd2c39cdab1e3aa07a5f2474c2/sqlalchemy-2.0.46-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:96c7cca1a4babaaf3bfff3e4e606e38578856917e52f0384635a95b226c87764", size = 3523208, upload-time = "2026-01-21T18:45:08.436Z" }, + { url = "https://files.pythonhosted.org/packages/a1/d8/41e0bdfc0f930ff236f86fccd12962d8fa03713f17ed57332d38af6a3782/sqlalchemy-2.0.46-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:b2a9f9aee38039cf4755891a1e50e1effcc42ea6ba053743f452c372c3152b1b", size = 3464292, upload-time = "2026-01-21T18:33:08.208Z" }, + { url = "https://files.pythonhosted.org/packages/f0/8b/9dcbec62d95bea85f5ecad9b8d65b78cc30fb0ffceeb3597961f3712549b/sqlalchemy-2.0.46-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:db23b1bf8cfe1f7fda19018e7207b20cdb5168f83c437ff7e95d19e39289c447", size = 3473497, upload-time = "2026-01-21T18:45:10.552Z" }, + { url = "https://files.pythonhosted.org/packages/e9/f8/5ecdfc73383ec496de038ed1614de9e740a82db9ad67e6e4514ebc0708a3/sqlalchemy-2.0.46-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:56bdd261bfd0895452006d5316cbf35739c53b9bb71a170a331fa0ea560b2ada", size = 2152079, upload-time = "2026-01-21T19:05:58.477Z" }, + { url = "https://files.pythonhosted.org/packages/e5/bf/eba3036be7663ce4d9c050bc3d63794dc29fbe01691f2bf5ccb64e048d20/sqlalchemy-2.0.46-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:33e462154edb9493f6c3ad2125931e273bbd0be8ae53f3ecd1c161ea9a1dd366", size = 3272216, upload-time = "2026-01-21T18:46:52.634Z" }, + { url = "https://files.pythonhosted.org/packages/05/45/1256fb597bb83b58a01ddb600c59fe6fdf0e5afe333f0456ed75c0f8d7bd/sqlalchemy-2.0.46-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9bcdce05f056622a632f1d44bb47dbdb677f58cad393612280406ce37530eb6d", size = 3277208, upload-time = "2026-01-21T18:40:16.38Z" }, + { url = "https://files.pythonhosted.org/packages/d9/a0/2053b39e4e63b5d7ceb3372cface0859a067c1ddbd575ea7e9985716f771/sqlalchemy-2.0.46-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8e84b09a9b0f19accedcbeff5c2caf36e0dd537341a33aad8d680336152dc34e", size = 3221994, upload-time = "2026-01-21T18:46:54.622Z" }, + { url = "https://files.pythonhosted.org/packages/1e/87/97713497d9502553c68f105a1cb62786ba1ee91dea3852ae4067ed956a50/sqlalchemy-2.0.46-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4f52f7291a92381e9b4de9050b0a65ce5d6a763333406861e33906b8aa4906bf", size = 3243990, upload-time = "2026-01-21T18:40:18.253Z" }, + { url = "https://files.pythonhosted.org/packages/a8/87/5d1b23548f420ff823c236f8bea36b1a997250fd2f892e44a3838ca424f4/sqlalchemy-2.0.46-cp314-cp314-win32.whl", hash = "sha256:70ed2830b169a9960193f4d4322d22be5c0925357d82cbf485b3369893350908", size = 2114215, upload-time = "2026-01-21T18:42:55.232Z" }, + { url = "https://files.pythonhosted.org/packages/3a/20/555f39cbcf0c10cf452988b6a93c2a12495035f68b3dbd1a408531049d31/sqlalchemy-2.0.46-cp314-cp314-win_amd64.whl", hash = "sha256:3c32e993bc57be6d177f7d5d31edb93f30726d798ad86ff9066d75d9bf2e0b6b", size = 2139867, upload-time = "2026-01-21T18:42:56.474Z" }, + { url = "https://files.pythonhosted.org/packages/3e/f0/f96c8057c982d9d8a7a68f45d69c674bc6f78cad401099692fe16521640a/sqlalchemy-2.0.46-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4dafb537740eef640c4d6a7c254611dca2df87eaf6d14d6a5fca9d1f4c3fc0fa", size = 3561202, upload-time = "2026-01-21T18:33:10.337Z" }, + { url = "https://files.pythonhosted.org/packages/d7/53/3b37dda0a5b137f21ef608d8dfc77b08477bab0fe2ac9d3e0a66eaeab6fc/sqlalchemy-2.0.46-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:42a1643dc5427b69aca967dae540a90b0fbf57eaf248f13a90ea5930e0966863", size = 3526296, upload-time = "2026-01-21T18:45:12.657Z" }, + { url = "https://files.pythonhosted.org/packages/33/75/f28622ba6dde79cd545055ea7bd4062dc934e0621f7b3be2891f8563f8de/sqlalchemy-2.0.46-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ff33c6e6ad006bbc0f34f5faf941cfc62c45841c64c0a058ac38c799f15b5ede", size = 3470008, upload-time = "2026-01-21T18:33:11.725Z" }, + { url = "https://files.pythonhosted.org/packages/a9/42/4afecbbc38d5e99b18acef446453c76eec6fbd03db0a457a12a056836e22/sqlalchemy-2.0.46-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:82ec52100ec1e6ec671563bbd02d7c7c8d0b9e71a0723c72f22ecf52d1755330", size = 3476137, upload-time = "2026-01-21T18:45:15.001Z" }, + { url = "https://files.pythonhosted.org/packages/fc/a1/9c4efa03300926601c19c18582531b45aededfb961ab3c3585f1e24f120b/sqlalchemy-2.0.46-py3-none-any.whl", hash = "sha256:f9c11766e7e7c0a2767dda5acb006a118640c9fc0a4104214b96269bfb78399e", size = 1937882, upload-time = "2026-01-21T18:22:10.456Z" }, +] + +[[package]] +name = "stac-pydantic" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "geojson-pydantic" }, + { name = "pydantic" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/30/54/95586c73f097df47340dffbe19ae3db4eda832af997e834c9396d4bdcc83/stac_pydantic-3.4.0.tar.gz", hash = "sha256:5e7a45d38df18c4148fe45469447288a5b2eb15b10737608da4fba3dccc50683", size = 22943, upload-time = "2025-07-17T11:17:28.83Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/c7/8ee49430a0a745559dab4205ad9f946a264e793061fe5d1456a4b7cd2f27/stac_pydantic-3.4.0-py3-none-any.whl", hash = "sha256:ce2e7b377db078abbb164f378e18d54b53cda2953e44b643cdfa3adc831ca1c8", size = 24851, upload-time = "2025-07-17T11:17:27.966Z" }, +] + +[[package]] +name = "strictyaml" +version = "1.7.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/08/efd28d49162ce89c2ad61a88bd80e11fb77bc9f6c145402589112d38f8af/strictyaml-1.7.3.tar.gz", hash = "sha256:22f854a5fcab42b5ddba8030a0e4be51ca89af0267961c8d6cfa86395586c407", size = 115206, upload-time = "2023-03-10T12:50:27.062Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/7c/a81ef5ef10978dd073a854e0fa93b5d8021d0594b639cc8f6453c3c78a1d/strictyaml-1.7.3-py3-none-any.whl", hash = "sha256:fb5c8a4edb43bebb765959e420f9b3978d7f1af88c80606c03fb420888f5d1c7", size = 123917, upload-time = "2023-03-10T12:50:17.242Z" }, +] + +[[package]] +name = "tenacity" +version = "9.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0a/d4/2b0cd0fe285e14b36db076e78c93766ff1d529d70408bd1d2a5a84f1d929/tenacity-9.1.2.tar.gz", hash = "sha256:1169d376c297e7de388d18b4481760d478b0e99a777cad3a9c86e556f4b697cb", size = 48036, upload-time = "2025-04-02T08:25:09.966Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/30/643397144bfbfec6f6ef821f36f33e57d35946c44a2352d3c9f0ae847619/tenacity-9.1.2-py3-none-any.whl", hash = "sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138", size = 28248, upload-time = "2025-04-02T08:25:07.678Z" }, +] + +[[package]] +name = "typeguard" +version = "2.13.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3a/38/c61bfcf62a7b572b5e9363a802ff92559cb427ee963048e1442e3aef7490/typeguard-2.13.3.tar.gz", hash = "sha256:00edaa8da3a133674796cf5ea87d9f4b4c367d77476e185e80251cc13dfbb8c4", size = 40604, upload-time = "2021-12-10T21:09:39.158Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/bb/d43e5c75054e53efce310e79d63df0ac3f25e34c926be5dffb7d283fb2a8/typeguard-2.13.3-py3-none-any.whl", hash = "sha256:5e3e3be01e887e7eafae5af63d1f36c849aaa94e3a0112097312aabfa16284f1", size = 17605, upload-time = "2021-12-10T21:09:37.844Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +] + +[[package]] +name = "urllib3" +version = "2.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", size = 435556, upload-time = "2026-01-07T16:24:43.925Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" }, +] From ce2c08efbc304df630e468a0492aefe522d59de9 Mon Sep 17 00:00:00 2001 From: hrodmn Date: Thu, 29 Jan 2026 11:41:35 -0600 Subject: [PATCH 02/23] feat: add ci --- .github/workflows/ci.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..491e977 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,30 @@ +name: Test + +on: + workflow_dispatch: + push: + branches: + - main + - develop + tags: + pull_request: + +jobs: + tests: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v5 + + - name: Install uv + uses: astral-sh/setup-uv@v6 + with: + version: "0.7.*" + enable-cache: true + + - name: Install dependencies + run: | + uv sync + + - name: Run tests + run: uv run pytest From 1adf77360350770c66e9da5d63d94856782130b5 Mon Sep 17 00:00:00 2001 From: hrodmn Date: Wed, 25 Feb 2026 06:42:05 -0600 Subject: [PATCH 03/23] feat: do not sanitize collection ids --- src/icestac/errors.py | 2 + src/icestac/item_table.py | 82 +++++++++++---------------------------- tests/test_item_table.py | 81 ++++++++------------------------------ 3 files changed, 41 insertions(+), 124 deletions(-) create mode 100644 src/icestac/errors.py diff --git a/src/icestac/errors.py b/src/icestac/errors.py new file mode 100644 index 0000000..80f58f7 --- /dev/null +++ b/src/icestac/errors.py @@ -0,0 +1,2 @@ +class InvalidCollectionIdError(Exception): + """Invalid collection id""" diff --git a/src/icestac/item_table.py b/src/icestac/item_table.py index b6e19ca..f27d9eb 100644 --- a/src/icestac/item_table.py +++ b/src/icestac/item_table.py @@ -1,9 +1,7 @@ -import hashlib -import re - import pyarrow from arro3.core import Schema as ArrowSchema from pyiceberg.catalog import Catalog +from pyiceberg.exceptions import NoSuchNamespaceError from pyiceberg.io.pyarrow import _pyarrow_to_schema_without_ids from pyiceberg.partitioning import PartitionField, PartitionSpec from pyiceberg.schema import Schema as IcebergSchema @@ -12,52 +10,10 @@ from pyiceberg.types import NestedField from icestac.constants import DEFAULT_NAMESPACE +from icestac.errors import InvalidCollectionIdError from icestac.schema import enforce_required_fields, validate_schema -def sanitize_collection_id(collection_id: str) -> str: - """ - Sanitize a STAC collection ID to a valid Iceberg table name. - - Creates a deterministic, unique table identifier by: - 1. Converting to lowercase - 2. Replacing non-alphanumeric characters with underscores - 3. Collapsing consecutive underscores - 4. Ensuring it starts with a letter or underscore - 5. Appending an 8-character hash suffix to guarantee uniqueness - - This prevents collisions where different collection IDs might otherwise - map to the same table name (e.g., "my.collection" vs "my_collection"). - - Args: - collection_id: STAC collection identifier - - Returns: - Sanitized table name that is valid for Iceberg and guaranteed unique - - Examples: - >>> sanitize_collection_id("sentinel-2-l2a") - 'sentinel_2_l2a_a1b2c3d4' - >>> sanitize_collection_id("my.collection") - 'my_collection_e5f6g7h8' - >>> sanitize_collection_id("my_collection") - 'my_collection_i9j0k1l2' - """ - # Convert to lowercase and replace non-alphanumeric chars with underscores - sanitized = re.sub(r"[^a-z0-9_]", "_", collection_id.lower()) - sanitized = re.sub(r"_+", "_", sanitized) - sanitized = sanitized.strip("_") - - if sanitized and sanitized[0].isdigit(): - sanitized = f"c_{sanitized}" - - # Generate a short hash of the original collection_id for uniqueness - hash_suffix = hashlib.sha256(collection_id.encode()).hexdigest()[:8] - - # Combine sanitized name with hash suffix - return f"{sanitized}_{hash_suffix}" - - def create_item_table( arrow_schema: ArrowSchema, collection_id: str, @@ -93,22 +49,30 @@ def create_item_table( field_dict["id"] = i fields.append(NestedField(**field_dict)) - table_id = f"{namespace}.{sanitize_collection_id(collection_id)}" + table_id = f"{namespace}.{collection_id}" iceberg_schema = IcebergSchema(*fields) catalog.create_namespace_if_not_exists(namespace) - return catalog.create_table_if_not_exists( - identifier=table_id, - schema=iceberg_schema, - partition_spec=PartitionSpec( - # TODO: make temporal partitioning configurable - PartitionField( - source_id=iceberg_schema.find_field("datetime").field_id, - field_id=1000, - transform=MonthTransform(), - name="datetime_month", + try: + return catalog.create_table_if_not_exists( + identifier=table_id, + schema=iceberg_schema, + partition_spec=PartitionSpec( + # TODO: make temporal partitioning configurable + PartitionField( + source_id=iceberg_schema.find_field("datetime").field_id, + field_id=1000, + transform=MonthTransform(), + name="datetime_month", + ) + ), + ) + except NoSuchNamespaceError as e: + if "." in collection_id: + raise InvalidCollectionIdError( + f"{collection_id} contains a '.' character which is not allowed" ) - ), - ) + else: + raise e diff --git a/tests/test_item_table.py b/tests/test_item_table.py index 6af1640..3e776e5 100644 --- a/tests/test_item_table.py +++ b/tests/test_item_table.py @@ -1,10 +1,12 @@ from typing import Any import pyarrow +import pytest from pyiceberg.catalog import Catalog from rustac import to_arrow -from icestac.item_table import create_item_table, sanitize_collection_id +from icestac.errors import InvalidCollectionIdError +from icestac.item_table import create_item_table from icestac.schema import enforce_required_fields, get_schema_from_item @@ -39,67 +41,16 @@ def test_create_item_table( assert result.column("id").to_pylist() == [item["id"] for item in sample_stac_items] -def test_sanitize_collection_id_basic(): - """Test basic sanitization of collection IDs.""" - result = sanitize_collection_id("sentinel-2-l2a") - # Should be lowercase with underscores and have 8-char hash suffix - assert result.startswith("sentinel_2_l2a_") - assert len(result.split("_")[-1]) == 8 - assert result.islower() or "_" in result - - -def test_sanitize_collection_id_dots(): - """Test that dots are replaced with underscores.""" - result = sanitize_collection_id("my.collection.id") - assert result.startswith("my_collection_id_") - assert ".." not in result - - -def test_sanitize_collection_id_uniqueness(): - """Test that different collection IDs produce different sanitized names.""" - # These would collide with simple character replacement - id1 = sanitize_collection_id("my.collection") - id2 = sanitize_collection_id("my_collection") - id3 = sanitize_collection_id("my-collection") - - # All should be different due to hash suffix - assert id1 != id2 - assert id2 != id3 - assert id1 != id3 - - -def test_sanitize_collection_id_deterministic(): - """Test that sanitization is deterministic.""" - collection_id = "test-collection-123" - result1 = sanitize_collection_id(collection_id) - result2 = sanitize_collection_id(collection_id) - - assert result1 == result2 - - -def test_sanitize_collection_id_special_chars(): - """Test handling of various special characters.""" - result = sanitize_collection_id("my@collection#with$special%chars!") - # Should only contain lowercase alphanumeric and underscores - assert all(c.islower() or c.isdigit() or c == "_" for c in result) - - -def test_sanitize_collection_id_consecutive_underscores(): - """Test that consecutive underscores are collapsed.""" - result = sanitize_collection_id("my___collection___id") - # Should not have triple underscores in the sanitized portion - base_name = "_".join(result.split("_")[:-1]) # exclude hash suffix - assert "___" not in base_name - - -def test_sanitize_collection_id_starts_with_digit(): - """Test handling of collection IDs that start with a digit.""" - result = sanitize_collection_id("3dep-lidar") - # Should be prepended with 'c_' to make it valid - assert result.startswith("c_3") - - -def test_sanitize_collection_id_uppercase(): - """Test that uppercase letters are converted to lowercase.""" - result = sanitize_collection_id("MyCollection-ID") - assert result == result.lower() +def test_create_item_table_bad_collection_id( + test_catalog: Catalog, + test_namespace: str, + sample_stac_items: list[dict[str, Any]], +) -> None: + arrow_schema = get_schema_from_item(sample_stac_items[0]) + with pytest.raises(InvalidCollectionIdError): + create_item_table( + arrow_schema=arrow_schema, + collection_id="bad.collection", + catalog=test_catalog, + namespace=test_namespace, + ) From 0c08ae75a8b2a1c14691c6fd656b3871b858f034 Mon Sep 17 00:00:00 2001 From: hrodmn Date: Thu, 26 Feb 2026 08:00:28 -0600 Subject: [PATCH 04/23] chore: add type checking and ruff checks --- .github/workflows/ci.yml | 7 +- .pre-commit-config.yaml | 17 +++ README.md | 44 ++++++- main.py | 4 + pyproject.toml | 10 +- src/icestac/config.py | 4 +- src/icestac/item_table.py | 64 ++++------ src/icestac/schema.py | 29 ++++- tests/test_config.py | 8 +- tests/test_item_table.py | 9 ++ tests/test_load.py | 26 ++++ uv.lock | 254 +++++++++++++++++++++++++++++++++++++- 12 files changed, 414 insertions(+), 62 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 491e977..ddeedf9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,14 +17,17 @@ jobs: - uses: actions/checkout@v5 - name: Install uv - uses: astral-sh/setup-uv@v6 + uses: astral-sh/setup-uv@v7 with: - version: "0.7.*" + version: "0.10.*" enable-cache: true - name: Install dependencies run: | uv sync + - name: Run pre-commit + run: uv run pre-commit run --all-files + - name: Run tests run: uv run pytest diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..d92ff2c --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,17 @@ +default_stages: [pre-commit, pre-push] +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.15.2 + hooks: + - id: ruff + args: ["--fix", "--show-fixes"] + - id: ruff-format + + - repo: local + hooks: + - id: ty + name: ty (type check) + entry: uv run ty check + language: system + types: [python] + pass_filenames: false diff --git a/README.md b/README.md index 91d126a..d9a6765 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,48 @@ ICESTAC_S3_PATH_STYLE_ACCESS=true docker compose up ``` -**Note:** [main.py](./main.py) currently uses an older API signature and needs to be updated to match the current `create_item_table` function signature. +**Querying with DuckDB:** + +After ingesting items (e.g. via `uv run python main.py`), you can query the Iceberg tables using DuckDB's `iceberg` extension. Tables live under the `icestac` namespace, named by the sanitized collection ID. + +First, configure the extensions and MinIO credentials: + +```sql +INSTALL iceberg; LOAD iceberg; +INSTALL httpfs; LOAD httpfs; +INSTALL spatial; LOAD spatial; + +CREATE OR REPLACE SECRET minio ( + TYPE S3, + KEY_ID 'admin', + SECRET 'password', + ENDPOINT 'localhost:9000', + USE_SSL false, + URL_STYLE 'path' +); +``` + +Query via the REST catalog: + +```sql +ATTACH 'http://localhost:8181' AS catalog ( + TYPE ICEBERG, + WAREHOUSE 's3://warehouse/' +); + +SELECT id, datetime, collection, geometry +FROM catalog.icestac.icesat2_boreal_v3_1_agb +LIMIT 10; +``` + +Or scan the table directly from its S3 path (no catalog required): + +```sql +SET unsafe_enable_version_guessing = true; +DESCRIBE SELECT * +FROM iceberg_scan('s3://warehouse/icestac/icesat2_boreal_v3_1_agb') +LIMIT 10; +``` ## Current Implementation Status @@ -213,4 +254,3 @@ infrastructure/ 5. **Batch size**: How many STAC items per SQS batch for optimal performance? 6. **Error handling**: Retry strategy for failed items? DLQ processing? 7. **S3 bucket structure**: How to organize Iceberg table data and metadata? ->>>>>>> a2b56ec (initial commit) diff --git a/main.py b/main.py index baaf765..5c87703 100644 --- a/main.py +++ b/main.py @@ -18,6 +18,10 @@ async def run(): collections="icesat2-boreal-v3.1-agb", max_items=5, ) + + for item in items: + item["collection"] = "icesat2_boreal_v3_1_agb" + schema = get_schema_from_item(items[0]) table = create_item_table( arrow_schema=schema, diff --git a/pyproject.toml b/pyproject.toml index 187f5f4..463109e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,12 +1,12 @@ [project] name = "icestac" -version = "0.1.0" -description = "Add your description here" +version = "0.0.1" +description = "Manage STAC metadata in Apache Iceberg" readme = "README.md" authors = [ - { name = "hrodmn", email = "henry.rodman@gmail.com" } + { name = "hrodmn", email = "henry@developmentseed.org" } ] -requires-python = ">=3.13" +requires-python = ">=3.11" dependencies = [ "pyarrow>=23.0.0", "pydantic-settings>=2.12.0", @@ -28,7 +28,9 @@ deploy = [ ] dev = [ "pytest>=9.0.2", + "ruff>=0.15.2", "sqlalchemy>=2.0.46", + "ty>=0.0.18", ] [tool.pytest.ini_options] diff --git a/src/icestac/config.py b/src/icestac/config.py index 45b92aa..1f661a7 100644 --- a/src/icestac/config.py +++ b/src/icestac/config.py @@ -157,9 +157,7 @@ def get_catalog_properties(self) -> dict[str, str]: properties["s3.path-style-access"] = str(self.s3_path_style_access).lower() # Include any extra fields from environment (for catalog-specific properties) - for key, value in ( - self.model_extra.items() if hasattr(self, "model_extra") else [] - ): + for key, value in self.model_extra.items() if self.model_extra else []: if value is not None: properties[key] = str(value) diff --git a/src/icestac/item_table.py b/src/icestac/item_table.py index f27d9eb..5adc36f 100644 --- a/src/icestac/item_table.py +++ b/src/icestac/item_table.py @@ -1,17 +1,19 @@ -import pyarrow from arro3.core import Schema as ArrowSchema from pyiceberg.catalog import Catalog -from pyiceberg.exceptions import NoSuchNamespaceError -from pyiceberg.io.pyarrow import _pyarrow_to_schema_without_ids from pyiceberg.partitioning import PartitionField, PartitionSpec -from pyiceberg.schema import Schema as IcebergSchema from pyiceberg.table import Table from pyiceberg.transforms import MonthTransform -from pyiceberg.types import NestedField from icestac.constants import DEFAULT_NAMESPACE from icestac.errors import InvalidCollectionIdError -from icestac.schema import enforce_required_fields, validate_schema +from icestac.schema import convert_schema, validate_schema + + +def validate_collection_id(collection_id: str) -> None: + """Ensure collection id is valid for icestac schema""" + + if "." in collection_id: + raise InvalidCollectionIdError def create_item_table( @@ -36,43 +38,25 @@ def create_item_table( PyIceberg Table instance """ + validate_collection_id(collection_id) validate_schema(arrow_schema) - # Ensure required STAC fields are marked as non-nullable - pa_schema = pyarrow.schema(enforce_required_fields(arrow_schema)) - _schema = _pyarrow_to_schema_without_ids(pa_schema) - - # assign iceberg field ids manually - fields = [] - for i, _field in enumerate(_schema.fields, start=1): - field_dict = _field.model_dump() - field_dict["id"] = i - fields.append(NestedField(**field_dict)) - - table_id = f"{namespace}.{collection_id}" + catalog.create_namespace_if_not_exists(namespace) - iceberg_schema = IcebergSchema(*fields) + # TODO: check if collection record is present in collections table - catalog.create_namespace_if_not_exists(namespace) + iceberg_schema = convert_schema(arrow_schema) - try: - return catalog.create_table_if_not_exists( - identifier=table_id, - schema=iceberg_schema, - partition_spec=PartitionSpec( - # TODO: make temporal partitioning configurable - PartitionField( - source_id=iceberg_schema.find_field("datetime").field_id, - field_id=1000, - transform=MonthTransform(), - name="datetime_month", - ) - ), - ) - except NoSuchNamespaceError as e: - if "." in collection_id: - raise InvalidCollectionIdError( - f"{collection_id} contains a '.' character which is not allowed" + return catalog.create_table( + identifier=f"{namespace}.{collection_id}", + schema=iceberg_schema, + partition_spec=PartitionSpec( + # TODO: make temporal partitioning configurable + PartitionField( + source_id=iceberg_schema.find_field("datetime").field_id, + field_id=1000, + transform=MonthTransform(), + name="datetime_month", ) - else: - raise e + ), + ) diff --git a/src/icestac/schema.py b/src/icestac/schema.py index 386ca99..eabc427 100644 --- a/src/icestac/schema.py +++ b/src/icestac/schema.py @@ -1,7 +1,10 @@ from typing import Any import pyarrow as pa -from arro3.core import Schema +from arro3.core import Schema as ArrowSchema +from pyiceberg.io.pyarrow import _pyarrow_to_schema_without_ids +from pyiceberg.schema import Schema as IcebergSchema +from pyiceberg.types import NestedField from rustac import to_arrow from stac_pydantic.item import Item @@ -10,7 +13,7 @@ class IcestacItem(Item): collection: str -def get_schema_from_item(item: dict[str, Any]) -> Schema: +def get_schema_from_item(item: dict[str, Any]) -> ArrowSchema: # validate stac item _ = IcestacItem(**item) @@ -38,7 +41,7 @@ def get_required_fields() -> set[str]: return required_fields -def enforce_required_fields(schema: Schema) -> Schema: +def enforce_required_fields(schema: ArrowSchema) -> ArrowSchema: """ Ensure required STAC fields are marked as non-nullable in the Arrow schema. @@ -66,10 +69,10 @@ def enforce_required_fields(schema: Schema) -> Schema: # Keep original nullable setting new_fields.append(field) - return Schema.from_arrow(pa.schema(new_fields)) + return ArrowSchema.from_arrow(pa.schema(new_fields)) -def validate_schema(schema: Schema) -> None: +def validate_schema(schema: ArrowSchema) -> None: """ Validate that an Arrow schema contains required STAC item fields. @@ -91,3 +94,19 @@ def validate_schema(schema: Schema) -> None: raise ValueError( f"Arrow schema is missing required STAC fields: {sorted(missing_fields)}" ) + + +def convert_schema(schema: ArrowSchema) -> IcebergSchema: + """Convert the arrow schema to an iceberg schema with field ids + + Necessary because built-in converter functions do not assign field ids. + """ + _schema = _pyarrow_to_schema_without_ids(pa.schema(enforce_required_fields(schema))) + + fields = [] + for i, _field in enumerate(_schema.fields, start=1): + field_dict = _field.model_dump() + field_dict["id"] = i + fields.append(NestedField(**field_dict)) + + return IcebergSchema(*fields) diff --git a/tests/test_config.py b/tests/test_config.py index 46fef78..919557d 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -27,7 +27,7 @@ def test_default_settings(self, monkeypatch, tmp_path): monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", str(warehouse_path)) # Disable .env file reading to avoid pollution from project .env file - settings = IcebergCatalogConfig(_env_file=None) + settings = IcebergCatalogConfig(_env_file=None) # ty: ignore[unknown-argument] # Verify defaults are applied assert settings.catalog_name == "default" @@ -96,7 +96,7 @@ def test_sql_catalog_requires_uri(self, monkeypatch): monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", "/tmp/warehouse") with pytest.raises(ValidationError) as exc_info: - IcebergCatalogConfig(_env_file=None) + IcebergCatalogConfig(_env_file=None) # ty: ignore[unknown-argument] assert "catalog_uri is required" in str(exc_info.value) @@ -106,7 +106,7 @@ def test_sql_catalog_requires_warehouse_path(self, monkeypatch): monkeypatch.setenv("ICESTAC_CATALOG_URI", "sqlite:///catalog.db") with pytest.raises(ValidationError) as exc_info: - IcebergCatalogConfig(_env_file=None) + IcebergCatalogConfig(_env_file=None) # ty: ignore[unknown-argument] assert "warehouse_path is required" in str(exc_info.value) @@ -115,7 +115,7 @@ def test_rest_catalog_requires_uri(self, monkeypatch): monkeypatch.setenv("ICESTAC_CATALOG_TYPE", "rest") with pytest.raises(ValidationError) as exc_info: - IcebergCatalogConfig(_env_file=None) + IcebergCatalogConfig(_env_file=None) # ty: ignore[unknown-argument] assert "catalog_uri is required" in str(exc_info.value) diff --git a/tests/test_item_table.py b/tests/test_item_table.py index 3e776e5..df0b31e 100644 --- a/tests/test_item_table.py +++ b/tests/test_item_table.py @@ -3,6 +3,7 @@ import pyarrow import pytest from pyiceberg.catalog import Catalog +from pyiceberg.exceptions import TableAlreadyExistsError from rustac import to_arrow from icestac.errors import InvalidCollectionIdError @@ -40,6 +41,14 @@ def test_create_item_table( assert len(result) == len(sample_stac_items) assert result.column("id").to_pylist() == [item["id"] for item in sample_stac_items] + with pytest.raises(TableAlreadyExistsError): + create_item_table( + arrow_schema=arrow_schema, + collection_id=sample_stac_items[0]["collection"], + catalog=test_catalog, + namespace=test_namespace, + ) + def test_create_item_table_bad_collection_id( test_catalog: Catalog, diff --git a/tests/test_load.py b/tests/test_load.py index 29f4fc0..f8e6a42 100644 --- a/tests/test_load.py +++ b/tests/test_load.py @@ -1,5 +1,6 @@ from typing import Any +import pytest from pyiceberg.catalog import Catalog from icestac.item_table import create_item_table @@ -191,3 +192,28 @@ def test_load_items_single_item( result = table.scan().to_arrow() assert len(result) == 1 assert result.column("id").to_pylist()[0] == sample_stac_item["id"] + + +def test_load_items_different_schema( + test_catalog: Catalog, + test_namespace: str, + sample_stac_item: dict[str, Any], +) -> None: + + # Create the table + arrow_schema = get_schema_from_item(sample_stac_item) + table = create_item_table( + arrow_schema=arrow_schema, + collection_id=sample_stac_item["collection"], + catalog=test_catalog, + namespace=test_namespace, + ) + + # load an item + load_items([sample_stac_item], table) + + # change the schema + item_new_schema = sample_stac_item.copy() + item_new_schema["properties"]["new_field"] = True + with pytest.raises(ValueError, match="Update the schema first"): + load_items([item_new_schema], table) diff --git a/uv.lock b/uv.lock index 675af43..d914d9d 100644 --- a/uv.lock +++ b/uv.lock @@ -1,6 +1,6 @@ version = 1 revision = 2 -requires-python = ">=3.13" +requires-python = ">=3.11" [[package]] name = "annotated-types" @@ -15,6 +15,9 @@ wheels = [ name = "arro3-core" version = "0.6.5" source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.12'" }, +] sdist = { url = "https://files.pythonhosted.org/packages/2a/01/f06342d2eb822153f63d188153e41fbeabb29b48247f7a11ce76c538f7d1/arro3_core-0.6.5.tar.gz", hash = "sha256:768078887cd7ac82de4736f94bbd91f6d660f10779848bd5b019f511badd9d75", size = 107522, upload-time = "2025-10-13T23:12:38.872Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/78/85/20e46d3ed59d2f93be4a4d1abea4f6bef3e96acd59bf5a50726f84303c51/arro3_core-0.6.5-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:9d5999506daec1ab31096b3deb1e3573041d6ecadb4ca99c96f7ab26720c592c", size = 2685615, upload-time = "2025-10-13T23:09:41.793Z" }, @@ -151,6 +154,38 @@ version = "3.4.4" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/13/69/33ddede1939fdd074bce5434295f38fae7136463422fe4fd3e0e89b98062/charset_normalizer-3.4.4.tar.gz", hash = "sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a", size = 129418, upload-time = "2025-10-14T04:42:32.879Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/ed/27/c6491ff4954e58a10f69ad90aca8a1b6fe9c5d3c6f380907af3c37435b59/charset_normalizer-3.4.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6e1fcf0720908f200cd21aa4e6750a48ff6ce4afe7ff5a79a90d5ed8a08296f8", size = 206988, upload-time = "2025-10-14T04:40:33.79Z" }, + { url = "https://files.pythonhosted.org/packages/94/59/2e87300fe67ab820b5428580a53cad894272dbb97f38a7a814a2a1ac1011/charset_normalizer-3.4.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f819d5fe9234f9f82d75bdfa9aef3a3d72c4d24a6e57aeaebba32a704553aa0", size = 147324, upload-time = "2025-10-14T04:40:34.961Z" }, + { url = "https://files.pythonhosted.org/packages/07/fb/0cf61dc84b2b088391830f6274cb57c82e4da8bbc2efeac8c025edb88772/charset_normalizer-3.4.4-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a59cb51917aa591b1c4e6a43c132f0cdc3c76dbad6155df4e28ee626cc77a0a3", size = 142742, upload-time = "2025-10-14T04:40:36.105Z" }, + { url = "https://files.pythonhosted.org/packages/62/8b/171935adf2312cd745d290ed93cf16cf0dfe320863ab7cbeeae1dcd6535f/charset_normalizer-3.4.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8ef3c867360f88ac904fd3f5e1f902f13307af9052646963ee08ff4f131adafc", size = 160863, upload-time = "2025-10-14T04:40:37.188Z" }, + { url = "https://files.pythonhosted.org/packages/09/73/ad875b192bda14f2173bfc1bc9a55e009808484a4b256748d931b6948442/charset_normalizer-3.4.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d9e45d7faa48ee908174d8fe84854479ef838fc6a705c9315372eacbc2f02897", size = 157837, upload-time = "2025-10-14T04:40:38.435Z" }, + { url = "https://files.pythonhosted.org/packages/6d/fc/de9cce525b2c5b94b47c70a4b4fb19f871b24995c728e957ee68ab1671ea/charset_normalizer-3.4.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:840c25fb618a231545cbab0564a799f101b63b9901f2569faecd6b222ac72381", size = 151550, upload-time = "2025-10-14T04:40:40.053Z" }, + { url = "https://files.pythonhosted.org/packages/55/c2/43edd615fdfba8c6f2dfbd459b25a6b3b551f24ea21981e23fb768503ce1/charset_normalizer-3.4.4-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ca5862d5b3928c4940729dacc329aa9102900382fea192fc5e52eb69d6093815", size = 149162, upload-time = "2025-10-14T04:40:41.163Z" }, + { url = "https://files.pythonhosted.org/packages/03/86/bde4ad8b4d0e9429a4e82c1e8f5c659993a9a863ad62c7df05cf7b678d75/charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d9c7f57c3d666a53421049053eaacdd14bbd0a528e2186fcb2e672effd053bb0", size = 150019, upload-time = "2025-10-14T04:40:42.276Z" }, + { url = "https://files.pythonhosted.org/packages/1f/86/a151eb2af293a7e7bac3a739b81072585ce36ccfb4493039f49f1d3cae8c/charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:277e970e750505ed74c832b4bf75dac7476262ee2a013f5574dd49075879e161", size = 143310, upload-time = "2025-10-14T04:40:43.439Z" }, + { url = "https://files.pythonhosted.org/packages/b5/fe/43dae6144a7e07b87478fdfc4dbe9efd5defb0e7ec29f5f58a55aeef7bf7/charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:31fd66405eaf47bb62e8cd575dc621c56c668f27d46a61d975a249930dd5e2a4", size = 162022, upload-time = "2025-10-14T04:40:44.547Z" }, + { url = "https://files.pythonhosted.org/packages/80/e6/7aab83774f5d2bca81f42ac58d04caf44f0cc2b65fc6db2b3b2e8a05f3b3/charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:0d3d8f15c07f86e9ff82319b3d9ef6f4bf907608f53fe9d92b28ea9ae3d1fd89", size = 149383, upload-time = "2025-10-14T04:40:46.018Z" }, + { url = "https://files.pythonhosted.org/packages/4f/e8/b289173b4edae05c0dde07f69f8db476a0b511eac556dfe0d6bda3c43384/charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:9f7fcd74d410a36883701fafa2482a6af2ff5ba96b9a620e9e0721e28ead5569", size = 159098, upload-time = "2025-10-14T04:40:47.081Z" }, + { url = "https://files.pythonhosted.org/packages/d8/df/fe699727754cae3f8478493c7f45f777b17c3ef0600e28abfec8619eb49c/charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ebf3e58c7ec8a8bed6d66a75d7fb37b55e5015b03ceae72a8e7c74495551e224", size = 152991, upload-time = "2025-10-14T04:40:48.246Z" }, + { url = "https://files.pythonhosted.org/packages/1a/86/584869fe4ddb6ffa3bd9f491b87a01568797fb9bd8933f557dba9771beaf/charset_normalizer-3.4.4-cp311-cp311-win32.whl", hash = "sha256:eecbc200c7fd5ddb9a7f16c7decb07b566c29fa2161a16cf67b8d068bd21690a", size = 99456, upload-time = "2025-10-14T04:40:49.376Z" }, + { url = "https://files.pythonhosted.org/packages/65/f6/62fdd5feb60530f50f7e38b4f6a1d5203f4d16ff4f9f0952962c044e919a/charset_normalizer-3.4.4-cp311-cp311-win_amd64.whl", hash = "sha256:5ae497466c7901d54b639cf42d5b8c1b6a4fead55215500d2f486d34db48d016", size = 106978, upload-time = "2025-10-14T04:40:50.844Z" }, + { url = "https://files.pythonhosted.org/packages/7a/9d/0710916e6c82948b3be62d9d398cb4fcf4e97b56d6a6aeccd66c4b2f2bd5/charset_normalizer-3.4.4-cp311-cp311-win_arm64.whl", hash = "sha256:65e2befcd84bc6f37095f5961e68a6f077bf44946771354a28ad434c2cce0ae1", size = 99969, upload-time = "2025-10-14T04:40:52.272Z" }, + { url = "https://files.pythonhosted.org/packages/f3/85/1637cd4af66fa687396e757dec650f28025f2a2f5a5531a3208dc0ec43f2/charset_normalizer-3.4.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0a98e6759f854bd25a58a73fa88833fba3b7c491169f86ce1180c948ab3fd394", size = 208425, upload-time = "2025-10-14T04:40:53.353Z" }, + { url = "https://files.pythonhosted.org/packages/9d/6a/04130023fef2a0d9c62d0bae2649b69f7b7d8d24ea5536feef50551029df/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b5b290ccc2a263e8d185130284f8501e3e36c5e02750fc6b6bdeb2e9e96f1e25", size = 148162, upload-time = "2025-10-14T04:40:54.558Z" }, + { url = "https://files.pythonhosted.org/packages/78/29/62328d79aa60da22c9e0b9a66539feae06ca0f5a4171ac4f7dc285b83688/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74bb723680f9f7a6234dcf67aea57e708ec1fbdf5699fb91dfd6f511b0a320ef", size = 144558, upload-time = "2025-10-14T04:40:55.677Z" }, + { url = "https://files.pythonhosted.org/packages/86/bb/b32194a4bf15b88403537c2e120b817c61cd4ecffa9b6876e941c3ee38fe/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f1e34719c6ed0b92f418c7c780480b26b5d9c50349e9a9af7d76bf757530350d", size = 161497, upload-time = "2025-10-14T04:40:57.217Z" }, + { url = "https://files.pythonhosted.org/packages/19/89/a54c82b253d5b9b111dc74aca196ba5ccfcca8242d0fb64146d4d3183ff1/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2437418e20515acec67d86e12bf70056a33abdacb5cb1655042f6538d6b085a8", size = 159240, upload-time = "2025-10-14T04:40:58.358Z" }, + { url = "https://files.pythonhosted.org/packages/c0/10/d20b513afe03acc89ec33948320a5544d31f21b05368436d580dec4e234d/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86", size = 153471, upload-time = "2025-10-14T04:40:59.468Z" }, + { url = "https://files.pythonhosted.org/packages/61/fa/fbf177b55bdd727010f9c0a3c49eefa1d10f960e5f09d1d887bf93c2e698/charset_normalizer-3.4.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ac1c4a689edcc530fc9d9aa11f5774b9e2f33f9a0c6a57864e90908f5208d30a", size = 150864, upload-time = "2025-10-14T04:41:00.623Z" }, + { url = "https://files.pythonhosted.org/packages/05/12/9fbc6a4d39c0198adeebbde20b619790e9236557ca59fc40e0e3cebe6f40/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:21d142cc6c0ec30d2efee5068ca36c128a30b0f2c53c1c07bd78cb6bc1d3be5f", size = 150647, upload-time = "2025-10-14T04:41:01.754Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1f/6a9a593d52e3e8c5d2b167daf8c6b968808efb57ef4c210acb907c365bc4/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5dbe56a36425d26d6cfb40ce79c314a2e4dd6211d51d6d2191c00bed34f354cc", size = 145110, upload-time = "2025-10-14T04:41:03.231Z" }, + { url = "https://files.pythonhosted.org/packages/30/42/9a52c609e72471b0fc54386dc63c3781a387bb4fe61c20231a4ebcd58bdd/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5bfbb1b9acf3334612667b61bd3002196fe2a1eb4dd74d247e0f2a4d50ec9bbf", size = 162839, upload-time = "2025-10-14T04:41:04.715Z" }, + { url = "https://files.pythonhosted.org/packages/c4/5b/c0682bbf9f11597073052628ddd38344a3d673fda35a36773f7d19344b23/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d055ec1e26e441f6187acf818b73564e6e6282709e9bcb5b63f5b23068356a15", size = 150667, upload-time = "2025-10-14T04:41:05.827Z" }, + { url = "https://files.pythonhosted.org/packages/e4/24/a41afeab6f990cf2daf6cb8c67419b63b48cf518e4f56022230840c9bfb2/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:af2d8c67d8e573d6de5bc30cdb27e9b95e49115cd9baad5ddbd1a6207aaa82a9", size = 160535, upload-time = "2025-10-14T04:41:06.938Z" }, + { url = "https://files.pythonhosted.org/packages/2a/e5/6a4ce77ed243c4a50a1fecca6aaaab419628c818a49434be428fe24c9957/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:780236ac706e66881f3b7f2f32dfe90507a09e67d1d454c762cf642e6e1586e0", size = 154816, upload-time = "2025-10-14T04:41:08.101Z" }, + { url = "https://files.pythonhosted.org/packages/a8/ef/89297262b8092b312d29cdb2517cb1237e51db8ecef2e9af5edbe7b683b1/charset_normalizer-3.4.4-cp312-cp312-win32.whl", hash = "sha256:5833d2c39d8896e4e19b689ffc198f08ea58116bee26dea51e362ecc7cd3ed26", size = 99694, upload-time = "2025-10-14T04:41:09.23Z" }, + { url = "https://files.pythonhosted.org/packages/3d/2d/1e5ed9dd3b3803994c155cd9aacb60c82c331bad84daf75bcb9c91b3295e/charset_normalizer-3.4.4-cp312-cp312-win_amd64.whl", hash = "sha256:a79cfe37875f822425b89a82333404539ae63dbdddf97f84dcbc3d339aae9525", size = 107131, upload-time = "2025-10-14T04:41:10.467Z" }, + { url = "https://files.pythonhosted.org/packages/d0/d9/0ed4c7098a861482a7b6a95603edce4c0d9db2311af23da1fb2b75ec26fc/charset_normalizer-3.4.4-cp312-cp312-win_arm64.whl", hash = "sha256:376bec83a63b8021bb5c8ea75e21c4ccb86e7e45ca4eb81146091b56599b80c3", size = 100390, upload-time = "2025-10-14T04:41:11.915Z" }, { url = "https://files.pythonhosted.org/packages/97/45/4b3a1239bbacd321068ea6e7ac28875b03ab8bc0aa0966452db17cd36714/charset_normalizer-3.4.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794", size = 208091, upload-time = "2025-10-14T04:41:13.346Z" }, { url = "https://files.pythonhosted.org/packages/7d/62/73a6d7450829655a35bb88a88fca7d736f9882a27eacdca2c6d505b57e2e/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed", size = 147936, upload-time = "2025-10-14T04:41:14.461Z" }, { url = "https://files.pythonhosted.org/packages/89/c5/adb8c8b3d6625bef6d88b251bbb0d95f8205831b987631ab0c8bb5d937c2/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72", size = 144180, upload-time = "2025-10-14T04:41:15.588Z" }, @@ -248,6 +283,24 @@ version = "3.3.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/8a/99/1cd3411c56a410994669062bd73dd58270c00cc074cac15f385a1fd91f8a/greenlet-3.3.1.tar.gz", hash = "sha256:41848f3230b58c08bb43dee542e74a2a2e34d3c59dc3076cec9151aeeedcae98", size = 184690, upload-time = "2026-01-23T15:31:02.076Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/e8/2e1462c8fdbe0f210feb5ac7ad2d9029af8be3bf45bd9fa39765f821642f/greenlet-3.3.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:5fd23b9bc6d37b563211c6abbb1b3cab27db385a4449af5c32e932f93017080c", size = 274974, upload-time = "2026-01-23T15:31:02.891Z" }, + { url = "https://files.pythonhosted.org/packages/7e/a8/530a401419a6b302af59f67aaf0b9ba1015855ea7e56c036b5928793c5bd/greenlet-3.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09f51496a0bfbaa9d74d36a52d2580d1ef5ed4fdfcff0a73730abfbbbe1403dd", size = 577175, upload-time = "2026-01-23T16:00:56.213Z" }, + { url = "https://files.pythonhosted.org/packages/8e/89/7e812bb9c05e1aaef9b597ac1d0962b9021d2c6269354966451e885c4e6b/greenlet-3.3.1-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb0feb07fe6e6a74615ee62a880007d976cf739b6669cce95daa7373d4fc69c5", size = 590401, upload-time = "2026-01-23T16:05:26.365Z" }, + { url = "https://files.pythonhosted.org/packages/70/ae/e2d5f0e59b94a2269b68a629173263fa40b63da32f5c231307c349315871/greenlet-3.3.1-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:67ea3fc73c8cd92f42467a72b75e8f05ed51a0e9b1d15398c913416f2dafd49f", size = 601161, upload-time = "2026-01-23T16:15:53.456Z" }, + { url = "https://files.pythonhosted.org/packages/5c/ae/8d472e1f5ac5efe55c563f3eabb38c98a44b832602e12910750a7c025802/greenlet-3.3.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:39eda9ba259cc9801da05351eaa8576e9aa83eb9411e8f0c299e05d712a210f2", size = 590272, upload-time = "2026-01-23T15:32:49.411Z" }, + { url = "https://files.pythonhosted.org/packages/a8/51/0fde34bebfcadc833550717eade64e35ec8738e6b097d5d248274a01258b/greenlet-3.3.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e2e7e882f83149f0a71ac822ebf156d902e7a5d22c9045e3e0d1daf59cee2cc9", size = 1550729, upload-time = "2026-01-23T16:04:20.867Z" }, + { url = "https://files.pythonhosted.org/packages/16/c9/2fb47bee83b25b119d5a35d580807bb8b92480a54b68fef009a02945629f/greenlet-3.3.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:80aa4d79eb5564f2e0a6144fcc744b5a37c56c4a92d60920720e99210d88db0f", size = 1615552, upload-time = "2026-01-23T15:33:45.743Z" }, + { url = "https://files.pythonhosted.org/packages/1f/54/dcf9f737b96606f82f8dd05becfb8d238db0633dd7397d542a296fe9cad3/greenlet-3.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:32e4ca9777c5addcbf42ff3915d99030d8e00173a56f80001fb3875998fe410b", size = 226462, upload-time = "2026-01-23T15:36:50.422Z" }, + { url = "https://files.pythonhosted.org/packages/91/37/61e1015cf944ddd2337447d8e97fb423ac9bc21f9963fb5f206b53d65649/greenlet-3.3.1-cp311-cp311-win_arm64.whl", hash = "sha256:da19609432f353fed186cc1b85e9440db93d489f198b4bdf42ae19cc9d9ac9b4", size = 225715, upload-time = "2026-01-23T15:33:17.298Z" }, + { url = "https://files.pythonhosted.org/packages/f9/c8/9d76a66421d1ae24340dfae7e79c313957f6e3195c144d2c73333b5bfe34/greenlet-3.3.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:7e806ca53acf6d15a888405880766ec84721aa4181261cd11a457dfe9a7a4975", size = 276443, upload-time = "2026-01-23T15:30:10.066Z" }, + { url = "https://files.pythonhosted.org/packages/81/99/401ff34bb3c032d1f10477d199724f5e5f6fbfb59816ad1455c79c1eb8e7/greenlet-3.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d842c94b9155f1c9b3058036c24ffb8ff78b428414a19792b2380be9cecf4f36", size = 597359, upload-time = "2026-01-23T16:00:57.394Z" }, + { url = "https://files.pythonhosted.org/packages/2b/bc/4dcc0871ed557792d304f50be0f7487a14e017952ec689effe2180a6ff35/greenlet-3.3.1-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:20fedaadd422fa02695f82093f9a98bad3dab5fcda793c658b945fcde2ab27ba", size = 607805, upload-time = "2026-01-23T16:05:28.068Z" }, + { url = "https://files.pythonhosted.org/packages/3b/cd/7a7ca57588dac3389e97f7c9521cb6641fd8b6602faf1eaa4188384757df/greenlet-3.3.1-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c620051669fd04ac6b60ebc70478210119c56e2d5d5df848baec4312e260e4ca", size = 622363, upload-time = "2026-01-23T16:15:54.754Z" }, + { url = "https://files.pythonhosted.org/packages/cf/05/821587cf19e2ce1f2b24945d890b164401e5085f9d09cbd969b0c193cd20/greenlet-3.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:14194f5f4305800ff329cbf02c5fcc88f01886cadd29941b807668a45f0d2336", size = 609947, upload-time = "2026-01-23T15:32:51.004Z" }, + { url = "https://files.pythonhosted.org/packages/a4/52/ee8c46ed9f8babaa93a19e577f26e3d28a519feac6350ed6f25f1afee7e9/greenlet-3.3.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7b2fe4150a0cf59f847a67db8c155ac36aed89080a6a639e9f16df5d6c6096f1", size = 1567487, upload-time = "2026-01-23T16:04:22.125Z" }, + { url = "https://files.pythonhosted.org/packages/8f/7c/456a74f07029597626f3a6db71b273a3632aecb9afafeeca452cfa633197/greenlet-3.3.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:49f4ad195d45f4a66a0eb9c1ba4832bb380570d361912fa3554746830d332149", size = 1636087, upload-time = "2026-01-23T15:33:47.486Z" }, + { url = "https://files.pythonhosted.org/packages/34/2f/5e0e41f33c69655300a5e54aeb637cf8ff57f1786a3aba374eacc0228c1d/greenlet-3.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:cc98b9c4e4870fa983436afa999d4eb16b12872fab7071423d5262fa7120d57a", size = 227156, upload-time = "2026-01-23T15:34:34.808Z" }, + { url = "https://files.pythonhosted.org/packages/c8/ab/717c58343cf02c5265b531384b248787e04d8160b8afe53d9eec053d7b44/greenlet-3.3.1-cp312-cp312-win_arm64.whl", hash = "sha256:bfb2d1763d777de5ee495c85309460f6fd8146e50ec9d0ae0183dbf6f0a829d1", size = 226403, upload-time = "2026-01-23T15:31:39.372Z" }, { url = "https://files.pythonhosted.org/packages/ec/ab/d26750f2b7242c2b90ea2ad71de70cfcd73a948a49513188a0fc0d6fc15a/greenlet-3.3.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:7ab327905cabb0622adca5971e488064e35115430cec2c35a50fd36e72a315b3", size = 275205, upload-time = "2026-01-23T15:30:24.556Z" }, { url = "https://files.pythonhosted.org/packages/10/d3/be7d19e8fad7c5a78eeefb2d896a08cd4643e1e90c605c4be3b46264998f/greenlet-3.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:65be2f026ca6a176f88fb935ee23c18333ccea97048076aef4db1ef5bc0713ac", size = 599284, upload-time = "2026-01-23T16:00:58.584Z" }, { url = "https://files.pythonhosted.org/packages/ae/21/fe703aaa056fdb0f17e5afd4b5c80195bbdab701208918938bd15b00d39b/greenlet-3.3.1-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7a3ae05b3d225b4155bda56b072ceb09d05e974bc74be6c3fc15463cf69f33fd", size = 610274, upload-time = "2026-01-23T16:05:29.312Z" }, @@ -278,7 +331,7 @@ wheels = [ [[package]] name = "icestac" -version = "0.1.0" +version = "0.0.1" source = { editable = "." } dependencies = [ { name = "pyarrow" }, @@ -294,7 +347,9 @@ deploy = [ ] dev = [ { name = "pytest" }, + { name = "ruff" }, { name = "sqlalchemy" }, + { name = "ty" }, ] [package.metadata] @@ -310,7 +365,9 @@ requires-dist = [ deploy = [{ name = "aws-cdk-lib", specifier = ">=2.236.0" }] dev = [ { name = "pytest", specifier = ">=9.0.2" }, + { name = "ruff", specifier = ">=0.15.2" }, { name = "sqlalchemy", specifier = ">=2.0.46" }, + { name = "ty", specifier = ">=0.0.18" }, ] [[package]] @@ -385,6 +442,38 @@ version = "5.2.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/a7/af/f28c2c2f51f31abb4725f9a64bc7863d5f491f6539bd26aee2a1d21a649e/mmh3-5.2.0.tar.gz", hash = "sha256:1efc8fec8478e9243a78bb993422cf79f8ff85cb4cf6b79647480a31e0d950a8", size = 33582, upload-time = "2025-07-29T07:43:48.49Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/87/399567b3796e134352e11a8b973cd470c06b2ecfad5468fe580833be442b/mmh3-5.2.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7901c893e704ee3c65f92d39b951f8f34ccf8e8566768c58103fb10e55afb8c1", size = 56107, upload-time = "2025-07-29T07:41:57.07Z" }, + { url = "https://files.pythonhosted.org/packages/c3/09/830af30adf8678955b247d97d3d9543dd2fd95684f3cd41c0cd9d291da9f/mmh3-5.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4a5f5536b1cbfa72318ab3bfc8a8188b949260baed186b75f0abc75b95d8c051", size = 40635, upload-time = "2025-07-29T07:41:57.903Z" }, + { url = "https://files.pythonhosted.org/packages/07/14/eaba79eef55b40d653321765ac5e8f6c9ac38780b8a7c2a2f8df8ee0fb72/mmh3-5.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cedac4f4054b8f7859e5aed41aaa31ad03fce6851901a7fdc2af0275ac533c10", size = 40078, upload-time = "2025-07-29T07:41:58.772Z" }, + { url = "https://files.pythonhosted.org/packages/bb/26/83a0f852e763f81b2265d446b13ed6d49ee49e1fc0c47b9655977e6f3d81/mmh3-5.2.0-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:eb756caf8975882630ce4e9fbbeb9d3401242a72528230422c9ab3a0d278e60c", size = 97262, upload-time = "2025-07-29T07:41:59.678Z" }, + { url = "https://files.pythonhosted.org/packages/00/7d/b7133b10d12239aeaebf6878d7eaf0bf7d3738c44b4aba3c564588f6d802/mmh3-5.2.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:097e13c8b8a66c5753c6968b7640faefe85d8e38992703c1f666eda6ef4c3762", size = 103118, upload-time = "2025-07-29T07:42:01.197Z" }, + { url = "https://files.pythonhosted.org/packages/7b/3e/62f0b5dce2e22fd5b7d092aba285abd7959ea2b17148641e029f2eab1ffa/mmh3-5.2.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7c0c7845566b9686480e6a7e9044db4afb60038d5fabd19227443f0104eeee4", size = 106072, upload-time = "2025-07-29T07:42:02.601Z" }, + { url = "https://files.pythonhosted.org/packages/66/84/ea88bb816edfe65052c757a1c3408d65c4201ddbd769d4a287b0f1a628b2/mmh3-5.2.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:61ac226af521a572700f863d6ecddc6ece97220ce7174e311948ff8c8919a363", size = 112925, upload-time = "2025-07-29T07:42:03.632Z" }, + { url = "https://files.pythonhosted.org/packages/2e/13/c9b1c022807db575fe4db806f442d5b5784547e2e82cff36133e58ea31c7/mmh3-5.2.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:582f9dbeefe15c32a5fa528b79b088b599a1dfe290a4436351c6090f90ddebb8", size = 120583, upload-time = "2025-07-29T07:42:04.991Z" }, + { url = "https://files.pythonhosted.org/packages/8a/5f/0e2dfe1a38f6a78788b7eb2b23432cee24623aeabbc907fed07fc17d6935/mmh3-5.2.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2ebfc46b39168ab1cd44670a32ea5489bcbc74a25795c61b6d888c5c2cf654ed", size = 99127, upload-time = "2025-07-29T07:42:05.929Z" }, + { url = "https://files.pythonhosted.org/packages/77/27/aefb7d663b67e6a0c4d61a513c83e39ba2237e8e4557fa7122a742a23de5/mmh3-5.2.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1556e31e4bd0ac0c17eaf220be17a09c171d7396919c3794274cb3415a9d3646", size = 98544, upload-time = "2025-07-29T07:42:06.87Z" }, + { url = "https://files.pythonhosted.org/packages/ab/97/a21cc9b1a7c6e92205a1b5fa030cdf62277d177570c06a239eca7bd6dd32/mmh3-5.2.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:81df0dae22cd0da87f1c978602750f33d17fb3d21fb0f326c89dc89834fea79b", size = 106262, upload-time = "2025-07-29T07:42:07.804Z" }, + { url = "https://files.pythonhosted.org/packages/43/18/db19ae82ea63c8922a880e1498a75342311f8aa0c581c4dd07711473b5f7/mmh3-5.2.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:eba01ec3bd4a49b9ac5ca2bc6a73ff5f3af53374b8556fcc2966dd2af9eb7779", size = 109824, upload-time = "2025-07-29T07:42:08.735Z" }, + { url = "https://files.pythonhosted.org/packages/9f/f5/41dcf0d1969125fc6f61d8618b107c79130b5af50b18a4651210ea52ab40/mmh3-5.2.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e9a011469b47b752e7d20de296bb34591cdfcbe76c99c2e863ceaa2aa61113d2", size = 97255, upload-time = "2025-07-29T07:42:09.706Z" }, + { url = "https://files.pythonhosted.org/packages/32/b3/cce9eaa0efac1f0e735bb178ef9d1d2887b4927fe0ec16609d5acd492dda/mmh3-5.2.0-cp311-cp311-win32.whl", hash = "sha256:bc44fc2b886243d7c0d8daeb37864e16f232e5b56aaec27cc781d848264cfd28", size = 40779, upload-time = "2025-07-29T07:42:10.546Z" }, + { url = "https://files.pythonhosted.org/packages/7c/e9/3fa0290122e6d5a7041b50ae500b8a9f4932478a51e48f209a3879fe0b9b/mmh3-5.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:8ebf241072cf2777a492d0e09252f8cc2b3edd07dfdb9404b9757bffeb4f2cee", size = 41549, upload-time = "2025-07-29T07:42:11.399Z" }, + { url = "https://files.pythonhosted.org/packages/3a/54/c277475b4102588e6f06b2e9095ee758dfe31a149312cdbf62d39a9f5c30/mmh3-5.2.0-cp311-cp311-win_arm64.whl", hash = "sha256:b5f317a727bba0e633a12e71228bc6a4acb4f471a98b1c003163b917311ea9a9", size = 39336, upload-time = "2025-07-29T07:42:12.209Z" }, + { url = "https://files.pythonhosted.org/packages/bf/6a/d5aa7edb5c08e0bd24286c7d08341a0446f9a2fbbb97d96a8a6dd81935ee/mmh3-5.2.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:384eda9361a7bf83a85e09447e1feafe081034af9dd428893701b959230d84be", size = 56141, upload-time = "2025-07-29T07:42:13.456Z" }, + { url = "https://files.pythonhosted.org/packages/08/49/131d0fae6447bc4a7299ebdb1a6fb9d08c9f8dcf97d75ea93e8152ddf7ab/mmh3-5.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c9da0d568569cc87315cb063486d761e38458b8ad513fedd3dc9263e1b81bcd", size = 40681, upload-time = "2025-07-29T07:42:14.306Z" }, + { url = "https://files.pythonhosted.org/packages/8f/6f/9221445a6bcc962b7f5ff3ba18ad55bba624bacdc7aa3fc0a518db7da8ec/mmh3-5.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:86d1be5d63232e6eb93c50881aea55ff06eb86d8e08f9b5417c8c9b10db9db96", size = 40062, upload-time = "2025-07-29T07:42:15.08Z" }, + { url = "https://files.pythonhosted.org/packages/1e/d4/6bb2d0fef81401e0bb4c297d1eb568b767de4ce6fc00890bc14d7b51ecc4/mmh3-5.2.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bf7bee43e17e81671c447e9c83499f53d99bf440bc6d9dc26a841e21acfbe094", size = 97333, upload-time = "2025-07-29T07:42:16.436Z" }, + { url = "https://files.pythonhosted.org/packages/44/e0/ccf0daff8134efbb4fbc10a945ab53302e358c4b016ada9bf97a6bdd50c1/mmh3-5.2.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7aa18cdb58983ee660c9c400b46272e14fa253c675ed963d3812487f8ca42037", size = 103310, upload-time = "2025-07-29T07:42:17.796Z" }, + { url = "https://files.pythonhosted.org/packages/02/63/1965cb08a46533faca0e420e06aff8bbaf9690a6f0ac6ae6e5b2e4544687/mmh3-5.2.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ae9d032488fcec32d22be6542d1a836f00247f40f320844dbb361393b5b22773", size = 106178, upload-time = "2025-07-29T07:42:19.281Z" }, + { url = "https://files.pythonhosted.org/packages/c2/41/c883ad8e2c234013f27f92061200afc11554ea55edd1bcf5e1accd803a85/mmh3-5.2.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1861fb6b1d0453ed7293200139c0a9011eeb1376632e048e3766945b13313c5", size = 113035, upload-time = "2025-07-29T07:42:20.356Z" }, + { url = "https://files.pythonhosted.org/packages/df/b5/1ccade8b1fa625d634a18bab7bf08a87457e09d5ec8cf83ca07cbea9d400/mmh3-5.2.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:99bb6a4d809aa4e528ddfe2c85dd5239b78b9dd14be62cca0329db78505e7b50", size = 120784, upload-time = "2025-07-29T07:42:21.377Z" }, + { url = "https://files.pythonhosted.org/packages/77/1c/919d9171fcbdcdab242e06394464ccf546f7d0f3b31e0d1e3a630398782e/mmh3-5.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1f8d8b627799f4e2fcc7c034fed8f5f24dc7724ff52f69838a3d6d15f1ad4765", size = 99137, upload-time = "2025-07-29T07:42:22.344Z" }, + { url = "https://files.pythonhosted.org/packages/66/8a/1eebef5bd6633d36281d9fc83cf2e9ba1ba0e1a77dff92aacab83001cee4/mmh3-5.2.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b5995088dd7023d2d9f310a0c67de5a2b2e06a570ecfd00f9ff4ab94a67cde43", size = 98664, upload-time = "2025-07-29T07:42:23.269Z" }, + { url = "https://files.pythonhosted.org/packages/13/41/a5d981563e2ee682b21fb65e29cc0f517a6734a02b581359edd67f9d0360/mmh3-5.2.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1a5f4d2e59d6bba8ef01b013c472741835ad961e7c28f50c82b27c57748744a4", size = 106459, upload-time = "2025-07-29T07:42:24.238Z" }, + { url = "https://files.pythonhosted.org/packages/24/31/342494cd6ab792d81e083680875a2c50fa0c5df475ebf0b67784f13e4647/mmh3-5.2.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:fd6e6c3d90660d085f7e73710eab6f5545d4854b81b0135a3526e797009dbda3", size = 110038, upload-time = "2025-07-29T07:42:25.629Z" }, + { url = "https://files.pythonhosted.org/packages/28/44/efda282170a46bb4f19c3e2b90536513b1d821c414c28469a227ca5a1789/mmh3-5.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c4a2f3d83879e3de2eb8cbf562e71563a8ed15ee9b9c2e77ca5d9f73072ac15c", size = 97545, upload-time = "2025-07-29T07:42:27.04Z" }, + { url = "https://files.pythonhosted.org/packages/68/8f/534ae319c6e05d714f437e7206f78c17e66daca88164dff70286b0e8ea0c/mmh3-5.2.0-cp312-cp312-win32.whl", hash = "sha256:2421b9d665a0b1ad724ec7332fb5a98d075f50bc51a6ff854f3a1882bd650d49", size = 40805, upload-time = "2025-07-29T07:42:28.032Z" }, + { url = "https://files.pythonhosted.org/packages/b8/f6/f6abdcfefcedab3c964868048cfe472764ed358c2bf6819a70dd4ed4ed3a/mmh3-5.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:72d80005b7634a3a2220f81fbeb94775ebd12794623bb2e1451701ea732b4aa3", size = 41597, upload-time = "2025-07-29T07:42:28.894Z" }, + { url = "https://files.pythonhosted.org/packages/15/fd/f7420e8cbce45c259c770cac5718badf907b302d3a99ec587ba5ce030237/mmh3-5.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:3d6bfd9662a20c054bc216f861fa330c2dac7c81e7fb8307b5e32ab5b9b4d2e0", size = 39350, upload-time = "2025-07-29T07:42:29.794Z" }, { url = "https://files.pythonhosted.org/packages/d8/fa/27f6ab93995ef6ad9f940e96593c5dd24744d61a7389532b0fec03745607/mmh3-5.2.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:e79c00eba78f7258e5b354eccd4d7907d60317ced924ea4a5f2e9d83f5453065", size = 40874, upload-time = "2025-07-29T07:42:30.662Z" }, { url = "https://files.pythonhosted.org/packages/11/9c/03d13bcb6a03438bc8cac3d2e50f80908d159b31a4367c2e1a7a077ded32/mmh3-5.2.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:956127e663d05edbeec54df38885d943dfa27406594c411139690485128525de", size = 42012, upload-time = "2025-07-29T07:42:31.539Z" }, { url = "https://files.pythonhosted.org/packages/4e/78/0865d9765408a7d504f1789944e678f74e0888b96a766d578cb80b040999/mmh3-5.2.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:c3dca4cb5b946ee91b3d6bb700d137b1cd85c20827f89fdf9c16258253489044", size = 39197, upload-time = "2025-07-29T07:42:32.374Z" }, @@ -476,6 +565,20 @@ version = "23.0.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/01/33/ffd9c3eb087fa41dd79c3cf20c4c0ae3cdb877c4f8e1107a446006344924/pyarrow-23.0.0.tar.gz", hash = "sha256:180e3150e7edfcd182d3d9afba72f7cf19839a497cc76555a8dce998a8f67615", size = 1167185, upload-time = "2026-01-18T16:19:42.218Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/aa/c0/57fe251102ca834fee0ef69a84ad33cc0ff9d5dfc50f50b466846356ecd7/pyarrow-23.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:5574d541923efcbfdf1294a2746ae3b8c2498a2dc6cd477882f6f4e7b1ac08d3", size = 34276762, upload-time = "2026-01-18T16:14:34.128Z" }, + { url = "https://files.pythonhosted.org/packages/f8/4e/24130286548a5bc250cbed0b6bbf289a2775378a6e0e6f086ae8c68fc098/pyarrow-23.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:2ef0075c2488932e9d3c2eb3482f9459c4be629aa673b725d5e3cf18f777f8e4", size = 35821420, upload-time = "2026-01-18T16:14:40.699Z" }, + { url = "https://files.pythonhosted.org/packages/ee/55/a869e8529d487aa2e842d6c8865eb1e2c9ec33ce2786eb91104d2c3e3f10/pyarrow-23.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:65666fc269669af1ef1c14478c52222a2aa5c907f28b68fb50a203c777e4f60c", size = 44457412, upload-time = "2026-01-18T16:14:49.051Z" }, + { url = "https://files.pythonhosted.org/packages/36/81/1de4f0edfa9a483bbdf0082a05790bd6a20ed2169ea12a65039753be3a01/pyarrow-23.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:4d85cb6177198f3812db4788e394b757223f60d9a9f5ad6634b3e32be1525803", size = 47534285, upload-time = "2026-01-18T16:14:56.748Z" }, + { url = "https://files.pythonhosted.org/packages/f2/04/464a052d673b5ece074518f27377861662449f3c1fdb39ce740d646fd098/pyarrow-23.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1a9ff6fa4141c24a03a1a434c63c8fa97ce70f8f36bccabc18ebba905ddf0f17", size = 48157913, upload-time = "2026-01-18T16:15:05.114Z" }, + { url = "https://files.pythonhosted.org/packages/f4/1b/32a4de9856ee6688c670ca2def588382e573cce45241a965af04c2f61687/pyarrow-23.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:84839d060a54ae734eb60a756aeacb62885244aaa282f3c968f5972ecc7b1ecc", size = 50582529, upload-time = "2026-01-18T16:15:12.846Z" }, + { url = "https://files.pythonhosted.org/packages/db/c7/d6581f03e9b9e44ea60b52d1750ee1a7678c484c06f939f45365a45f7eef/pyarrow-23.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:a149a647dbfe928ce8830a713612aa0b16e22c64feac9d1761529778e4d4eaa5", size = 27542646, upload-time = "2026-01-18T16:15:18.89Z" }, + { url = "https://files.pythonhosted.org/packages/3d/bd/c861d020831ee57609b73ea721a617985ece817684dc82415b0bc3e03ac3/pyarrow-23.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:5961a9f646c232697c24f54d3419e69b4261ba8a8b66b0ac54a1851faffcbab8", size = 34189116, upload-time = "2026-01-18T16:15:28.054Z" }, + { url = "https://files.pythonhosted.org/packages/8c/23/7725ad6cdcbaf6346221391e7b3eecd113684c805b0a95f32014e6fa0736/pyarrow-23.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:632b3e7c3d232f41d64e1a4a043fb82d44f8a349f339a1188c6a0dd9d2d47d8a", size = 35803831, upload-time = "2026-01-18T16:15:33.798Z" }, + { url = "https://files.pythonhosted.org/packages/57/06/684a421543455cdc2944d6a0c2cc3425b028a4c6b90e34b35580c4899743/pyarrow-23.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:76242c846db1411f1d6c2cc3823be6b86b40567ee24493344f8226ba34a81333", size = 44436452, upload-time = "2026-01-18T16:15:41.598Z" }, + { url = "https://files.pythonhosted.org/packages/c6/6f/8f9eb40c2328d66e8b097777ddcf38494115ff9f1b5bc9754ba46991191e/pyarrow-23.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b73519f8b52ae28127000986bf228fda781e81d3095cd2d3ece76eb5cf760e1b", size = 47557396, upload-time = "2026-01-18T16:15:51.252Z" }, + { url = "https://files.pythonhosted.org/packages/10/6e/f08075f1472e5159553501fde2cc7bc6700944bdabe49a03f8a035ee6ccd/pyarrow-23.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:068701f6823449b1b6469120f399a1239766b117d211c5d2519d4ed5861f75de", size = 48147129, upload-time = "2026-01-18T16:16:00.299Z" }, + { url = "https://files.pythonhosted.org/packages/7d/82/d5a680cd507deed62d141cc7f07f7944a6766fc51019f7f118e4d8ad0fb8/pyarrow-23.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1801ba947015d10e23bca9dd6ef5d0e9064a81569a89b6e9a63b59224fd060df", size = 50596642, upload-time = "2026-01-18T16:16:08.502Z" }, + { url = "https://files.pythonhosted.org/packages/a9/26/4f29c61b3dce9fa7780303b86895ec6a0917c9af927101daaaf118fbe462/pyarrow-23.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:52265266201ec25b6839bf6bd4ea918ca6d50f31d13e1cf200b4261cd11dc25c", size = 27660628, upload-time = "2026-01-18T16:16:15.28Z" }, { url = "https://files.pythonhosted.org/packages/66/34/564db447d083ec7ff93e0a883a597d2f214e552823bfc178a2d0b1f2c257/pyarrow-23.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:ad96a597547af7827342ffb3c503c8316e5043bb09b47a84885ce39394c96e00", size = 34184630, upload-time = "2026-01-18T16:16:22.141Z" }, { url = "https://files.pythonhosted.org/packages/aa/3a/3999daebcb5e6119690c92a621c4d78eef2ffba7a0a1b56386d2875fcd77/pyarrow-23.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:b9edf990df77c2901e79608f08c13fbde60202334a4fcadb15c1f57bf7afee43", size = 35796820, upload-time = "2026-01-18T16:16:29.441Z" }, { url = "https://files.pythonhosted.org/packages/ec/ee/39195233056c6a8d0976d7d1ac1cd4fe21fb0ec534eca76bc23ef3f60e11/pyarrow-23.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:36d1b5bc6ddcaff0083ceec7e2561ed61a51f49cce8be079ee8ed406acb6fdef", size = 44438735, upload-time = "2026-01-18T16:16:38.79Z" }, @@ -530,6 +633,34 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/71/70/23b021c950c2addd24ec408e9ab05d59b035b39d97cdc1130e1bce647bb6/pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e", size = 460952, upload-time = "2025-11-04T13:43:49.098Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/72/74a989dd9f2084b3d9530b0915fdda64ac48831c30dbf7c72a41a5232db8/pydantic_core-2.41.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a3a52f6156e73e7ccb0f8cced536adccb7042be67cb45f9562e12b319c119da6", size = 2105873, upload-time = "2025-11-04T13:39:31.373Z" }, + { url = "https://files.pythonhosted.org/packages/12/44/37e403fd9455708b3b942949e1d7febc02167662bf1a7da5b78ee1ea2842/pydantic_core-2.41.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7f3bf998340c6d4b0c9a2f02d6a400e51f123b59565d74dc60d252ce888c260b", size = 1899826, upload-time = "2025-11-04T13:39:32.897Z" }, + { url = "https://files.pythonhosted.org/packages/33/7f/1d5cab3ccf44c1935a359d51a8a2a9e1a654b744b5e7f80d41b88d501eec/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:378bec5c66998815d224c9ca994f1e14c0c21cb95d2f52b6021cc0b2a58f2a5a", size = 1917869, upload-time = "2025-11-04T13:39:34.469Z" }, + { url = "https://files.pythonhosted.org/packages/6e/6a/30d94a9674a7fe4f4744052ed6c5e083424510be1e93da5bc47569d11810/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7b576130c69225432866fe2f4a469a85a54ade141d96fd396dffcf607b558f8", size = 2063890, upload-time = "2025-11-04T13:39:36.053Z" }, + { url = "https://files.pythonhosted.org/packages/50/be/76e5d46203fcb2750e542f32e6c371ffa9b8ad17364cf94bb0818dbfb50c/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6cb58b9c66f7e4179a2d5e0f849c48eff5c1fca560994d6eb6543abf955a149e", size = 2229740, upload-time = "2025-11-04T13:39:37.753Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ee/fed784df0144793489f87db310a6bbf8118d7b630ed07aa180d6067e653a/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:88942d3a3dff3afc8288c21e565e476fc278902ae4d6d134f1eeda118cc830b1", size = 2350021, upload-time = "2025-11-04T13:39:40.94Z" }, + { url = "https://files.pythonhosted.org/packages/c8/be/8fed28dd0a180dca19e72c233cbf58efa36df055e5b9d90d64fd1740b828/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f31d95a179f8d64d90f6831d71fa93290893a33148d890ba15de25642c5d075b", size = 2066378, upload-time = "2025-11-04T13:39:42.523Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3b/698cf8ae1d536a010e05121b4958b1257f0b5522085e335360e53a6b1c8b/pydantic_core-2.41.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c1df3d34aced70add6f867a8cf413e299177e0c22660cc767218373d0779487b", size = 2175761, upload-time = "2025-11-04T13:39:44.553Z" }, + { url = "https://files.pythonhosted.org/packages/b8/ba/15d537423939553116dea94ce02f9c31be0fa9d0b806d427e0308ec17145/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4009935984bd36bd2c774e13f9a09563ce8de4abaa7226f5108262fa3e637284", size = 2146303, upload-time = "2025-11-04T13:39:46.238Z" }, + { url = "https://files.pythonhosted.org/packages/58/7f/0de669bf37d206723795f9c90c82966726a2ab06c336deba4735b55af431/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:34a64bc3441dc1213096a20fe27e8e128bd3ff89921706e83c0b1ac971276594", size = 2340355, upload-time = "2025-11-04T13:39:48.002Z" }, + { url = "https://files.pythonhosted.org/packages/e5/de/e7482c435b83d7e3c3ee5ee4451f6e8973cff0eb6007d2872ce6383f6398/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c9e19dd6e28fdcaa5a1de679aec4141f691023916427ef9bae8584f9c2fb3b0e", size = 2319875, upload-time = "2025-11-04T13:39:49.705Z" }, + { url = "https://files.pythonhosted.org/packages/fe/e6/8c9e81bb6dd7560e33b9053351c29f30c8194b72f2d6932888581f503482/pydantic_core-2.41.5-cp311-cp311-win32.whl", hash = "sha256:2c010c6ded393148374c0f6f0bf89d206bf3217f201faa0635dcd56bd1520f6b", size = 1987549, upload-time = "2025-11-04T13:39:51.842Z" }, + { url = "https://files.pythonhosted.org/packages/11/66/f14d1d978ea94d1bc21fc98fcf570f9542fe55bfcc40269d4e1a21c19bf7/pydantic_core-2.41.5-cp311-cp311-win_amd64.whl", hash = "sha256:76ee27c6e9c7f16f47db7a94157112a2f3a00e958bc626e2f4ee8bec5c328fbe", size = 2011305, upload-time = "2025-11-04T13:39:53.485Z" }, + { url = "https://files.pythonhosted.org/packages/56/d8/0e271434e8efd03186c5386671328154ee349ff0354d83c74f5caaf096ed/pydantic_core-2.41.5-cp311-cp311-win_arm64.whl", hash = "sha256:4bc36bbc0b7584de96561184ad7f012478987882ebf9f9c389b23f432ea3d90f", size = 1972902, upload-time = "2025-11-04T13:39:56.488Z" }, + { url = "https://files.pythonhosted.org/packages/5f/5d/5f6c63eebb5afee93bcaae4ce9a898f3373ca23df3ccaef086d0233a35a7/pydantic_core-2.41.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7", size = 2110990, upload-time = "2025-11-04T13:39:58.079Z" }, + { url = "https://files.pythonhosted.org/packages/aa/32/9c2e8ccb57c01111e0fd091f236c7b371c1bccea0fa85247ac55b1e2b6b6/pydantic_core-2.41.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0", size = 1896003, upload-time = "2025-11-04T13:39:59.956Z" }, + { url = "https://files.pythonhosted.org/packages/68/b8/a01b53cb0e59139fbc9e4fda3e9724ede8de279097179be4ff31f1abb65a/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69", size = 1919200, upload-time = "2025-11-04T13:40:02.241Z" }, + { url = "https://files.pythonhosted.org/packages/38/de/8c36b5198a29bdaade07b5985e80a233a5ac27137846f3bc2d3b40a47360/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75", size = 2052578, upload-time = "2025-11-04T13:40:04.401Z" }, + { url = "https://files.pythonhosted.org/packages/00/b5/0e8e4b5b081eac6cb3dbb7e60a65907549a1ce035a724368c330112adfdd/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05", size = 2208504, upload-time = "2025-11-04T13:40:06.072Z" }, + { url = "https://files.pythonhosted.org/packages/77/56/87a61aad59c7c5b9dc8caad5a41a5545cba3810c3e828708b3d7404f6cef/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc", size = 2335816, upload-time = "2025-11-04T13:40:07.835Z" }, + { url = "https://files.pythonhosted.org/packages/0d/76/941cc9f73529988688a665a5c0ecff1112b3d95ab48f81db5f7606f522d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c", size = 2075366, upload-time = "2025-11-04T13:40:09.804Z" }, + { url = "https://files.pythonhosted.org/packages/d3/43/ebef01f69baa07a482844faaa0a591bad1ef129253ffd0cdaa9d8a7f72d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5", size = 2171698, upload-time = "2025-11-04T13:40:12.004Z" }, + { url = "https://files.pythonhosted.org/packages/b1/87/41f3202e4193e3bacfc2c065fab7706ebe81af46a83d3e27605029c1f5a6/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c", size = 2132603, upload-time = "2025-11-04T13:40:13.868Z" }, + { url = "https://files.pythonhosted.org/packages/49/7d/4c00df99cb12070b6bccdef4a195255e6020a550d572768d92cc54dba91a/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294", size = 2329591, upload-time = "2025-11-04T13:40:15.672Z" }, + { url = "https://files.pythonhosted.org/packages/cc/6a/ebf4b1d65d458f3cda6a7335d141305dfa19bdc61140a884d165a8a1bbc7/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1", size = 2319068, upload-time = "2025-11-04T13:40:17.532Z" }, + { url = "https://files.pythonhosted.org/packages/49/3b/774f2b5cd4192d5ab75870ce4381fd89cf218af999515baf07e7206753f0/pydantic_core-2.41.5-cp312-cp312-win32.whl", hash = "sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d", size = 1985908, upload-time = "2025-11-04T13:40:19.309Z" }, + { url = "https://files.pythonhosted.org/packages/86/45/00173a033c801cacf67c190fef088789394feaf88a98a7035b0e40d53dc9/pydantic_core-2.41.5-cp312-cp312-win_amd64.whl", hash = "sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815", size = 2020145, upload-time = "2025-11-04T13:40:21.548Z" }, + { url = "https://files.pythonhosted.org/packages/f9/22/91fbc821fa6d261b376a3f73809f907cec5ca6025642c463d3488aad22fb/pydantic_core-2.41.5-cp312-cp312-win_arm64.whl", hash = "sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3", size = 1976179, upload-time = "2025-11-04T13:40:23.393Z" }, { url = "https://files.pythonhosted.org/packages/87/06/8806241ff1f70d9939f9af039c6c35f2360cf16e93c2ca76f184e76b1564/pydantic_core-2.41.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9", size = 2120403, upload-time = "2025-11-04T13:40:25.248Z" }, { url = "https://files.pythonhosted.org/packages/94/02/abfa0e0bda67faa65fef1c84971c7e45928e108fe24333c81f3bfe35d5f5/pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34", size = 1896206, upload-time = "2025-11-04T13:40:27.099Z" }, { url = "https://files.pythonhosted.org/packages/15/df/a4c740c0943e93e6500f9eb23f4ca7ec9bf71b19e608ae5b579678c8d02f/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0", size = 1919307, upload-time = "2025-11-04T13:40:29.806Z" }, @@ -572,6 +703,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5c/96/5fb7d8c3c17bc8c62fdb031c47d77a1af698f1d7a406b0f79aaa1338f9ad/pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa", size = 1988906, upload-time = "2025-11-04T13:41:56.606Z" }, { url = "https://files.pythonhosted.org/packages/22/ed/182129d83032702912c2e2d8bbe33c036f342cc735737064668585dac28f/pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c", size = 1981607, upload-time = "2025-11-04T13:41:58.889Z" }, { url = "https://files.pythonhosted.org/packages/9f/ed/068e41660b832bb0b1aa5b58011dea2a3fe0ba7861ff38c4d4904c1c1a99/pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008", size = 1974769, upload-time = "2025-11-04T13:42:01.186Z" }, + { url = "https://files.pythonhosted.org/packages/11/72/90fda5ee3b97e51c494938a4a44c3a35a9c96c19bba12372fb9c634d6f57/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:b96d5f26b05d03cc60f11a7761a5ded1741da411e7fe0909e27a5e6a0cb7b034", size = 2115441, upload-time = "2025-11-04T13:42:39.557Z" }, + { url = "https://files.pythonhosted.org/packages/1f/53/8942f884fa33f50794f119012dc6a1a02ac43a56407adaac20463df8e98f/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:634e8609e89ceecea15e2d61bc9ac3718caaaa71963717bf3c8f38bfde64242c", size = 1930291, upload-time = "2025-11-04T13:42:42.169Z" }, + { url = "https://files.pythonhosted.org/packages/79/c8/ecb9ed9cd942bce09fc888ee960b52654fbdbede4ba6c2d6e0d3b1d8b49c/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:93e8740d7503eb008aa2df04d3b9735f845d43ae845e6dcd2be0b55a2da43cd2", size = 1948632, upload-time = "2025-11-04T13:42:44.564Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1b/687711069de7efa6af934e74f601e2a4307365e8fdc404703afc453eab26/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f15489ba13d61f670dcc96772e733aad1a6f9c429cc27574c6cdaed82d0146ad", size = 2138905, upload-time = "2025-11-04T13:42:47.156Z" }, + { url = "https://files.pythonhosted.org/packages/09/32/59b0c7e63e277fa7911c2fc70ccfb45ce4b98991e7ef37110663437005af/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd", size = 2110495, upload-time = "2025-11-04T13:42:49.689Z" }, + { url = "https://files.pythonhosted.org/packages/aa/81/05e400037eaf55ad400bcd318c05bb345b57e708887f07ddb2d20e3f0e98/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc", size = 1915388, upload-time = "2025-11-04T13:42:52.215Z" }, + { url = "https://files.pythonhosted.org/packages/6e/0d/e3549b2399f71d56476b77dbf3cf8937cec5cd70536bdc0e374a421d0599/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56", size = 1942879, upload-time = "2025-11-04T13:42:56.483Z" }, + { url = "https://files.pythonhosted.org/packages/f7/07/34573da085946b6a313d7c42f82f16e8920bfd730665de2d11c0c37a74b5/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b", size = 2139017, upload-time = "2025-11-04T13:42:59.471Z" }, + { url = "https://files.pythonhosted.org/packages/5f/9b/1b3f0e9f9305839d7e84912f9e8bfbd191ed1b1ef48083609f0dabde978c/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b2379fa7ed44ddecb5bfe4e48577d752db9fc10be00a6b7446e9663ba143de26", size = 2101980, upload-time = "2025-11-04T13:43:25.97Z" }, + { url = "https://files.pythonhosted.org/packages/a4/ed/d71fefcb4263df0da6a85b5d8a7508360f2f2e9b3bf5814be9c8bccdccc1/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:266fb4cbf5e3cbd0b53669a6d1b039c45e3ce651fd5442eff4d07c2cc8d66808", size = 1923865, upload-time = "2025-11-04T13:43:28.763Z" }, + { url = "https://files.pythonhosted.org/packages/ce/3a/626b38db460d675f873e4444b4bb030453bbe7b4ba55df821d026a0493c4/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58133647260ea01e4d0500089a8c4f07bd7aa6ce109682b1426394988d8aaacc", size = 2134256, upload-time = "2025-11-04T13:43:31.71Z" }, + { url = "https://files.pythonhosted.org/packages/83/d9/8412d7f06f616bbc053d30cb4e5f76786af3221462ad5eee1f202021eb4e/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:287dad91cfb551c363dc62899a80e9e14da1f0e2b6ebde82c806612ca2a13ef1", size = 2174762, upload-time = "2025-11-04T13:43:34.744Z" }, + { url = "https://files.pythonhosted.org/packages/55/4c/162d906b8e3ba3a99354e20faa1b49a85206c47de97a639510a0e673f5da/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:03b77d184b9eb40240ae9fd676ca364ce1085f203e1b1256f8ab9984dca80a84", size = 2143141, upload-time = "2025-11-04T13:43:37.701Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f2/f11dd73284122713f5f89fc940f370d035fa8e1e078d446b3313955157fe/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:a668ce24de96165bb239160b3d854943128f4334822900534f2fe947930e5770", size = 2330317, upload-time = "2025-11-04T13:43:40.406Z" }, + { url = "https://files.pythonhosted.org/packages/88/9d/b06ca6acfe4abb296110fb1273a4d848a0bfb2ff65f3ee92127b3244e16b/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f14f8f046c14563f8eb3f45f499cc658ab8d10072961e07225e507adb700e93f", size = 2316992, upload-time = "2025-11-04T13:43:43.602Z" }, + { url = "https://files.pythonhosted.org/packages/36/c7/cfc8e811f061c841d7990b0201912c3556bfeb99cdcb7ed24adc8d6f8704/pydantic_core-2.41.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51", size = 2145302, upload-time = "2025-11-04T13:43:46.64Z" }, ] [[package]] @@ -616,6 +763,18 @@ dependencies = [ { name = "tenacity" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a3/0e/90e61c38504f4fbd5ed79631f85da7d5ea5e5bf997bdeaa65b28ebf04cab/pyiceberg-0.10.0.tar.gz", hash = "sha256:2525afa5e7e5fc4e72b291f8e1cc219e982d2bda5ff17e62cd05b8d91c4139f5", size = 842633, upload-time = "2025-09-11T14:59:34.044Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/62/b6f7bed760d0896958d046ca3c188fd15467c6502bcc2dc301ac0554c1ce/pyiceberg-0.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2c799c9149e06ef9ece22945d5c198ffc69f5c04b314b59a43c2d4c1bb9ade84", size = 591127, upload-time = "2025-09-11T14:59:08.72Z" }, + { url = "https://files.pythonhosted.org/packages/4c/b2/294c74e70c68744a8246924fee350095cc46f97f81d1e37125011d8e1bcb/pyiceberg-0.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a8c7070fe1262f50694b12241b5373ee89c8aededda82ef325cb14e5a95cc461", size = 587041, upload-time = "2025-09-11T14:59:10.643Z" }, + { url = "https://files.pythonhosted.org/packages/7a/2f/9a9f0a01f0dae2cefc024a2bd84a00ff2a5d8d952f37053c46523c1dd7a6/pyiceberg-0.10.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e0d1a4896f546b1e115ece4212dd02b383eeb3c7ff5c072624b15f531b776f36", size = 1135929, upload-time = "2025-09-11T14:59:12.164Z" }, + { url = "https://files.pythonhosted.org/packages/e1/c2/51deddeec916d44a04cc26053179b560ffceba72e4561b6cf58a64aea209/pyiceberg-0.10.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1b0ef2f1880dd7549cc54ccb1a25f61ad5329e079cba372b4c239b0012aecac6", size = 1131851, upload-time = "2025-09-11T14:59:13.792Z" }, + { url = "https://files.pythonhosted.org/packages/ba/cc/e9cf3fa56d67306ba29352d56152907a91ca29eabc1a30d3177cee0d1418/pyiceberg-0.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:2127c795e451b971bd3f55cbda2d2c8200182bec3476e590e4a3453e60efda3c", size = 583472, upload-time = "2025-09-11T14:59:15.173Z" }, + { url = "https://files.pythonhosted.org/packages/03/61/f5042dd09cb91deed908a39acd5012f1ac6910ddf84ada889751732f0df8/pyiceberg-0.10.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:64cad9d1db08192605875a872152cbcaca147ea486cfa94773fa5f4f65d78a23", size = 629281, upload-time = "2025-09-11T14:59:17.585Z" }, + { url = "https://files.pythonhosted.org/packages/8e/50/960f7239eedd4b1bab2a611f5e100fffc138549c1213760a57cd24a5bac1/pyiceberg-0.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3e12cf585318f0f48d31a77b4149e0e5b4c41e03a24aa8612e060f20ff41eb10", size = 623424, upload-time = "2025-09-11T14:59:19.045Z" }, + { url = "https://files.pythonhosted.org/packages/f5/2b/756a74c80db6edd82c8d3f23c3ae13e7d6620300b87ef792c2a4d3935b30/pyiceberg-0.10.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6979dd741cee263c1235595f71888c73365f2725697411027c4bd81046db3294", size = 1377048, upload-time = "2025-09-11T14:59:20.541Z" }, + { url = "https://files.pythonhosted.org/packages/bb/35/9c18cb4ddc7d371db63714abb2f5e8414bc7a4d63f474644a2aea2933fe6/pyiceberg-0.10.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:13fd03ec3da6eb4d3b55ff94b647946a7749bede5d743c75b39deaad26421200", size = 1369921, upload-time = "2025-09-11T14:59:22.134Z" }, + { url = "https://files.pythonhosted.org/packages/7b/b3/c012dc6b5bc3d0a84821936789c753f5c44aec619b64fbcf7f90038d172e/pyiceberg-0.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:33367c84bcb0a2fbbe54cbbfe062691ab93b91a2e3d319bb546ec5b9b45b6057", size = 617722, upload-time = "2025-09-11T14:59:23.67Z" }, +] [package.optional-dependencies] pyiceberg-core = [ @@ -650,6 +809,34 @@ version = "1.0.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/0f/e4/975f0fa77fc3590820b4a3ac49704644b389795409bc12eb91729f845812/pyroaring-1.0.3.tar.gz", hash = "sha256:cd7392d1c010c9e41c11c62cd0610c8852e7e9698b1f7f6c2fcdefe50e7ef6da", size = 188688, upload-time = "2025-10-09T09:08:22.448Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/39/ed/5e555dd99b12318ea1c7666b773fc4f097aeb609eeb1c1b3da519d445f71/pyroaring-1.0.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:755cdac1f9a1b7b5c621e570d4f6dbcf3b8e4a1e35a66f976104ecb35dce4ed2", size = 675916, upload-time = "2025-10-09T09:06:53.174Z" }, + { url = "https://files.pythonhosted.org/packages/da/06/dd8a9a87b90c4560f8384ab1dbafcd40c2a16f6777a07334a8e341bd7383/pyroaring-1.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ebab073db620f26f0ba11e13fa2f35e3b1298209fba47b6bc8cb6f0e2c9627f9", size = 369743, upload-time = "2025-10-09T09:06:54.421Z" }, + { url = "https://files.pythonhosted.org/packages/35/aa/da882011045ddacffe818a4fcbdd7e609a15f9c83d536222ec5b17af4aa9/pyroaring-1.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:684fb8dffe19bdb7f91897c65eac6eee23b1e46043c47eb24288f28a1170fe04", size = 313981, upload-time = "2025-10-09T09:06:55.514Z" }, + { url = "https://files.pythonhosted.org/packages/ed/3c/f6534844b02e2505ccdc9aae461c9838ab96f72b5688c045448761735512/pyroaring-1.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:678d31fc24e82945a1bfb14816c77823983382ffea76985d494782aa2f058427", size = 1923181, upload-time = "2025-10-09T09:06:56.897Z" }, + { url = "https://files.pythonhosted.org/packages/ea/82/9f1a85ba33e3d89b9cdb8183fb2fd2f25720d10742dd8827508ccccc13ae/pyroaring-1.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d815f624e0285db3669f673d1725cb754b120ec70d0032d7c7166103a96c96d", size = 2113222, upload-time = "2025-10-09T09:06:58.388Z" }, + { url = "https://files.pythonhosted.org/packages/a7/f8/4d4340971cbc1379f987c847080bcb7f9765a57e122f392c3a3485c9587e/pyroaring-1.0.3-cp311-cp311-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:57fd5b80dacb8e888402b6b7508a734c6a527063e4e24e882ff2e0fd90721ada", size = 1837385, upload-time = "2025-10-09T09:06:59.449Z" }, + { url = "https://files.pythonhosted.org/packages/c6/58/d14cc561685e4c224af26b4fdb4f6c7e643294ac5a4b29f178b5cbb71af1/pyroaring-1.0.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ab26a7a45a0bb46c00394d1a60a9f2d57c220f84586e30d59b39784b0f94aee6", size = 1856170, upload-time = "2025-10-09T09:07:00.608Z" }, + { url = "https://files.pythonhosted.org/packages/d1/d2/d2d9790c373f6438d4d0958bc4c79f3dc77826d8553743ff3f64acdc9ab3/pyroaring-1.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9232f3f606315d59049c128154100fd05008d5c5c211e48b21848cd41ee64d26", size = 2909282, upload-time = "2025-10-09T09:07:02.124Z" }, + { url = "https://files.pythonhosted.org/packages/bc/28/4b2277982302b5b406998064ca1eaef1a79e4ea87185f511e33e7a7e3511/pyroaring-1.0.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:f34b44b3ec3df97b978799f2901fefb2a48d367496fd1cde3cc5fe8b3bc13510", size = 2701034, upload-time = "2025-10-09T09:07:03.403Z" }, + { url = "https://files.pythonhosted.org/packages/d2/91/b2340193825fa2431cf735f0ecb23206fb31f386fecca38336935a294513/pyroaring-1.0.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:25a83ec6bac3106568bd3fdd316f0fee52aa0be8c72da565ad02b10ae7905924", size = 3028962, upload-time = "2025-10-09T09:07:05.558Z" }, + { url = "https://files.pythonhosted.org/packages/07/ea/ad79073cc5d8dcca35d1a955bb886d96905e9dacc58d1971fda012a5ad18/pyroaring-1.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c17d4ec53b5b6b333d9a9515051213a691293ada785dc8c025d3641482597ed3", size = 3152109, upload-time = "2025-10-09T09:07:06.887Z" }, + { url = "https://files.pythonhosted.org/packages/9a/de/f55a1093acb16d25ff9811546823e59078e4a3e56d2eb0ff5d10f696933d/pyroaring-1.0.3-cp311-cp311-win32.whl", hash = "sha256:d54024459ace600f1d1ffbc6dc3c60eb47cca3b678701f06148f59e10f6f8d7b", size = 204246, upload-time = "2025-10-09T09:07:08.036Z" }, + { url = "https://files.pythonhosted.org/packages/c6/e5/36bf3039733b8e00732892c9334b2f5309f38e72af0b3b40b8729b5857a3/pyroaring-1.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:c28750148ef579a7447a8cb60b39e5943e03f8c29bce8f2788728f6f23d1887a", size = 254637, upload-time = "2025-10-09T09:07:09.103Z" }, + { url = "https://files.pythonhosted.org/packages/d6/e8/e2b78e595b5a82a6014af327614756a55f17ec4120a2ab197f1762641316/pyroaring-1.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:535d8deccbd8db2c6bf38629243e9646756905574a742b2a72ff51d6461d616c", size = 219597, upload-time = "2025-10-09T09:07:10.38Z" }, + { url = "https://files.pythonhosted.org/packages/dd/09/a5376d55672e0535019ba1469888909d0046cea0cfb969a4aa1f99caaf22/pyroaring-1.0.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:add3e4c78eb590a76526ecce8d1566eecdd5822e351c36b3697997f4a80ed808", size = 681056, upload-time = "2025-10-09T09:07:11.497Z" }, + { url = "https://files.pythonhosted.org/packages/23/dd/78f59d361bd9ebf8de3660408b0c48664ade0a057ebcf4b207d99ac1a698/pyroaring-1.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ebaffe846cf4ba4f00ce6b8a9f39613f24e2d09447e77be4fa6e898bc36451b6", size = 375111, upload-time = "2025-10-09T09:07:12.597Z" }, + { url = "https://files.pythonhosted.org/packages/bf/03/10dc93f83a5453eb40a69c79106a8385b40aa12cf4531ca72bd9d7f45cb2/pyroaring-1.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a9459f27498f97d08031a34a5ead230b77eb0ab3cc3d85b7f54faa2fd548acd6", size = 314319, upload-time = "2025-10-09T09:07:13.579Z" }, + { url = "https://files.pythonhosted.org/packages/86/9e/b00c38a7e62a73e152055f593595c37152e61fc2896fd11538a7c71fbe4e/pyroaring-1.0.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2b2eb8bd1c35c772994889be9f7dda09477475d7aa1e2af9ab4ef18619326f6", size = 1869251, upload-time = "2025-10-09T09:07:14.584Z" }, + { url = "https://files.pythonhosted.org/packages/4f/33/f32d00ca105b66303deab43d027c3574c8ade8525dac0e5b50a9fb4d1b76/pyroaring-1.0.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d31f4c1c906f1af14ce61a3959d04a14a64c594f8a768399146a45bbd341f21f", size = 2071551, upload-time = "2025-10-09T09:07:15.713Z" }, + { url = "https://files.pythonhosted.org/packages/5d/89/e953cae181ba4c7523334855a1ca0ae8eeea3cee8d7cd39c56bd99709d3f/pyroaring-1.0.3-cp312-cp312-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:53be988fc86698d56c11049bfe5113a2f6990adb1fa2782b29636509808b6aa7", size = 1781071, upload-time = "2025-10-09T09:07:17.19Z" }, + { url = "https://files.pythonhosted.org/packages/fa/db/65d4be532e68b62a84a9c89b24d0a1394f452f484fa29392142d9a3b9c48/pyroaring-1.0.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7df84d223424523b19a23781f4246cc247fd6d821e1bc0853c2f25669136f7d0", size = 1795670, upload-time = "2025-10-09T09:07:18.524Z" }, + { url = "https://files.pythonhosted.org/packages/f5/9e/684ea0568ce7d30fc4e01ad1c666e9ce1a5b1702fa630231f4f6bdb96539/pyroaring-1.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:34a781f1f9766897f63ef18be129827340ae37764015b83fdcff1efb9e29136d", size = 2849305, upload-time = "2025-10-09T09:07:20.388Z" }, + { url = "https://files.pythonhosted.org/packages/7c/fd/d7773a2adf91f45d8924197954c66b1694325afd2f27e02edaac07338402/pyroaring-1.0.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:1f414343b4ed0756734328cdf2a91022fc54503769e3f8d79bd0b672ea815a16", size = 2692843, upload-time = "2025-10-09T09:07:22.042Z" }, + { url = "https://files.pythonhosted.org/packages/13/72/b8a99ba138eebd8ff9bf8d15f3942e9e43e8e45723e2e6b7b09e542b7448/pyroaring-1.0.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:d16ae185c72dc64f76335dbe53e53a892e78115adc92194957d1b7ef74d230b9", size = 2983440, upload-time = "2025-10-09T09:07:23.419Z" }, + { url = "https://files.pythonhosted.org/packages/ca/94/e6ed1f682d850e039c71b2032bacdefc5082dc809796cf34b9e6f24c604d/pyroaring-1.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f888447bf22dde7759108bfe6dfbeb6bbb61b14948de9c4cb6843c4dd57e2215", size = 3117542, upload-time = "2025-10-09T09:07:25.104Z" }, + { url = "https://files.pythonhosted.org/packages/8f/89/d55b0ed3e098ef89c421b43b748afe3d90eb250cab50b9e53e3a3449ac58/pyroaring-1.0.3-cp312-cp312-win32.whl", hash = "sha256:fbbdc44c51a0a3efd7be3dbe04466278ce098fcd101aa1905849319042159770", size = 205118, upload-time = "2025-10-09T09:07:26.532Z" }, + { url = "https://files.pythonhosted.org/packages/c8/e1/b71fef6a73efb50110d33d714235ff7059f4ebae98dc474b6549b322f48f/pyroaring-1.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:3b217c4b3ad953b4c759a0d2f9bd95316f0c345b9f7adb49e6ded7a1f5106bd4", size = 260629, upload-time = "2025-10-09T09:07:27.528Z" }, + { url = "https://files.pythonhosted.org/packages/57/33/66ee872079c9c47512d6e17d374bcad8d91350c24dc20fbe678c34b33745/pyroaring-1.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:e6bcf838564c21bab8fe6c2748b4990d4cd90612d8c470c04889def7bb5114ea", size = 219032, upload-time = "2025-10-09T09:07:28.754Z" }, { url = "https://files.pythonhosted.org/packages/1f/95/97142ee32587ddda9e2cd614b865eeb5c0ee91006a51928f4074cd6e8e5f/pyroaring-1.0.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:20bc947054b197d1baa76cd05d70b8e04f95b82e698266e2f8f2f4b36d764477", size = 678813, upload-time = "2025-10-09T09:07:29.936Z" }, { url = "https://files.pythonhosted.org/packages/70/5e/cff22be3a76a80024bdf00a9decdffedc6e80f037328a58b58c1b521442d/pyroaring-1.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ba5909b4c66bb85cab345e2f3a87e5ce671509c94b8c9823d8db64e107cbe854", size = 373661, upload-time = "2025-10-09T09:07:30.983Z" }, { url = "https://files.pythonhosted.org/packages/86/73/fc406a67cd49e1707d1c3d08214458959dd579eff88c28587b356dfa068b/pyroaring-1.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b744746ba5da27fad760067f12633f5d384db6a1e65648d00244ceacbbd87731", size = 313559, upload-time = "2025-10-09T09:07:32.099Z" }, @@ -731,6 +918,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/87/2a/a1810c8627b9ec8c57ec5ec325d306701ae7be50235e8fd81266e002a3cc/rich-14.3.1-py3-none-any.whl", hash = "sha256:da750b1aebbff0b372557426fb3f35ba56de8ef954b3190315eb64076d6fb54e", size = 309952, upload-time = "2026-01-24T21:40:42.969Z" }, ] +[[package]] +name = "ruff" +version = "0.15.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/06/04/eab13a954e763b0606f460443fcbf6bb5a0faf06890ea3754ff16523dce5/ruff-0.15.2.tar.gz", hash = "sha256:14b965afee0969e68bb871eba625343b8673375f457af4abe98553e8bbb98342", size = 4558148, upload-time = "2026-02-19T22:32:20.271Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2f/70/3a4dc6d09b13cb3e695f28307e5d889b2e1a66b7af9c5e257e796695b0e6/ruff-0.15.2-py3-none-linux_armv6l.whl", hash = "sha256:120691a6fdae2f16d65435648160f5b81a9625288f75544dc40637436b5d3c0d", size = 10430565, upload-time = "2026-02-19T22:32:41.824Z" }, + { url = "https://files.pythonhosted.org/packages/71/0b/bb8457b56185ece1305c666dc895832946d24055be90692381c31d57466d/ruff-0.15.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:a89056d831256099658b6bba4037ac6dd06f49d194199215befe2bb10457ea5e", size = 10820354, upload-time = "2026-02-19T22:32:07.366Z" }, + { url = "https://files.pythonhosted.org/packages/2d/c1/e0532d7f9c9e0b14c46f61b14afd563298b8b83f337b6789ddd987e46121/ruff-0.15.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e36dee3a64be0ebd23c86ffa3aa3fd3ac9a712ff295e192243f814a830b6bd87", size = 10170767, upload-time = "2026-02-19T22:32:13.188Z" }, + { url = "https://files.pythonhosted.org/packages/47/e8/da1aa341d3af017a21c7a62fb5ec31d4e7ad0a93ab80e3a508316efbcb23/ruff-0.15.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a9fb47b6d9764677f8c0a193c0943ce9a05d6763523f132325af8a858eadc2b9", size = 10529591, upload-time = "2026-02-19T22:32:02.547Z" }, + { url = "https://files.pythonhosted.org/packages/93/74/184fbf38e9f3510231fbc5e437e808f0b48c42d1df9434b208821efcd8d6/ruff-0.15.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f376990f9d0d6442ea9014b19621d8f2aaf2b8e39fdbfc79220b7f0c596c9b80", size = 10260771, upload-time = "2026-02-19T22:32:36.938Z" }, + { url = "https://files.pythonhosted.org/packages/05/ac/605c20b8e059a0bc4b42360414baa4892ff278cec1c91fff4be0dceedefd/ruff-0.15.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2dcc987551952d73cbf5c88d9fdee815618d497e4df86cd4c4824cc59d5dd75f", size = 11045791, upload-time = "2026-02-19T22:32:31.642Z" }, + { url = "https://files.pythonhosted.org/packages/fd/52/db6e419908f45a894924d410ac77d64bdd98ff86901d833364251bd08e22/ruff-0.15.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:42a47fd785cbe8c01b9ff45031af875d101b040ad8f4de7bbb716487c74c9a77", size = 11879271, upload-time = "2026-02-19T22:32:29.305Z" }, + { url = "https://files.pythonhosted.org/packages/3e/d8/7992b18f2008bdc9231d0f10b16df7dda964dbf639e2b8b4c1b4e91b83af/ruff-0.15.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cbe9f49354866e575b4c6943856989f966421870e85cd2ac94dccb0a9dcb2fea", size = 11303707, upload-time = "2026-02-19T22:32:22.492Z" }, + { url = "https://files.pythonhosted.org/packages/d7/02/849b46184bcfdd4b64cde61752cc9a146c54759ed036edd11857e9b8443b/ruff-0.15.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b7a672c82b5f9887576087d97be5ce439f04bbaf548ee987b92d3a7dede41d3a", size = 11149151, upload-time = "2026-02-19T22:32:44.234Z" }, + { url = "https://files.pythonhosted.org/packages/70/04/f5284e388bab60d1d3b99614a5a9aeb03e0f333847e2429bebd2aaa1feec/ruff-0.15.2-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:72ecc64f46f7019e2bcc3cdc05d4a7da958b629a5ab7033195e11a438403d956", size = 11091132, upload-time = "2026-02-19T22:32:24.691Z" }, + { url = "https://files.pythonhosted.org/packages/fa/ae/88d844a21110e14d92cf73d57363fab59b727ebeabe78009b9ccb23500af/ruff-0.15.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:8dcf243b15b561c655c1ef2f2b0050e5d50db37fe90115507f6ff37d865dc8b4", size = 10504717, upload-time = "2026-02-19T22:32:26.75Z" }, + { url = "https://files.pythonhosted.org/packages/64/27/867076a6ada7f2b9c8292884ab44d08fd2ba71bd2b5364d4136f3cd537e1/ruff-0.15.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:dab6941c862c05739774677c6273166d2510d254dac0695c0e3f5efa1b5585de", size = 10263122, upload-time = "2026-02-19T22:32:10.036Z" }, + { url = "https://files.pythonhosted.org/packages/e7/ef/faf9321d550f8ebf0c6373696e70d1758e20ccdc3951ad7af00c0956be7c/ruff-0.15.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:1b9164f57fc36058e9a6806eb92af185b0697c9fe4c7c52caa431c6554521e5c", size = 10735295, upload-time = "2026-02-19T22:32:39.227Z" }, + { url = "https://files.pythonhosted.org/packages/2f/55/e8089fec62e050ba84d71b70e7834b97709ca9b7aba10c1a0b196e493f97/ruff-0.15.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:80d24fcae24d42659db7e335b9e1531697a7102c19185b8dc4a028b952865fd8", size = 11241641, upload-time = "2026-02-19T22:32:34.617Z" }, + { url = "https://files.pythonhosted.org/packages/23/01/1c30526460f4d23222d0fabd5888868262fd0e2b71a00570ca26483cd993/ruff-0.15.2-py3-none-win32.whl", hash = "sha256:fd5ff9e5f519a7e1bd99cbe8daa324010a74f5e2ebc97c6242c08f26f3714f6f", size = 10507885, upload-time = "2026-02-19T22:32:15.635Z" }, + { url = "https://files.pythonhosted.org/packages/5c/10/3d18e3bbdf8fc50bbb4ac3cc45970aa5a9753c5cb51bf9ed9a3cd8b79fa3/ruff-0.15.2-py3-none-win_amd64.whl", hash = "sha256:d20014e3dfa400f3ff84830dfb5755ece2de45ab62ecea4af6b7262d0fb4f7c5", size = 11623725, upload-time = "2026-02-19T22:32:04.947Z" }, + { url = "https://files.pythonhosted.org/packages/6d/78/097c0798b1dab9f8affe73da9642bb4500e098cb27fd8dc9724816ac747b/ruff-0.15.2-py3-none-win_arm64.whl", hash = "sha256:cabddc5822acdc8f7b5527b36ceac55cc51eec7b1946e60181de8fe83ca8876e", size = 10941649, upload-time = "2026-02-19T22:32:18.108Z" }, +] + [[package]] name = "rustac" version = "0.9.3" @@ -782,6 +994,20 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/06/aa/9ce0f3e7a9829ead5c8ce549392f33a12c4555a6c0609bb27d882e9c7ddf/sqlalchemy-2.0.46.tar.gz", hash = "sha256:cf36851ee7219c170bb0793dbc3da3e80c582e04a5437bc601bfe8c85c9216d7", size = 9865393, upload-time = "2026-01-21T18:03:45.119Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/69/ac/b42ad16800d0885105b59380ad69aad0cce5a65276e269ce2729a2343b6a/sqlalchemy-2.0.46-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:261c4b1f101b4a411154f1da2b76497d73abbfc42740029205d4d01fa1052684", size = 2154851, upload-time = "2026-01-21T18:27:30.54Z" }, + { url = "https://files.pythonhosted.org/packages/a0/60/d8710068cb79f64d002ebed62a7263c00c8fd95f4ebd4b5be8f7ca93f2bc/sqlalchemy-2.0.46-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:181903fe8c1b9082995325f1b2e84ac078b1189e2819380c2303a5f90e114a62", size = 3311241, upload-time = "2026-01-21T18:32:33.45Z" }, + { url = "https://files.pythonhosted.org/packages/2b/0f/20c71487c7219ab3aa7421c7c62d93824c97c1460f2e8bb72404b0192d13/sqlalchemy-2.0.46-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:590be24e20e2424a4c3c1b0835e9405fa3d0af5823a1a9fc02e5dff56471515f", size = 3310741, upload-time = "2026-01-21T18:44:57.887Z" }, + { url = "https://files.pythonhosted.org/packages/65/80/d26d00b3b249ae000eee4db206fcfc564bf6ca5030e4747adf451f4b5108/sqlalchemy-2.0.46-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7568fe771f974abadce52669ef3a03150ff03186d8eb82613bc8adc435a03f01", size = 3263116, upload-time = "2026-01-21T18:32:35.044Z" }, + { url = "https://files.pythonhosted.org/packages/da/ee/74dda7506640923821340541e8e45bd3edd8df78664f1f2e0aae8077192b/sqlalchemy-2.0.46-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ebf7e1e78af38047e08836d33502c7a278915698b7c2145d045f780201679999", size = 3285327, upload-time = "2026-01-21T18:44:59.254Z" }, + { url = "https://files.pythonhosted.org/packages/9f/25/6dcf8abafff1389a21c7185364de145107b7394ecdcb05233815b236330d/sqlalchemy-2.0.46-cp311-cp311-win32.whl", hash = "sha256:9d80ea2ac519c364a7286e8d765d6cd08648f5b21ca855a8017d9871f075542d", size = 2114564, upload-time = "2026-01-21T18:33:15.85Z" }, + { url = "https://files.pythonhosted.org/packages/93/5f/e081490f8523adc0088f777e4ebad3cac21e498ec8a3d4067074e21447a1/sqlalchemy-2.0.46-cp311-cp311-win_amd64.whl", hash = "sha256:585af6afe518732d9ccd3aea33af2edaae4a7aa881af5d8f6f4fe3a368699597", size = 2139233, upload-time = "2026-01-21T18:33:17.528Z" }, + { url = "https://files.pythonhosted.org/packages/b6/35/d16bfa235c8b7caba3730bba43e20b1e376d2224f407c178fbf59559f23e/sqlalchemy-2.0.46-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3a9a72b0da8387f15d5810f1facca8f879de9b85af8c645138cba61ea147968c", size = 2153405, upload-time = "2026-01-21T19:05:54.143Z" }, + { url = "https://files.pythonhosted.org/packages/06/6c/3192e24486749862f495ddc6584ed730c0c994a67550ec395d872a2ad650/sqlalchemy-2.0.46-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2347c3f0efc4de367ba00218e0ae5c4ba2306e47216ef80d6e31761ac97cb0b9", size = 3334702, upload-time = "2026-01-21T18:46:45.384Z" }, + { url = "https://files.pythonhosted.org/packages/ea/a2/b9f33c8d68a3747d972a0bb758c6b63691f8fb8a49014bc3379ba15d4274/sqlalchemy-2.0.46-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9094c8b3197db12aa6f05c51c05daaad0a92b8c9af5388569847b03b1007fb1b", size = 3347664, upload-time = "2026-01-21T18:40:09.979Z" }, + { url = "https://files.pythonhosted.org/packages/aa/d2/3e59e2a91eaec9db7e8dc6b37b91489b5caeb054f670f32c95bcba98940f/sqlalchemy-2.0.46-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37fee2164cf21417478b6a906adc1a91d69ae9aba8f9533e67ce882f4bb1de53", size = 3277372, upload-time = "2026-01-21T18:46:47.168Z" }, + { url = "https://files.pythonhosted.org/packages/dd/dd/67bc2e368b524e2192c3927b423798deda72c003e73a1e94c21e74b20a85/sqlalchemy-2.0.46-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b1e14b2f6965a685c7128bd315e27387205429c2e339eeec55cb75ca4ab0ea2e", size = 3312425, upload-time = "2026-01-21T18:40:11.548Z" }, + { url = "https://files.pythonhosted.org/packages/43/82/0ecd68e172bfe62247e96cb47867c2d68752566811a4e8c9d8f6e7c38a65/sqlalchemy-2.0.46-cp312-cp312-win32.whl", hash = "sha256:412f26bb4ba942d52016edc8d12fb15d91d3cd46b0047ba46e424213ad407bcb", size = 2113155, upload-time = "2026-01-21T18:42:49.748Z" }, + { url = "https://files.pythonhosted.org/packages/bc/2a/2821a45742073fc0331dc132552b30de68ba9563230853437cac54b2b53e/sqlalchemy-2.0.46-cp312-cp312-win_amd64.whl", hash = "sha256:ea3cd46b6713a10216323cda3333514944e510aa691c945334713fca6b5279ff", size = 2140078, upload-time = "2026-01-21T18:42:51.197Z" }, { url = "https://files.pythonhosted.org/packages/b3/4b/fa7838fe20bb752810feed60e45625a9a8b0102c0c09971e2d1d95362992/sqlalchemy-2.0.46-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:93a12da97cca70cea10d4b4fc602589c4511f96c1f8f6c11817620c021d21d00", size = 2150268, upload-time = "2026-01-21T19:05:56.621Z" }, { url = "https://files.pythonhosted.org/packages/46/c1/b34dccd712e8ea846edf396e00973dda82d598cb93762e55e43e6835eba9/sqlalchemy-2.0.46-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:af865c18752d416798dae13f83f38927c52f085c52e2f32b8ab0fef46fdd02c2", size = 3276511, upload-time = "2026-01-21T18:46:49.022Z" }, { url = "https://files.pythonhosted.org/packages/96/48/a04d9c94753e5d5d096c628c82a98c4793b9c08ca0e7155c3eb7d7db9f24/sqlalchemy-2.0.46-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8d679b5f318423eacb61f933a9a0f75535bfca7056daeadbf6bd5bcee6183aee", size = 3292881, upload-time = "2026-01-21T18:40:13.089Z" }, @@ -842,6 +1068,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/30/643397144bfbfec6f6ef821f36f33e57d35946c44a2352d3c9f0ae847619/tenacity-9.1.2-py3-none-any.whl", hash = "sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138", size = 28248, upload-time = "2025-04-02T08:25:07.678Z" }, ] +[[package]] +name = "ty" +version = "0.0.18" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/74/15/9682700d8d60fdca7afa4febc83a2354b29cdcd56e66e19c92b521db3b39/ty-0.0.18.tar.gz", hash = "sha256:04ab7c3db5dcbcdac6ce62e48940d3a0124f377c05499d3f3e004e264ae94b83", size = 5214774, upload-time = "2026-02-20T21:51:31.173Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/d8/920460d4c22ea68fcdeb0b2fb53ea2aeb9c6d7875bde9278d84f2ac767b6/ty-0.0.18-py3-none-linux_armv6l.whl", hash = "sha256:4e5e91b0a79857316ef893c5068afc4b9872f9d257627d9bc8ac4d2715750d88", size = 10280825, upload-time = "2026-02-20T21:51:25.03Z" }, + { url = "https://files.pythonhosted.org/packages/83/56/62587de582d3d20d78fcdddd0594a73822ac5a399a12ef512085eb7a4de6/ty-0.0.18-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ee0e578b3f8416e2d5416da9553b78fd33857868aa1384cb7fefeceee5ff102d", size = 10118324, upload-time = "2026-02-20T21:51:22.27Z" }, + { url = "https://files.pythonhosted.org/packages/2f/2d/dbdace8d432a0755a7417f659bfd5b8a4261938ecbdfd7b42f4c454f5aa9/ty-0.0.18-py3-none-macosx_11_0_arm64.whl", hash = "sha256:3f7a0487d36b939546a91d141f7fc3dbea32fab4982f618d5b04dc9d5b6da21e", size = 9605861, upload-time = "2026-02-20T21:51:16.066Z" }, + { url = "https://files.pythonhosted.org/packages/6b/d9/de11c0280f778d5fc571393aada7fe9b8bc1dd6a738f2e2c45702b8b3150/ty-0.0.18-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5e2fa8d45f57ca487a470e4bf66319c09b561150e98ae2a6b1a97ef04c1a4eb", size = 10092701, upload-time = "2026-02-20T21:51:26.862Z" }, + { url = "https://files.pythonhosted.org/packages/0f/94/068d4d591d791041732171e7b63c37a54494b2e7d28e88d2167eaa9ad875/ty-0.0.18-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d75652e9e937f7044b1aca16091193e7ef11dac1c7ec952b7fb8292b7ba1f5f2", size = 10109203, upload-time = "2026-02-20T21:51:11.59Z" }, + { url = "https://files.pythonhosted.org/packages/34/e4/526a4aa56dc0ca2569aaa16880a1ab105c3b416dd70e87e25a05688999f3/ty-0.0.18-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:563c868edceb8f6ddd5e91113c17d3676b028f0ed380bdb3829b06d9beb90e58", size = 10614200, upload-time = "2026-02-20T21:51:20.298Z" }, + { url = "https://files.pythonhosted.org/packages/fd/3d/b68ab20a34122a395880922587fbfc3adf090d22e0fb546d4d20fe8c2621/ty-0.0.18-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:502e2a1f948bec563a0454fc25b074bf5cf041744adba8794d024277e151d3b0", size = 11153232, upload-time = "2026-02-20T21:51:14.121Z" }, + { url = "https://files.pythonhosted.org/packages/68/ea/678243c042343fcda7e6af36036c18676c355878dcdcd517639586d2cf9e/ty-0.0.18-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cc881dea97021a3aa29134a476937fd8054775c4177d01b94db27fcfb7aab65b", size = 10832934, upload-time = "2026-02-20T21:51:32.92Z" }, + { url = "https://files.pythonhosted.org/packages/d8/bd/7f8d647cef8b7b346c0163230a37e903c7461c7248574840b977045c77df/ty-0.0.18-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:421fcc3bc64cab56f48edb863c7c1c43649ec4d78ff71a1acb5366ad723b6021", size = 10700888, upload-time = "2026-02-20T21:51:09.673Z" }, + { url = "https://files.pythonhosted.org/packages/6e/06/cb3620dc48c5d335ba7876edfef636b2f4498eff4a262ff90033b9e88408/ty-0.0.18-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:0fe5038a7136a0e638a2fb1ad06e3d3c4045314c6ba165c9c303b9aeb4623d6c", size = 10078965, upload-time = "2026-02-20T21:51:07.678Z" }, + { url = "https://files.pythonhosted.org/packages/60/27/c77a5a84533fa3b685d592de7b4b108eb1f38851c40fac4e79cc56ec7350/ty-0.0.18-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:d123600a52372677613a719bbb780adeb9b68f47fb5f25acb09171de390e0035", size = 10134659, upload-time = "2026-02-20T21:51:18.311Z" }, + { url = "https://files.pythonhosted.org/packages/43/6e/60af6b88c73469e628ba5253a296da6984e0aa746206f3034c31f1a04ed1/ty-0.0.18-py3-none-musllinux_1_2_i686.whl", hash = "sha256:bb4bc11d32a1bf96a829bf6b9696545a30a196ac77bbc07cc8d3dfee35e03723", size = 10297494, upload-time = "2026-02-20T21:51:39.631Z" }, + { url = "https://files.pythonhosted.org/packages/33/90/612dc0b68224c723faed6adac2bd3f930a750685db76dfe17e6b9e534a83/ty-0.0.18-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:dda2efbf374ba4cd704053d04e32f2f784e85c2ddc2400006b0f96f5f7e4b667", size = 10791944, upload-time = "2026-02-20T21:51:37.13Z" }, + { url = "https://files.pythonhosted.org/packages/0d/da/f4ada0fd08a9e4138fe3fd2bcd3797753593f423f19b1634a814b9b2a401/ty-0.0.18-py3-none-win32.whl", hash = "sha256:c5768607c94977dacddc2f459ace6a11a408a0f57888dd59abb62d28d4fee4f7", size = 9677964, upload-time = "2026-02-20T21:51:42.039Z" }, + { url = "https://files.pythonhosted.org/packages/5e/fa/090ed9746e5c59fc26d8f5f96dc8441825171f1f47752f1778dad690b08b/ty-0.0.18-py3-none-win_amd64.whl", hash = "sha256:b78d0fa1103d36fc2fce92f2092adace52a74654ab7884d54cdaec8eb5016a4d", size = 10636576, upload-time = "2026-02-20T21:51:29.159Z" }, + { url = "https://files.pythonhosted.org/packages/92/4f/5dd60904c8105cda4d0be34d3a446c180933c76b84ae0742e58f02133713/ty-0.0.18-py3-none-win_arm64.whl", hash = "sha256:01770c3c82137c6b216aa3251478f0b197e181054ee92243772de553d3586398", size = 10095449, upload-time = "2026-02-20T21:51:34.914Z" }, +] + [[package]] name = "typeguard" version = "2.13.3" From f3aaebf21eb2f2cc7a2c48e47230f59252a3f8c5 Mon Sep 17 00:00:00 2001 From: hrodmn Date: Thu, 26 Feb 2026 12:02:28 -0600 Subject: [PATCH 05/23] chore: fix test warnings --- .github/workflows/ci.yml | 4 + pyproject.toml | 16 ++++ src/icestac/__init__.py | 3 +- tests/conftest.py | 6 +- uv.lock | 174 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 200 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ddeedf9..7f6a5f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,10 @@ on: jobs: tests: + strategy: + matrix: + python-version: [3.11, 3.12, 3.13, 3.14] + runs-on: ubuntu-latest steps: diff --git a/pyproject.toml b/pyproject.toml index 463109e..afd9d4f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,16 @@ readme = "README.md" authors = [ { name = "hrodmn", email = "henry@developmentseed.org" } ] +classifiers = [ + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] requires-python = ">=3.11" dependencies = [ "pyarrow>=23.0.0", @@ -28,12 +38,18 @@ deploy = [ ] dev = [ "pytest>=9.0.2", + "pytest-cov>=7.0.0", "ruff>=0.15.2", "sqlalchemy>=2.0.46", "ty>=0.0.18", ] [tool.pytest.ini_options] +addopts = [ + "-v", + "--cov-config=pyproject.toml", + "--cov=src" +] filterwarnings = [ "ignore::DeprecationWarning:pyiceberg.*", "ignore::pydantic.PydanticDeprecatedSince20", diff --git a/src/icestac/__init__.py b/src/icestac/__init__.py index a570042..646031a 100644 --- a/src/icestac/__init__.py +++ b/src/icestac/__init__.py @@ -1,2 +1 @@ -def main() -> None: - print("Hello from icestac!") +"""icestac""" diff --git a/tests/conftest.py b/tests/conftest.py index 9e15541..d8ab3c6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,4 @@ +import gc import tempfile from pathlib import Path from typing import Any @@ -30,7 +31,10 @@ def test_catalog(temp_warehouse): # Create test namespace catalog.create_namespace("test_namespace") - return catalog + yield catalog + + gc.collect() + catalog.close() @pytest.fixture diff --git a/uv.lock b/uv.lock index d914d9d..b5cd488 100644 --- a/uv.lock +++ b/uv.lock @@ -256,6 +256,110 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cd/a1/0cbba03ce3c1377a788192163f2614f396a8614cf5022ac54bdc2085c078/constructs-10.4.5-py3-none-any.whl", hash = "sha256:e63d6675ba2e8a9076db8df1d4c7af78efdb96182ba28abe938384ba321e4b81", size = 63037, upload-time = "2026-01-16T16:09:09.178Z" }, ] +[[package]] +name = "coverage" +version = "7.13.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/56/95b7e30fa389756cb56630faa728da46a27b8c6eb46f9d557c68fff12b65/coverage-7.13.4.tar.gz", hash = "sha256:e5c8f6ed1e61a8b2dcdf31eb0b9bbf0130750ca79c1c49eb898e2ad86f5ccc91", size = 827239, upload-time = "2026-02-09T12:59:03.86Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/ad/b59e5b451cf7172b8d1043dc0fa718f23aab379bc1521ee13d4bd9bfa960/coverage-7.13.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d490ba50c3f35dd7c17953c68f3270e7ccd1c6642e2d2afe2d8e720b98f5a053", size = 219278, upload-time = "2026-02-09T12:56:31.673Z" }, + { url = "https://files.pythonhosted.org/packages/f1/17/0cb7ca3de72e5f4ef2ec2fa0089beafbcaaaead1844e8b8a63d35173d77d/coverage-7.13.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:19bc3c88078789f8ef36acb014d7241961dbf883fd2533d18cb1e7a5b4e28b11", size = 219783, upload-time = "2026-02-09T12:56:33.104Z" }, + { url = "https://files.pythonhosted.org/packages/ab/63/325d8e5b11e0eaf6d0f6a44fad444ae58820929a9b0de943fa377fe73e85/coverage-7.13.4-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3998e5a32e62fdf410c0dbd3115df86297995d6e3429af80b8798aad894ca7aa", size = 250200, upload-time = "2026-02-09T12:56:34.474Z" }, + { url = "https://files.pythonhosted.org/packages/76/53/c16972708cbb79f2942922571a687c52bd109a7bd51175aeb7558dff2236/coverage-7.13.4-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8e264226ec98e01a8e1054314af91ee6cde0eacac4f465cc93b03dbe0bce2fd7", size = 252114, upload-time = "2026-02-09T12:56:35.749Z" }, + { url = "https://files.pythonhosted.org/packages/eb/c2/7ab36d8b8cc412bec9ea2d07c83c48930eb4ba649634ba00cb7e4e0f9017/coverage-7.13.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3aa4e7b9e416774b21797365b358a6e827ffadaaca81b69ee02946852449f00", size = 254220, upload-time = "2026-02-09T12:56:37.796Z" }, + { url = "https://files.pythonhosted.org/packages/d6/4d/cf52c9a3322c89a0e6febdfbc83bb45c0ed3c64ad14081b9503adee702e7/coverage-7.13.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:71ca20079dd8f27fcf808817e281e90220475cd75115162218d0e27549f95fef", size = 256164, upload-time = "2026-02-09T12:56:39.016Z" }, + { url = "https://files.pythonhosted.org/packages/78/e9/eb1dd17bd6de8289df3580e967e78294f352a5df8a57ff4671ee5fc3dcd0/coverage-7.13.4-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e2f25215f1a359ab17320b47bcdaca3e6e6356652e8256f2441e4ef972052903", size = 250325, upload-time = "2026-02-09T12:56:40.668Z" }, + { url = "https://files.pythonhosted.org/packages/71/07/8c1542aa873728f72267c07278c5cc0ec91356daf974df21335ccdb46368/coverage-7.13.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d65b2d373032411e86960604dc4edac91fdfb5dca539461cf2cbe78327d1e64f", size = 251913, upload-time = "2026-02-09T12:56:41.97Z" }, + { url = "https://files.pythonhosted.org/packages/74/d7/c62e2c5e4483a748e27868e4c32ad3daa9bdddbba58e1bc7a15e252baa74/coverage-7.13.4-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:94eb63f9b363180aff17de3e7c8760c3ba94664ea2695c52f10111244d16a299", size = 249974, upload-time = "2026-02-09T12:56:43.323Z" }, + { url = "https://files.pythonhosted.org/packages/98/9f/4c5c015a6e98ced54efd0f5cf8d31b88e5504ecb6857585fc0161bb1e600/coverage-7.13.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:e856bf6616714c3a9fbc270ab54103f4e685ba236fa98c054e8f87f266c93505", size = 253741, upload-time = "2026-02-09T12:56:45.155Z" }, + { url = "https://files.pythonhosted.org/packages/bd/59/0f4eef89b9f0fcd9633b5d350016f54126ab49426a70ff4c4e87446cabdc/coverage-7.13.4-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:65dfcbe305c3dfe658492df2d85259e0d79ead4177f9ae724b6fb245198f55d6", size = 249695, upload-time = "2026-02-09T12:56:46.636Z" }, + { url = "https://files.pythonhosted.org/packages/b5/2c/b7476f938deb07166f3eb281a385c262675d688ff4659ad56c6c6b8e2e70/coverage-7.13.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b507778ae8a4c915436ed5c2e05b4a6cecfa70f734e19c22a005152a11c7b6a9", size = 250599, upload-time = "2026-02-09T12:56:48.13Z" }, + { url = "https://files.pythonhosted.org/packages/b8/34/c3420709d9846ee3785b9f2831b4d94f276f38884032dca1457fa83f7476/coverage-7.13.4-cp311-cp311-win32.whl", hash = "sha256:784fc3cf8be001197b652d51d3fd259b1e2262888693a4636e18879f613a62a9", size = 221780, upload-time = "2026-02-09T12:56:50.479Z" }, + { url = "https://files.pythonhosted.org/packages/61/08/3d9c8613079d2b11c185b865de9a4c1a68850cfda2b357fae365cf609f29/coverage-7.13.4-cp311-cp311-win_amd64.whl", hash = "sha256:2421d591f8ca05b308cf0092807308b2facbefe54af7c02ac22548b88b95c98f", size = 222715, upload-time = "2026-02-09T12:56:51.815Z" }, + { url = "https://files.pythonhosted.org/packages/18/1a/54c3c80b2f056164cc0a6cdcb040733760c7c4be9d780fe655f356f433e4/coverage-7.13.4-cp311-cp311-win_arm64.whl", hash = "sha256:79e73a76b854d9c6088fe5d8b2ebe745f8681c55f7397c3c0a016192d681045f", size = 221385, upload-time = "2026-02-09T12:56:53.194Z" }, + { url = "https://files.pythonhosted.org/packages/d1/81/4ce2fdd909c5a0ed1f6dedb88aa57ab79b6d1fbd9b588c1ac7ef45659566/coverage-7.13.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:02231499b08dabbe2b96612993e5fc34217cdae907a51b906ac7fca8027a4459", size = 219449, upload-time = "2026-02-09T12:56:54.889Z" }, + { url = "https://files.pythonhosted.org/packages/5d/96/5238b1efc5922ddbdc9b0db9243152c09777804fb7c02ad1741eb18a11c0/coverage-7.13.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40aa8808140e55dc022b15d8aa7f651b6b3d68b365ea0398f1441e0b04d859c3", size = 219810, upload-time = "2026-02-09T12:56:56.33Z" }, + { url = "https://files.pythonhosted.org/packages/78/72/2f372b726d433c9c35e56377cf1d513b4c16fe51841060d826b95caacec1/coverage-7.13.4-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5b856a8ccf749480024ff3bd7310adaef57bf31fd17e1bfc404b7940b6986634", size = 251308, upload-time = "2026-02-09T12:56:57.858Z" }, + { url = "https://files.pythonhosted.org/packages/5d/a0/2ea570925524ef4e00bb6c82649f5682a77fac5ab910a65c9284de422600/coverage-7.13.4-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2c048ea43875fbf8b45d476ad79f179809c590ec7b79e2035c662e7afa3192e3", size = 254052, upload-time = "2026-02-09T12:56:59.754Z" }, + { url = "https://files.pythonhosted.org/packages/e8/ac/45dc2e19a1939098d783c846e130b8f862fbb50d09e0af663988f2f21973/coverage-7.13.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b7b38448866e83176e28086674fe7368ab8590e4610fb662b44e345b86d63ffa", size = 255165, upload-time = "2026-02-09T12:57:01.287Z" }, + { url = "https://files.pythonhosted.org/packages/2d/4d/26d236ff35abc3b5e63540d3386e4c3b192168c1d96da5cb2f43c640970f/coverage-7.13.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:de6defc1c9badbf8b9e67ae90fd00519186d6ab64e5cc5f3d21359c2a9b2c1d3", size = 257432, upload-time = "2026-02-09T12:57:02.637Z" }, + { url = "https://files.pythonhosted.org/packages/ec/55/14a966c757d1348b2e19caf699415a2a4c4f7feaa4bbc6326a51f5c7dd1b/coverage-7.13.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7eda778067ad7ffccd23ecffce537dface96212576a07924cbf0d8799d2ded5a", size = 251716, upload-time = "2026-02-09T12:57:04.056Z" }, + { url = "https://files.pythonhosted.org/packages/77/33/50116647905837c66d28b2af1321b845d5f5d19be9655cb84d4a0ea806b4/coverage-7.13.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e87f6c587c3f34356c3759f0420693e35e7eb0e2e41e4c011cb6ec6ecbbf1db7", size = 253089, upload-time = "2026-02-09T12:57:05.503Z" }, + { url = "https://files.pythonhosted.org/packages/c2/b4/8efb11a46e3665d92635a56e4f2d4529de6d33f2cb38afd47d779d15fc99/coverage-7.13.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:8248977c2e33aecb2ced42fef99f2d319e9904a36e55a8a68b69207fb7e43edc", size = 251232, upload-time = "2026-02-09T12:57:06.879Z" }, + { url = "https://files.pythonhosted.org/packages/51/24/8cd73dd399b812cc76bb0ac260e671c4163093441847ffe058ac9fda1e32/coverage-7.13.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:25381386e80ae727608e662474db537d4df1ecd42379b5ba33c84633a2b36d47", size = 255299, upload-time = "2026-02-09T12:57:08.245Z" }, + { url = "https://files.pythonhosted.org/packages/03/94/0a4b12f1d0e029ce1ccc1c800944a9984cbe7d678e470bb6d3c6bc38a0da/coverage-7.13.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:ee756f00726693e5ba94d6df2bdfd64d4852d23b09bb0bc700e3b30e6f333985", size = 250796, upload-time = "2026-02-09T12:57:10.142Z" }, + { url = "https://files.pythonhosted.org/packages/73/44/6002fbf88f6698ca034360ce474c406be6d5a985b3fdb3401128031eef6b/coverage-7.13.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fdfc1e28e7c7cdce44985b3043bc13bbd9c747520f94a4d7164af8260b3d91f0", size = 252673, upload-time = "2026-02-09T12:57:12.197Z" }, + { url = "https://files.pythonhosted.org/packages/de/c6/a0279f7c00e786be75a749a5674e6fa267bcbd8209cd10c9a450c655dfa7/coverage-7.13.4-cp312-cp312-win32.whl", hash = "sha256:01d4cbc3c283a17fc1e42d614a119f7f438eabb593391283adca8dc86eff1246", size = 221990, upload-time = "2026-02-09T12:57:14.085Z" }, + { url = "https://files.pythonhosted.org/packages/77/4e/c0a25a425fcf5557d9abd18419c95b63922e897bc86c1f327f155ef234a9/coverage-7.13.4-cp312-cp312-win_amd64.whl", hash = "sha256:9401ebc7ef522f01d01d45532c68c5ac40fb27113019b6b7d8b208f6e9baa126", size = 222800, upload-time = "2026-02-09T12:57:15.944Z" }, + { url = "https://files.pythonhosted.org/packages/47/ac/92da44ad9a6f4e3a7debd178949d6f3769bedca33830ce9b1dcdab589a37/coverage-7.13.4-cp312-cp312-win_arm64.whl", hash = "sha256:b1ec7b6b6e93255f952e27ab58fbc68dcc468844b16ecbee881aeb29b6ab4d8d", size = 221415, upload-time = "2026-02-09T12:57:17.497Z" }, + { url = "https://files.pythonhosted.org/packages/db/23/aad45061a31677d68e47499197a131eea55da4875d16c1f42021ab963503/coverage-7.13.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b66a2da594b6068b48b2692f043f35d4d3693fb639d5ea8b39533c2ad9ac3ab9", size = 219474, upload-time = "2026-02-09T12:57:19.332Z" }, + { url = "https://files.pythonhosted.org/packages/a5/70/9b8b67a0945f3dfec1fd896c5cefb7c19d5a3a6d74630b99a895170999ae/coverage-7.13.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3599eb3992d814d23b35c536c28df1a882caa950f8f507cef23d1cbf334995ac", size = 219844, upload-time = "2026-02-09T12:57:20.66Z" }, + { url = "https://files.pythonhosted.org/packages/97/fd/7e859f8fab324cef6c4ad7cff156ca7c489fef9179d5749b0c8d321281c2/coverage-7.13.4-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:93550784d9281e374fb5a12bf1324cc8a963fd63b2d2f223503ef0fd4aa339ea", size = 250832, upload-time = "2026-02-09T12:57:22.007Z" }, + { url = "https://files.pythonhosted.org/packages/e4/dc/b2442d10020c2f52617828862d8b6ee337859cd8f3a1f13d607dddda9cf7/coverage-7.13.4-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b720ce6a88a2755f7c697c23268ddc47a571b88052e6b155224347389fdf6a3b", size = 253434, upload-time = "2026-02-09T12:57:23.339Z" }, + { url = "https://files.pythonhosted.org/packages/5a/88/6728a7ad17428b18d836540630487231f5470fb82454871149502f5e5aa2/coverage-7.13.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7b322db1284a2ed3aa28ffd8ebe3db91c929b7a333c0820abec3d838ef5b3525", size = 254676, upload-time = "2026-02-09T12:57:24.774Z" }, + { url = "https://files.pythonhosted.org/packages/7c/bc/21244b1b8cedf0dff0a2b53b208015fe798d5f2a8d5348dbfece04224fff/coverage-7.13.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f4594c67d8a7c89cf922d9df0438c7c7bb022ad506eddb0fdb2863359ff78242", size = 256807, upload-time = "2026-02-09T12:57:26.125Z" }, + { url = "https://files.pythonhosted.org/packages/97/a0/ddba7ed3251cff51006737a727d84e05b61517d1784a9988a846ba508877/coverage-7.13.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:53d133df809c743eb8bce33b24bcababb371f4441340578cd406e084d94a6148", size = 251058, upload-time = "2026-02-09T12:57:27.614Z" }, + { url = "https://files.pythonhosted.org/packages/9b/55/e289addf7ff54d3a540526f33751951bf0878f3809b47f6dfb3def69c6f7/coverage-7.13.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:76451d1978b95ba6507a039090ba076105c87cc76fc3efd5d35d72093964d49a", size = 252805, upload-time = "2026-02-09T12:57:29.066Z" }, + { url = "https://files.pythonhosted.org/packages/13/4e/cc276b1fa4a59be56d96f1dabddbdc30f4ba22e3b1cd42504c37b3313255/coverage-7.13.4-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:7f57b33491e281e962021de110b451ab8a24182589be17e12a22c79047935e23", size = 250766, upload-time = "2026-02-09T12:57:30.522Z" }, + { url = "https://files.pythonhosted.org/packages/94/44/1093b8f93018f8b41a8cf29636c9292502f05e4a113d4d107d14a3acd044/coverage-7.13.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:1731dc33dc276dafc410a885cbf5992f1ff171393e48a21453b78727d090de80", size = 254923, upload-time = "2026-02-09T12:57:31.946Z" }, + { url = "https://files.pythonhosted.org/packages/8b/55/ea2796da2d42257f37dbea1aab239ba9263b31bd91d5527cdd6db5efe174/coverage-7.13.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:bd60d4fe2f6fa7dff9223ca1bbc9f05d2b6697bc5961072e5d3b952d46e1b1ea", size = 250591, upload-time = "2026-02-09T12:57:33.842Z" }, + { url = "https://files.pythonhosted.org/packages/d4/fa/7c4bb72aacf8af5020675aa633e59c1fbe296d22aed191b6a5b711eb2bc7/coverage-7.13.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9181a3ccead280b828fae232df12b16652702b49d41e99d657f46cc7b1f6ec7a", size = 252364, upload-time = "2026-02-09T12:57:35.743Z" }, + { url = "https://files.pythonhosted.org/packages/5c/38/a8d2ec0146479c20bbaa7181b5b455a0c41101eed57f10dd19a78ab44c80/coverage-7.13.4-cp313-cp313-win32.whl", hash = "sha256:f53d492307962561ac7de4cd1de3e363589b000ab69617c6156a16ba7237998d", size = 222010, upload-time = "2026-02-09T12:57:37.25Z" }, + { url = "https://files.pythonhosted.org/packages/e2/0c/dbfafbe90a185943dcfbc766fe0e1909f658811492d79b741523a414a6cc/coverage-7.13.4-cp313-cp313-win_amd64.whl", hash = "sha256:e6f70dec1cc557e52df5306d051ef56003f74d56e9c4dd7ddb07e07ef32a84dd", size = 222818, upload-time = "2026-02-09T12:57:38.734Z" }, + { url = "https://files.pythonhosted.org/packages/04/d1/934918a138c932c90d78301f45f677fb05c39a3112b96fd2c8e60503cdc7/coverage-7.13.4-cp313-cp313-win_arm64.whl", hash = "sha256:fb07dc5da7e849e2ad31a5d74e9bece81f30ecf5a42909d0a695f8bd1874d6af", size = 221438, upload-time = "2026-02-09T12:57:40.223Z" }, + { url = "https://files.pythonhosted.org/packages/52/57/ee93ced533bcb3e6df961c0c6e42da2fc6addae53fb95b94a89b1e33ebd7/coverage-7.13.4-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:40d74da8e6c4b9ac18b15331c4b5ebc35a17069410cad462ad4f40dcd2d50c0d", size = 220165, upload-time = "2026-02-09T12:57:41.639Z" }, + { url = "https://files.pythonhosted.org/packages/c5/e0/969fc285a6fbdda49d91af278488d904dcd7651b2693872f0ff94e40e84a/coverage-7.13.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4223b4230a376138939a9173f1bdd6521994f2aff8047fae100d6d94d50c5a12", size = 220516, upload-time = "2026-02-09T12:57:44.215Z" }, + { url = "https://files.pythonhosted.org/packages/b1/b8/9531944e16267e2735a30a9641ff49671f07e8138ecf1ca13db9fd2560c7/coverage-7.13.4-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:1d4be36a5114c499f9f1f9195e95ebf979460dbe2d88e6816ea202010ba1c34b", size = 261804, upload-time = "2026-02-09T12:57:45.989Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f3/e63df6d500314a2a60390d1989240d5f27318a7a68fa30ad3806e2a9323e/coverage-7.13.4-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:200dea7d1e8095cc6e98cdabe3fd1d21ab17d3cee6dab00cadbb2fe35d9c15b9", size = 263885, upload-time = "2026-02-09T12:57:47.42Z" }, + { url = "https://files.pythonhosted.org/packages/f3/67/7654810de580e14b37670b60a09c599fa348e48312db5b216d730857ffe6/coverage-7.13.4-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b8eb931ee8e6d8243e253e5ed7336deea6904369d2fd8ae6e43f68abbf167092", size = 266308, upload-time = "2026-02-09T12:57:49.345Z" }, + { url = "https://files.pythonhosted.org/packages/37/6f/39d41eca0eab3cc82115953ad41c4e77935286c930e8fad15eaed1389d83/coverage-7.13.4-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:75eab1ebe4f2f64d9509b984f9314d4aa788540368218b858dad56dc8f3e5eb9", size = 267452, upload-time = "2026-02-09T12:57:50.811Z" }, + { url = "https://files.pythonhosted.org/packages/50/6d/39c0fbb8fc5cd4d2090811e553c2108cf5112e882f82505ee7495349a6bf/coverage-7.13.4-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c35eb28c1d085eb7d8c9b3296567a1bebe03ce72962e932431b9a61f28facf26", size = 261057, upload-time = "2026-02-09T12:57:52.447Z" }, + { url = "https://files.pythonhosted.org/packages/a4/a2/60010c669df5fa603bb5a97fb75407e191a846510da70ac657eb696b7fce/coverage-7.13.4-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:eb88b316ec33760714a4720feb2816a3a59180fd58c1985012054fa7aebee4c2", size = 263875, upload-time = "2026-02-09T12:57:53.938Z" }, + { url = "https://files.pythonhosted.org/packages/3e/d9/63b22a6bdbd17f1f96e9ed58604c2a6b0e72a9133e37d663bef185877cf6/coverage-7.13.4-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:7d41eead3cc673cbd38a4417deb7fd0b4ca26954ff7dc6078e33f6ff97bed940", size = 261500, upload-time = "2026-02-09T12:57:56.012Z" }, + { url = "https://files.pythonhosted.org/packages/70/bf/69f86ba1ad85bc3ad240e4c0e57a2e620fbc0e1645a47b5c62f0e941ad7f/coverage-7.13.4-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:fb26a934946a6afe0e326aebe0730cdff393a8bc0bbb65a2f41e30feddca399c", size = 265212, upload-time = "2026-02-09T12:57:57.5Z" }, + { url = "https://files.pythonhosted.org/packages/ae/f2/5f65a278a8c2148731831574c73e42f57204243d33bedaaf18fa79c5958f/coverage-7.13.4-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:dae88bc0fc77edaa65c14be099bd57ee140cf507e6bfdeea7938457ab387efb0", size = 260398, upload-time = "2026-02-09T12:57:59.027Z" }, + { url = "https://files.pythonhosted.org/packages/ef/80/6e8280a350ee9fea92f14b8357448a242dcaa243cb2c72ab0ca591f66c8c/coverage-7.13.4-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:845f352911777a8e722bfce168958214951e07e47e5d5d9744109fa5fe77f79b", size = 262584, upload-time = "2026-02-09T12:58:01.129Z" }, + { url = "https://files.pythonhosted.org/packages/22/63/01ff182fc95f260b539590fb12c11ad3e21332c15f9799cb5e2386f71d9f/coverage-7.13.4-cp313-cp313t-win32.whl", hash = "sha256:2fa8d5f8de70688a28240de9e139fa16b153cc3cbb01c5f16d88d6505ebdadf9", size = 222688, upload-time = "2026-02-09T12:58:02.736Z" }, + { url = "https://files.pythonhosted.org/packages/a9/43/89de4ef5d3cd53b886afa114065f7e9d3707bdb3e5efae13535b46ae483d/coverage-7.13.4-cp313-cp313t-win_amd64.whl", hash = "sha256:9351229c8c8407645840edcc277f4a2d44814d1bc34a2128c11c2a031d45a5dd", size = 223746, upload-time = "2026-02-09T12:58:05.362Z" }, + { url = "https://files.pythonhosted.org/packages/35/39/7cf0aa9a10d470a5309b38b289b9bb07ddeac5d61af9b664fe9775a4cb3e/coverage-7.13.4-cp313-cp313t-win_arm64.whl", hash = "sha256:30b8d0512f2dc8c8747557e8fb459d6176a2c9e5731e2b74d311c03b78451997", size = 222003, upload-time = "2026-02-09T12:58:06.952Z" }, + { url = "https://files.pythonhosted.org/packages/92/11/a9cf762bb83386467737d32187756a42094927150c3e107df4cb078e8590/coverage-7.13.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:300deaee342f90696ed186e3a00c71b5b3d27bffe9e827677954f4ee56969601", size = 219522, upload-time = "2026-02-09T12:58:08.623Z" }, + { url = "https://files.pythonhosted.org/packages/d3/28/56e6d892b7b052236d67c95f1936b6a7cf7c3e2634bf27610b8cbd7f9c60/coverage-7.13.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:29e3220258d682b6226a9b0925bc563ed9a1ebcff3cad30f043eceea7eaf2689", size = 219855, upload-time = "2026-02-09T12:58:10.176Z" }, + { url = "https://files.pythonhosted.org/packages/e5/69/233459ee9eb0c0d10fcc2fe425a029b3fa5ce0f040c966ebce851d030c70/coverage-7.13.4-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:391ee8f19bef69210978363ca930f7328081c6a0152f1166c91f0b5fdd2a773c", size = 250887, upload-time = "2026-02-09T12:58:12.503Z" }, + { url = "https://files.pythonhosted.org/packages/06/90/2cdab0974b9b5bbc1623f7876b73603aecac11b8d95b85b5b86b32de5eab/coverage-7.13.4-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0dd7ab8278f0d58a0128ba2fca25824321f05d059c1441800e934ff2efa52129", size = 253396, upload-time = "2026-02-09T12:58:14.615Z" }, + { url = "https://files.pythonhosted.org/packages/ac/15/ea4da0f85bf7d7b27635039e649e99deb8173fe551096ea15017f7053537/coverage-7.13.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78cdf0d578b15148b009ccf18c686aa4f719d887e76e6b40c38ffb61d264a552", size = 254745, upload-time = "2026-02-09T12:58:16.162Z" }, + { url = "https://files.pythonhosted.org/packages/99/11/bb356e86920c655ca4d61daee4e2bbc7258f0a37de0be32d233b561134ff/coverage-7.13.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:48685fee12c2eb3b27c62f2658e7ea21e9c3239cba5a8a242801a0a3f6a8c62a", size = 257055, upload-time = "2026-02-09T12:58:17.892Z" }, + { url = "https://files.pythonhosted.org/packages/c9/0f/9ae1f8cb17029e09da06ca4e28c9e1d5c1c0a511c7074592e37e0836c915/coverage-7.13.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4e83efc079eb39480e6346a15a1bcb3e9b04759c5202d157e1dd4303cd619356", size = 250911, upload-time = "2026-02-09T12:58:19.495Z" }, + { url = "https://files.pythonhosted.org/packages/89/3a/adfb68558fa815cbc29747b553bc833d2150228f251b127f1ce97e48547c/coverage-7.13.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ecae9737b72408d6a950f7e525f30aca12d4bd8dd95e37342e5beb3a2a8c4f71", size = 252754, upload-time = "2026-02-09T12:58:21.064Z" }, + { url = "https://files.pythonhosted.org/packages/32/b1/540d0c27c4e748bd3cd0bd001076ee416eda993c2bae47a73b7cc9357931/coverage-7.13.4-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ae4578f8528569d3cf303fef2ea569c7f4c4059a38c8667ccef15c6e1f118aa5", size = 250720, upload-time = "2026-02-09T12:58:22.622Z" }, + { url = "https://files.pythonhosted.org/packages/c7/95/383609462b3ffb1fe133014a7c84fc0dd01ed55ac6140fa1093b5af7ebb1/coverage-7.13.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:6fdef321fdfbb30a197efa02d48fcd9981f0d8ad2ae8903ac318adc653f5df98", size = 254994, upload-time = "2026-02-09T12:58:24.548Z" }, + { url = "https://files.pythonhosted.org/packages/f7/ba/1761138e86c81680bfc3c49579d66312865457f9fe405b033184e5793cb3/coverage-7.13.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b0f6ccf3dbe577170bebfce1318707d0e8c3650003cb4b3a9dd744575daa8b5", size = 250531, upload-time = "2026-02-09T12:58:26.271Z" }, + { url = "https://files.pythonhosted.org/packages/f8/8e/05900df797a9c11837ab59c4d6fe94094e029582aab75c3309a93e6fb4e3/coverage-7.13.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:75fcd519f2a5765db3f0e391eb3b7d150cce1a771bf4c9f861aeab86c767a3c0", size = 252189, upload-time = "2026-02-09T12:58:27.807Z" }, + { url = "https://files.pythonhosted.org/packages/00/bd/29c9f2db9ea4ed2738b8a9508c35626eb205d51af4ab7bf56a21a2e49926/coverage-7.13.4-cp314-cp314-win32.whl", hash = "sha256:8e798c266c378da2bd819b0677df41ab46d78065fb2a399558f3f6cae78b2fbb", size = 222258, upload-time = "2026-02-09T12:58:29.441Z" }, + { url = "https://files.pythonhosted.org/packages/a7/4d/1f8e723f6829977410efeb88f73673d794075091c8c7c18848d273dc9d73/coverage-7.13.4-cp314-cp314-win_amd64.whl", hash = "sha256:245e37f664d89861cf2329c9afa2c1fe9e6d4e1a09d872c947e70718aeeac505", size = 223073, upload-time = "2026-02-09T12:58:31.026Z" }, + { url = "https://files.pythonhosted.org/packages/51/5b/84100025be913b44e082ea32abcf1afbf4e872f5120b7a1cab1d331b1e13/coverage-7.13.4-cp314-cp314-win_arm64.whl", hash = "sha256:ad27098a189e5838900ce4c2a99f2fe42a0bf0c2093c17c69b45a71579e8d4a2", size = 221638, upload-time = "2026-02-09T12:58:32.599Z" }, + { url = "https://files.pythonhosted.org/packages/a7/e4/c884a405d6ead1370433dad1e3720216b4f9fd8ef5b64bfd984a2a60a11a/coverage-7.13.4-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:85480adfb35ffc32d40918aad81b89c69c9cc5661a9b8a81476d3e645321a056", size = 220246, upload-time = "2026-02-09T12:58:34.181Z" }, + { url = "https://files.pythonhosted.org/packages/81/5c/4d7ed8b23b233b0fffbc9dfec53c232be2e695468523242ea9fd30f97ad2/coverage-7.13.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:79be69cf7f3bf9b0deeeb062eab7ac7f36cd4cc4c4dd694bd28921ba4d8596cc", size = 220514, upload-time = "2026-02-09T12:58:35.704Z" }, + { url = "https://files.pythonhosted.org/packages/2f/6f/3284d4203fd2f28edd73034968398cd2d4cb04ab192abc8cff007ea35679/coverage-7.13.4-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:caa421e2684e382c5d8973ac55e4f36bed6821a9bad5c953494de960c74595c9", size = 261877, upload-time = "2026-02-09T12:58:37.864Z" }, + { url = "https://files.pythonhosted.org/packages/09/aa/b672a647bbe1556a85337dc95bfd40d146e9965ead9cc2fe81bde1e5cbce/coverage-7.13.4-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:14375934243ee05f56c45393fe2ce81fe5cc503c07cee2bdf1725fb8bef3ffaf", size = 264004, upload-time = "2026-02-09T12:58:39.492Z" }, + { url = "https://files.pythonhosted.org/packages/79/a1/aa384dbe9181f98bba87dd23dda436f0c6cf2e148aecbb4e50fc51c1a656/coverage-7.13.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:25a41c3104d08edb094d9db0d905ca54d0cd41c928bb6be3c4c799a54753af55", size = 266408, upload-time = "2026-02-09T12:58:41.852Z" }, + { url = "https://files.pythonhosted.org/packages/53/5e/5150bf17b4019bc600799f376bb9606941e55bd5a775dc1e096b6ffea952/coverage-7.13.4-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f01afcff62bf9a08fb32b2c1d6e924236c0383c02c790732b6537269e466a72", size = 267544, upload-time = "2026-02-09T12:58:44.093Z" }, + { url = "https://files.pythonhosted.org/packages/e0/ed/f1de5c675987a4a7a672250d2c5c9d73d289dbf13410f00ed7181d8017dd/coverage-7.13.4-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eb9078108fbf0bcdde37c3f4779303673c2fa1fe8f7956e68d447d0dd426d38a", size = 260980, upload-time = "2026-02-09T12:58:45.721Z" }, + { url = "https://files.pythonhosted.org/packages/b3/e3/fe758d01850aa172419a6743fe76ba8b92c29d181d4f676ffe2dae2ba631/coverage-7.13.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0e086334e8537ddd17e5f16a344777c1ab8194986ec533711cbe6c41cde841b6", size = 263871, upload-time = "2026-02-09T12:58:47.334Z" }, + { url = "https://files.pythonhosted.org/packages/b6/76/b829869d464115e22499541def9796b25312b8cf235d3bb00b39f1675395/coverage-7.13.4-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:725d985c5ab621268b2edb8e50dfe57633dc69bda071abc470fed55a14935fd3", size = 261472, upload-time = "2026-02-09T12:58:48.995Z" }, + { url = "https://files.pythonhosted.org/packages/14/9e/caedb1679e73e2f6ad240173f55218488bfe043e38da577c4ec977489915/coverage-7.13.4-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:3c06f0f1337c667b971ca2f975523347e63ec5e500b9aa5882d91931cd3ef750", size = 265210, upload-time = "2026-02-09T12:58:51.178Z" }, + { url = "https://files.pythonhosted.org/packages/3a/10/0dd02cb009b16ede425b49ec344aba13a6ae1dc39600840ea6abcb085ac4/coverage-7.13.4-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:590c0ed4bf8e85f745e6b805b2e1c457b2e33d5255dd9729743165253bc9ad39", size = 260319, upload-time = "2026-02-09T12:58:53.081Z" }, + { url = "https://files.pythonhosted.org/packages/92/8e/234d2c927af27c6d7a5ffad5bd2cf31634c46a477b4c7adfbfa66baf7ebb/coverage-7.13.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:eb30bf180de3f632cd043322dad5751390e5385108b2807368997d1a92a509d0", size = 262638, upload-time = "2026-02-09T12:58:55.258Z" }, + { url = "https://files.pythonhosted.org/packages/2f/64/e5547c8ff6964e5965c35a480855911b61509cce544f4d442caa759a0702/coverage-7.13.4-cp314-cp314t-win32.whl", hash = "sha256:c4240e7eded42d131a2d2c4dec70374b781b043ddc79a9de4d55ca71f8e98aea", size = 223040, upload-time = "2026-02-09T12:58:56.936Z" }, + { url = "https://files.pythonhosted.org/packages/c7/96/38086d58a181aac86d503dfa9c47eb20715a79c3e3acbdf786e92e5c09a8/coverage-7.13.4-cp314-cp314t-win_amd64.whl", hash = "sha256:4c7d3cc01e7350f2f0f6f7036caaf5673fb56b6998889ccfe9e1c1fe75a9c932", size = 224148, upload-time = "2026-02-09T12:58:58.645Z" }, + { url = "https://files.pythonhosted.org/packages/ce/72/8d10abd3740a0beb98c305e0c3faf454366221c0f37a8bcf8f60020bb65a/coverage-7.13.4-cp314-cp314t-win_arm64.whl", hash = "sha256:23e3f687cf945070d1c90f85db66d11e3025665d8dafa831301a0e0038f3db9b", size = 222172, upload-time = "2026-02-09T12:59:00.396Z" }, + { url = "https://files.pythonhosted.org/packages/0d/4a/331fe2caf6799d591109bb9c08083080f6de90a823695d412a935622abb2/coverage-7.13.4-py3-none-any.whl", hash = "sha256:1af1641e57cf7ba1bd67d677c9abdbcd6cc2ab7da3bca7fa1e2b7e50e65f2ad0", size = 211242, upload-time = "2026-02-09T12:59:02.032Z" }, +] + +[package.optional-dependencies] +toml = [ + { name = "tomli", marker = "python_full_version <= '3.11'" }, +] + [[package]] name = "fsspec" version = "2026.1.0" @@ -347,6 +451,7 @@ deploy = [ ] dev = [ { name = "pytest" }, + { name = "pytest-cov" }, { name = "ruff" }, { name = "sqlalchemy" }, { name = "ty" }, @@ -365,6 +470,7 @@ requires-dist = [ deploy = [{ name = "aws-cdk-lib", specifier = ">=2.236.0" }] dev = [ { name = "pytest", specifier = ">=9.0.2" }, + { name = "pytest-cov", specifier = ">=7.0.0" }, { name = "ruff", specifier = ">=0.15.2" }, { name = "sqlalchemy", specifier = ">=2.0.46" }, { name = "ty", specifier = ">=0.0.18" }, @@ -869,6 +975,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3b/ab/b3226f0bd7cdcf710fbede2b3548584366da3b19b5021e74f5bde2a8fa3f/pytest-9.0.2-py3-none-any.whl", hash = "sha256:711ffd45bf766d5264d487b917733b453d917afd2b0ad65223959f59089f875b", size = 374801, upload-time = "2025-12-06T21:30:49.154Z" }, ] +[[package]] +name = "pytest-cov" +version = "7.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage", extra = ["toml"] }, + { name = "pluggy" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5e/f7/c933acc76f5208b3b00089573cf6a2bc26dc80a8aece8f52bb7d6b1855ca/pytest_cov-7.0.0.tar.gz", hash = "sha256:33c97eda2e049a0c5298e91f519302a1334c26ac65c1a483d6206fd458361af1", size = 54328, upload-time = "2025-09-09T10:57:02.113Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ee/49/1377b49de7d0c1ce41292161ea0f721913fa8722c19fb9c1e3aa0367eecb/pytest_cov-7.0.0-py3-none-any.whl", hash = "sha256:3b8e9558b16cc1479da72058bdecf8073661c7f57f7d3c5f22a1c23507f2d861", size = 22424, upload-time = "2025-09-09T10:57:00.695Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -1068,6 +1188,60 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/30/643397144bfbfec6f6ef821f36f33e57d35946c44a2352d3c9f0ae847619/tenacity-9.1.2-py3-none-any.whl", hash = "sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138", size = 28248, upload-time = "2025-04-02T08:25:07.678Z" }, ] +[[package]] +name = "tomli" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/30/31573e9457673ab10aa432461bee537ce6cef177667deca369efb79df071/tomli-2.4.0.tar.gz", hash = "sha256:aa89c3f6c277dd275d8e243ad24f3b5e701491a860d5121f2cdd399fbb31fc9c", size = 17477, upload-time = "2026-01-11T11:22:38.165Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3c/d9/3dc2289e1f3b32eb19b9785b6a006b28ee99acb37d1d47f78d4c10e28bf8/tomli-2.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b5ef256a3fd497d4973c11bf142e9ed78b150d36f5773f1ca6088c230ffc5867", size = 153663, upload-time = "2026-01-11T11:21:45.27Z" }, + { url = "https://files.pythonhosted.org/packages/51/32/ef9f6845e6b9ca392cd3f64f9ec185cc6f09f0a2df3db08cbe8809d1d435/tomli-2.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5572e41282d5268eb09a697c89a7bee84fae66511f87533a6f88bd2f7b652da9", size = 148469, upload-time = "2026-01-11T11:21:46.873Z" }, + { url = "https://files.pythonhosted.org/packages/d6/c2/506e44cce89a8b1b1e047d64bd495c22c9f71f21e05f380f1a950dd9c217/tomli-2.4.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:551e321c6ba03b55676970b47cb1b73f14a0a4dce6a3e1a9458fd6d921d72e95", size = 236039, upload-time = "2026-01-11T11:21:48.503Z" }, + { url = "https://files.pythonhosted.org/packages/b3/40/e1b65986dbc861b7e986e8ec394598187fa8aee85b1650b01dd925ca0be8/tomli-2.4.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e3f639a7a8f10069d0e15408c0b96a2a828cfdec6fca05296ebcdcc28ca7c76", size = 243007, upload-time = "2026-01-11T11:21:49.456Z" }, + { url = "https://files.pythonhosted.org/packages/9c/6f/6e39ce66b58a5b7ae572a0f4352ff40c71e8573633deda43f6a379d56b3e/tomli-2.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1b168f2731796b045128c45982d3a4874057626da0e2ef1fdd722848b741361d", size = 240875, upload-time = "2026-01-11T11:21:50.755Z" }, + { url = "https://files.pythonhosted.org/packages/aa/ad/cb089cb190487caa80204d503c7fd0f4d443f90b95cf4ef5cf5aa0f439b0/tomli-2.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:133e93646ec4300d651839d382d63edff11d8978be23da4cc106f5a18b7d0576", size = 246271, upload-time = "2026-01-11T11:21:51.81Z" }, + { url = "https://files.pythonhosted.org/packages/0b/63/69125220e47fd7a3a27fd0de0c6398c89432fec41bc739823bcc66506af6/tomli-2.4.0-cp311-cp311-win32.whl", hash = "sha256:b6c78bdf37764092d369722d9946cb65b8767bfa4110f902a1b2542d8d173c8a", size = 96770, upload-time = "2026-01-11T11:21:52.647Z" }, + { url = "https://files.pythonhosted.org/packages/1e/0d/a22bb6c83f83386b0008425a6cd1fa1c14b5f3dd4bad05e98cf3dbbf4a64/tomli-2.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:d3d1654e11d724760cdb37a3d7691f0be9db5fbdaef59c9f532aabf87006dbaa", size = 107626, upload-time = "2026-01-11T11:21:53.459Z" }, + { url = "https://files.pythonhosted.org/packages/2f/6d/77be674a3485e75cacbf2ddba2b146911477bd887dda9d8c9dfb2f15e871/tomli-2.4.0-cp311-cp311-win_arm64.whl", hash = "sha256:cae9c19ed12d4e8f3ebf46d1a75090e4c0dc16271c5bce1c833ac168f08fb614", size = 94842, upload-time = "2026-01-11T11:21:54.831Z" }, + { url = "https://files.pythonhosted.org/packages/3c/43/7389a1869f2f26dba52404e1ef13b4784b6b37dac93bac53457e3ff24ca3/tomli-2.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:920b1de295e72887bafa3ad9f7a792f811847d57ea6b1215154030cf131f16b1", size = 154894, upload-time = "2026-01-11T11:21:56.07Z" }, + { url = "https://files.pythonhosted.org/packages/e9/05/2f9bf110b5294132b2edf13fe6ca6ae456204f3d749f623307cbb7a946f2/tomli-2.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7d6d9a4aee98fac3eab4952ad1d73aee87359452d1c086b5ceb43ed02ddb16b8", size = 149053, upload-time = "2026-01-11T11:21:57.467Z" }, + { url = "https://files.pythonhosted.org/packages/e8/41/1eda3ca1abc6f6154a8db4d714a4d35c4ad90adc0bcf700657291593fbf3/tomli-2.4.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:36b9d05b51e65b254ea6c2585b59d2c4cb91c8a3d91d0ed0f17591a29aaea54a", size = 243481, upload-time = "2026-01-11T11:21:58.661Z" }, + { url = "https://files.pythonhosted.org/packages/d2/6d/02ff5ab6c8868b41e7d4b987ce2b5f6a51d3335a70aa144edd999e055a01/tomli-2.4.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1c8a885b370751837c029ef9bc014f27d80840e48bac415f3412e6593bbc18c1", size = 251720, upload-time = "2026-01-11T11:22:00.178Z" }, + { url = "https://files.pythonhosted.org/packages/7b/57/0405c59a909c45d5b6f146107c6d997825aa87568b042042f7a9c0afed34/tomli-2.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8768715ffc41f0008abe25d808c20c3d990f42b6e2e58305d5da280ae7d1fa3b", size = 247014, upload-time = "2026-01-11T11:22:01.238Z" }, + { url = "https://files.pythonhosted.org/packages/2c/0e/2e37568edd944b4165735687cbaf2fe3648129e440c26d02223672ee0630/tomli-2.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7b438885858efd5be02a9a133caf5812b8776ee0c969fea02c45e8e3f296ba51", size = 251820, upload-time = "2026-01-11T11:22:02.727Z" }, + { url = "https://files.pythonhosted.org/packages/5a/1c/ee3b707fdac82aeeb92d1a113f803cf6d0f37bdca0849cb489553e1f417a/tomli-2.4.0-cp312-cp312-win32.whl", hash = "sha256:0408e3de5ec77cc7f81960c362543cbbd91ef883e3138e81b729fc3eea5b9729", size = 97712, upload-time = "2026-01-11T11:22:03.777Z" }, + { url = "https://files.pythonhosted.org/packages/69/13/c07a9177d0b3bab7913299b9278845fc6eaaca14a02667c6be0b0a2270c8/tomli-2.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:685306e2cc7da35be4ee914fd34ab801a6acacb061b6a7abca922aaf9ad368da", size = 108296, upload-time = "2026-01-11T11:22:04.86Z" }, + { url = "https://files.pythonhosted.org/packages/18/27/e267a60bbeeee343bcc279bb9e8fbed0cbe224bc7b2a3dc2975f22809a09/tomli-2.4.0-cp312-cp312-win_arm64.whl", hash = "sha256:5aa48d7c2356055feef06a43611fc401a07337d5b006be13a30f6c58f869e3c3", size = 94553, upload-time = "2026-01-11T11:22:05.854Z" }, + { url = "https://files.pythonhosted.org/packages/34/91/7f65f9809f2936e1f4ce6268ae1903074563603b2a2bd969ebbda802744f/tomli-2.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:84d081fbc252d1b6a982e1870660e7330fb8f90f676f6e78b052ad4e64714bf0", size = 154915, upload-time = "2026-01-11T11:22:06.703Z" }, + { url = "https://files.pythonhosted.org/packages/20/aa/64dd73a5a849c2e8f216b755599c511badde80e91e9bc2271baa7b2cdbb1/tomli-2.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:9a08144fa4cba33db5255f9b74f0b89888622109bd2776148f2597447f92a94e", size = 149038, upload-time = "2026-01-11T11:22:07.56Z" }, + { url = "https://files.pythonhosted.org/packages/9e/8a/6d38870bd3d52c8d1505ce054469a73f73a0fe62c0eaf5dddf61447e32fa/tomli-2.4.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c73add4bb52a206fd0c0723432db123c0c75c280cbd67174dd9d2db228ebb1b4", size = 242245, upload-time = "2026-01-11T11:22:08.344Z" }, + { url = "https://files.pythonhosted.org/packages/59/bb/8002fadefb64ab2669e5b977df3f5e444febea60e717e755b38bb7c41029/tomli-2.4.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fb2945cbe303b1419e2706e711b7113da57b7db31ee378d08712d678a34e51e", size = 250335, upload-time = "2026-01-11T11:22:09.951Z" }, + { url = "https://files.pythonhosted.org/packages/a5/3d/4cdb6f791682b2ea916af2de96121b3cb1284d7c203d97d92d6003e91c8d/tomli-2.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bbb1b10aa643d973366dc2cb1ad94f99c1726a02343d43cbc011edbfac579e7c", size = 245962, upload-time = "2026-01-11T11:22:11.27Z" }, + { url = "https://files.pythonhosted.org/packages/f2/4a/5f25789f9a460bd858ba9756ff52d0830d825b458e13f754952dd15fb7bb/tomli-2.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4cbcb367d44a1f0c2be408758b43e1ffb5308abe0ea222897d6bfc8e8281ef2f", size = 250396, upload-time = "2026-01-11T11:22:12.325Z" }, + { url = "https://files.pythonhosted.org/packages/aa/2f/b73a36fea58dfa08e8b3a268750e6853a6aac2a349241a905ebd86f3047a/tomli-2.4.0-cp313-cp313-win32.whl", hash = "sha256:7d49c66a7d5e56ac959cb6fc583aff0651094ec071ba9ad43df785abc2320d86", size = 97530, upload-time = "2026-01-11T11:22:13.865Z" }, + { url = "https://files.pythonhosted.org/packages/3b/af/ca18c134b5d75de7e8dc551c5234eaba2e8e951f6b30139599b53de9c187/tomli-2.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:3cf226acb51d8f1c394c1b310e0e0e61fecdd7adcb78d01e294ac297dd2e7f87", size = 108227, upload-time = "2026-01-11T11:22:15.224Z" }, + { url = "https://files.pythonhosted.org/packages/22/c3/b386b832f209fee8073c8138ec50f27b4460db2fdae9ffe022df89a57f9b/tomli-2.4.0-cp313-cp313-win_arm64.whl", hash = "sha256:d20b797a5c1ad80c516e41bc1fb0443ddb5006e9aaa7bda2d71978346aeb9132", size = 94748, upload-time = "2026-01-11T11:22:16.009Z" }, + { url = "https://files.pythonhosted.org/packages/f3/c4/84047a97eb1004418bc10bdbcfebda209fca6338002eba2dc27cc6d13563/tomli-2.4.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:26ab906a1eb794cd4e103691daa23d95c6919cc2fa9160000ac02370cc9dd3f6", size = 154725, upload-time = "2026-01-11T11:22:17.269Z" }, + { url = "https://files.pythonhosted.org/packages/a8/5d/d39038e646060b9d76274078cddf146ced86dc2b9e8bbf737ad5983609a0/tomli-2.4.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:20cedb4ee43278bc4f2fee6cb50daec836959aadaf948db5172e776dd3d993fc", size = 148901, upload-time = "2026-01-11T11:22:18.287Z" }, + { url = "https://files.pythonhosted.org/packages/73/e5/383be1724cb30f4ce44983d249645684a48c435e1cd4f8b5cded8a816d3c/tomli-2.4.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:39b0b5d1b6dd03684b3fb276407ebed7090bbec989fa55838c98560c01113b66", size = 243375, upload-time = "2026-01-11T11:22:19.154Z" }, + { url = "https://files.pythonhosted.org/packages/31/f0/bea80c17971c8d16d3cc109dc3585b0f2ce1036b5f4a8a183789023574f2/tomli-2.4.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a26d7ff68dfdb9f87a016ecfd1e1c2bacbe3108f4e0f8bcd2228ef9a766c787d", size = 250639, upload-time = "2026-01-11T11:22:20.168Z" }, + { url = "https://files.pythonhosted.org/packages/2c/8f/2853c36abbb7608e3f945d8a74e32ed3a74ee3a1f468f1ffc7d1cb3abba6/tomli-2.4.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:20ffd184fb1df76a66e34bd1b36b4a4641bd2b82954befa32fe8163e79f1a702", size = 246897, upload-time = "2026-01-11T11:22:21.544Z" }, + { url = "https://files.pythonhosted.org/packages/49/f0/6c05e3196ed5337b9fe7ea003e95fd3819a840b7a0f2bf5a408ef1dad8ed/tomli-2.4.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:75c2f8bbddf170e8effc98f5e9084a8751f8174ea6ccf4fca5398436e0320bc8", size = 254697, upload-time = "2026-01-11T11:22:23.058Z" }, + { url = "https://files.pythonhosted.org/packages/f3/f5/2922ef29c9f2951883525def7429967fc4d8208494e5ab524234f06b688b/tomli-2.4.0-cp314-cp314-win32.whl", hash = "sha256:31d556d079d72db7c584c0627ff3a24c5d3fb4f730221d3444f3efb1b2514776", size = 98567, upload-time = "2026-01-11T11:22:24.033Z" }, + { url = "https://files.pythonhosted.org/packages/7b/31/22b52e2e06dd2a5fdbc3ee73226d763b184ff21fc24e20316a44ccc4d96b/tomli-2.4.0-cp314-cp314-win_amd64.whl", hash = "sha256:43e685b9b2341681907759cf3a04e14d7104b3580f808cfde1dfdb60ada85475", size = 108556, upload-time = "2026-01-11T11:22:25.378Z" }, + { url = "https://files.pythonhosted.org/packages/48/3d/5058dff3255a3d01b705413f64f4306a141a8fd7a251e5a495e3f192a998/tomli-2.4.0-cp314-cp314-win_arm64.whl", hash = "sha256:3d895d56bd3f82ddd6faaff993c275efc2ff38e52322ea264122d72729dca2b2", size = 96014, upload-time = "2026-01-11T11:22:26.138Z" }, + { url = "https://files.pythonhosted.org/packages/b8/4e/75dab8586e268424202d3a1997ef6014919c941b50642a1682df43204c22/tomli-2.4.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5b5807f3999fb66776dbce568cc9a828544244a8eb84b84b9bafc080c99597b9", size = 163339, upload-time = "2026-01-11T11:22:27.143Z" }, + { url = "https://files.pythonhosted.org/packages/06/e3/b904d9ab1016829a776d97f163f183a48be6a4deb87304d1e0116a349519/tomli-2.4.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c084ad935abe686bd9c898e62a02a19abfc9760b5a79bc29644463eaf2840cb0", size = 159490, upload-time = "2026-01-11T11:22:28.399Z" }, + { url = "https://files.pythonhosted.org/packages/e3/5a/fc3622c8b1ad823e8ea98a35e3c632ee316d48f66f80f9708ceb4f2a0322/tomli-2.4.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0f2e3955efea4d1cfbcb87bc321e00dc08d2bcb737fd1d5e398af111d86db5df", size = 269398, upload-time = "2026-01-11T11:22:29.345Z" }, + { url = "https://files.pythonhosted.org/packages/fd/33/62bd6152c8bdd4c305ad9faca48f51d3acb2df1f8791b1477d46ff86e7f8/tomli-2.4.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e0fe8a0b8312acf3a88077a0802565cb09ee34107813bba1c7cd591fa6cfc8d", size = 276515, upload-time = "2026-01-11T11:22:30.327Z" }, + { url = "https://files.pythonhosted.org/packages/4b/ff/ae53619499f5235ee4211e62a8d7982ba9e439a0fb4f2f351a93d67c1dd2/tomli-2.4.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:413540dce94673591859c4c6f794dfeaa845e98bf35d72ed59636f869ef9f86f", size = 273806, upload-time = "2026-01-11T11:22:32.56Z" }, + { url = "https://files.pythonhosted.org/packages/47/71/cbca7787fa68d4d0a9f7072821980b39fbb1b6faeb5f5cf02f4a5559fa28/tomli-2.4.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:0dc56fef0e2c1c470aeac5b6ca8cc7b640bb93e92d9803ddaf9ea03e198f5b0b", size = 281340, upload-time = "2026-01-11T11:22:33.505Z" }, + { url = "https://files.pythonhosted.org/packages/f5/00/d595c120963ad42474cf6ee7771ad0d0e8a49d0f01e29576ee9195d9ecdf/tomli-2.4.0-cp314-cp314t-win32.whl", hash = "sha256:d878f2a6707cc9d53a1be1414bbb419e629c3d6e67f69230217bb663e76b5087", size = 108106, upload-time = "2026-01-11T11:22:34.451Z" }, + { url = "https://files.pythonhosted.org/packages/de/69/9aa0c6a505c2f80e519b43764f8b4ba93b5a0bbd2d9a9de6e2b24271b9a5/tomli-2.4.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2add28aacc7425117ff6364fe9e06a183bb0251b03f986df0e78e974047571fd", size = 120504, upload-time = "2026-01-11T11:22:35.764Z" }, + { url = "https://files.pythonhosted.org/packages/b3/9f/f1668c281c58cfae01482f7114a4b88d345e4c140386241a1a24dcc9e7bc/tomli-2.4.0-cp314-cp314t-win_arm64.whl", hash = "sha256:2b1e3b80e1d5e52e40e9b924ec43d81570f0e7d09d11081b797bc4692765a3d4", size = 99561, upload-time = "2026-01-11T11:22:36.624Z" }, + { url = "https://files.pythonhosted.org/packages/23/d1/136eb2cb77520a31e1f64cbae9d33ec6df0d78bdf4160398e86eec8a8754/tomli-2.4.0-py3-none-any.whl", hash = "sha256:1f776e7d669ebceb01dee46484485f43a4048746235e683bcdffacdf1fb4785a", size = 14477, upload-time = "2026-01-11T11:22:37.446Z" }, +] + [[package]] name = "ty" version = "0.0.18" From 8e11478fa965de26b95d72629824805e8c85b509 Mon Sep 17 00:00:00 2001 From: hrodmn Date: Thu, 26 Feb 2026 12:03:57 -0600 Subject: [PATCH 06/23] chore(deps): add pre-commit to dev deps --- pyproject.toml | 1 + uv.lock | 155 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 156 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index afd9d4f..54d5f6b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ deploy = [ "aws-cdk-lib>=2.236.0", ] dev = [ + "pre-commit>=4.5.1", "pytest>=9.0.2", "pytest-cov>=7.0.0", "ruff>=0.15.2", diff --git a/uv.lock b/uv.lock index b5cd488..669d1cd 100644 --- a/uv.lock +++ b/uv.lock @@ -148,6 +148,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e6/ad/3cc14f097111b4de0040c83a525973216457bbeeb63739ef1ed275c1c021/certifi-2026.1.4-py3-none-any.whl", hash = "sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c", size = 152900, upload-time = "2026-01-04T02:42:40.15Z" }, ] +[[package]] +name = "cfgv" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/b5/721b8799b04bf9afe054a3899c6cf4e880fcf8563cc71c15610242490a0c/cfgv-3.5.0.tar.gz", hash = "sha256:d5b1034354820651caa73ede66a6294d6e95c1b00acc5e9b098e917404669132", size = 7334, upload-time = "2025-11-19T20:55:51.612Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/3c/33bac158f8ab7f89b2e59426d5fe2e4f63f7ed25df84c036890172b412b5/cfgv-3.5.0-py2.py3-none-any.whl", hash = "sha256:a8dc6b26ad22ff227d2634a65cb388215ce6cc96bbcc5cfde7641ae87e8dacc0", size = 7445, upload-time = "2025-11-19T20:55:50.744Z" }, +] + [[package]] name = "charset-normalizer" version = "3.4.4" @@ -360,6 +369,24 @@ toml = [ { name = "tomli", marker = "python_full_version <= '3.11'" }, ] +[[package]] +name = "distlib" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/96/8e/709914eb2b5749865801041647dc7f4e6d00b549cfe88b65ca192995f07c/distlib-0.4.0.tar.gz", hash = "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d", size = 614605, upload-time = "2025-07-17T16:52:00.465Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16", size = 469047, upload-time = "2025-07-17T16:51:58.613Z" }, +] + +[[package]] +name = "filelock" +version = "3.24.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/73/92/a8e2479937ff39185d20dd6a851c1a63e55849e447a55e798cc2e1f49c65/filelock-3.24.3.tar.gz", hash = "sha256:011a5644dc937c22699943ebbfc46e969cdde3e171470a6e40b9533e5a72affa", size = 37935, upload-time = "2026-02-19T00:48:20.543Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/0f/5d0c71a1aefeb08efff26272149e07ab922b64f46c63363756224bd6872e/filelock-3.24.3-py3-none-any.whl", hash = "sha256:426e9a4660391f7f8a810d71b0555bce9008b0a1cc342ab1f6947d37639e002d", size = 24331, upload-time = "2026-02-19T00:48:18.465Z" }, +] + [[package]] name = "fsspec" version = "2026.1.0" @@ -450,6 +477,7 @@ deploy = [ { name = "aws-cdk-lib" }, ] dev = [ + { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-cov" }, { name = "ruff" }, @@ -469,6 +497,7 @@ requires-dist = [ [package.metadata.requires-dev] deploy = [{ name = "aws-cdk-lib", specifier = ">=2.236.0" }] dev = [ + { name = "pre-commit", specifier = ">=4.5.1" }, { name = "pytest", specifier = ">=9.0.2" }, { name = "pytest-cov", specifier = ">=7.0.0" }, { name = "ruff", specifier = ">=0.15.2" }, @@ -476,6 +505,15 @@ dev = [ { name = "ty", specifier = ">=0.0.18" }, ] +[[package]] +name = "identify" +version = "2.6.16" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5b/8d/e8b97e6bd3fb6fb271346f7981362f1e04d6a7463abd0de79e1fda17c067/identify-2.6.16.tar.gz", hash = "sha256:846857203b5511bbe94d5a352a48ef2359532bc8f6727b5544077a0dcfb24980", size = 99360, upload-time = "2026-01-12T18:58:58.201Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b8/58/40fbbcefeda82364720eba5cf2270f98496bdfa19ea75b4cccae79c698e6/identify-2.6.16-py2.py3-none-any.whl", hash = "sha256:391ee4d77741d994189522896270b787aed8670389bfd60f326d677d64a6dfb0", size = 99202, upload-time = "2026-01-12T18:58:56.627Z" }, +] + [[package]] name = "idna" version = "3.11" @@ -638,6 +676,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6a/fc/0e61d9a4e29c8679356795a40e48f647b4aad58d71bfc969f0f8f56fb912/mmh3-5.2.0-cp314-cp314t-win_arm64.whl", hash = "sha256:e7884931fe5e788163e7b3c511614130c2c59feffdc21112290a194487efb2e9", size = 40455, upload-time = "2025-07-29T07:43:29.563Z" }, ] +[[package]] +name = "nodeenv" +version = "1.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/bf/d1bda4f6168e0b2e9e5958945e01910052158313224ada5ce1fb2e1113b8/nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb", size = 55611, upload-time = "2025-12-20T14:08:54.006Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, +] + [[package]] name = "packaging" version = "26.0" @@ -647,6 +694,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, ] +[[package]] +name = "platformdirs" +version = "4.9.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/04/fea538adf7dbbd6d186f551d595961e564a3b6715bdf276b477460858672/platformdirs-4.9.2.tar.gz", hash = "sha256:9a33809944b9db043ad67ca0db94b14bf452cc6aeaac46a88ea55b26e2e9d291", size = 28394, upload-time = "2026-02-16T03:56:10.574Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/31/05e764397056194206169869b50cf2fee4dbbbc71b344705b9c0d878d4d8/platformdirs-4.9.2-py3-none-any.whl", hash = "sha256:9170634f126f8efdae22fb58ae8a0eaa86f38365bc57897a6c4f781d1f5875bd", size = 21168, upload-time = "2026-02-16T03:56:08.891Z" }, +] + [[package]] name = "pluggy" version = "1.6.0" @@ -656,6 +712,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "pre-commit" +version = "4.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cfgv" }, + { name = "identify" }, + { name = "nodeenv" }, + { name = "pyyaml" }, + { name = "virtualenv" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/40/f1/6d86a29246dfd2e9b6237f0b5823717f60cad94d47ddc26afa916d21f525/pre_commit-4.5.1.tar.gz", hash = "sha256:eb545fcff725875197837263e977ea257a402056661f09dae08e4b149b030a61", size = 198232, upload-time = "2025-12-16T21:14:33.552Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/19/fd3ef348460c80af7bb4669ea7926651d1f95c23ff2df18b9d24bab4f3fa/pre_commit-4.5.1-py2.py3-none-any.whl", hash = "sha256:3b3afd891e97337708c1674210f8eba659b52a38ea5f822ff142d10786221f77", size = 226437, upload-time = "2025-12-16T21:14:32.409Z" }, +] + [[package]] name = "publication" version = "0.0.3" @@ -1001,6 +1073,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, ] +[[package]] +name = "python-discovery" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, + { name = "platformdirs" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/82/bb/93a3e83bdf9322c7e21cafd092e56a4a17c4d8ef4277b6eb01af1a540a6f/python_discovery-1.1.0.tar.gz", hash = "sha256:447941ba1aed8cc2ab7ee3cb91be5fc137c5bdbb05b7e6ea62fbdcb66e50b268", size = 55674, upload-time = "2026-02-26T09:42:49.668Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/54/82a6e2ef37f0f23dccac604b9585bdcbd0698604feb64807dcb72853693e/python_discovery-1.1.0-py3-none-any.whl", hash = "sha256:a162893b8809727f54594a99ad2179d2ede4bf953e12d4c7abc3cc9cdbd1437b", size = 30687, upload-time = "2026-02-26T09:42:48.548Z" }, +] + [[package]] name = "python-dotenv" version = "1.2.1" @@ -1010,6 +1095,61 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/1b/a298b06749107c305e1fe0f814c6c74aea7b2f1e10989cb30f544a1b3253/python_dotenv-1.2.1-py3-none-any.whl", hash = "sha256:b81ee9561e9ca4004139c6cbba3a238c32b03e4894671e181b671e8cb8425d61", size = 21230, upload-time = "2025-10-26T15:12:09.109Z" }, ] +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + [[package]] name = "requests" version = "2.32.5" @@ -1304,3 +1444,18 @@ sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6 wheels = [ { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" }, ] + +[[package]] +name = "virtualenv" +version = "21.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "distlib" }, + { name = "filelock" }, + { name = "platformdirs" }, + { name = "python-discovery" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ce/4f/d6a5ff3b020c801c808b14e2d2330cdc8ebefe1cdfbc457ecc368e971fec/virtualenv-21.0.0.tar.gz", hash = "sha256:e8efe4271b4a5efe7a4dce9d60a05fd11859406c0d6aa8464f4cf451bc132889", size = 5836591, upload-time = "2026-02-25T20:21:07.691Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/d1/3f62e4f9577b28c352c11623a03fb916096d5c131303d4861b4914481b6b/virtualenv-21.0.0-py3-none-any.whl", hash = "sha256:d44e70637402c7f4b10f48491c02a6397a3a187152a70cba0b6bc7642d69fb05", size = 5817167, upload-time = "2026-02-25T20:21:05.476Z" }, +] From c5d2bc90b29263d0c9288c3073e6878c62752c35 Mon Sep 17 00:00:00 2001 From: hrodmn Date: Fri, 27 Feb 2026 16:19:44 -0600 Subject: [PATCH 07/23] apply feedback from review --- .env | 12 - .github/workflows/ci.yml | 4 - main.py | 25 +- src/icestac/catalog.py | 91 ++++++ src/icestac/config.py | 308 ++++++++++-------- src/icestac/item_table.py | 62 ---- src/icestac/lambda_handler.py | 0 tests/conftest.py | 36 +- tests/{test_item_table.py => test_catalog.py} | 46 ++- tests/test_config.py | 96 ++---- tests/test_load.py | 59 +--- 11 files changed, 369 insertions(+), 370 deletions(-) delete mode 100644 .env create mode 100644 src/icestac/catalog.py delete mode 100644 src/icestac/item_table.py delete mode 100644 src/icestac/lambda_handler.py rename tests/{test_item_table.py => test_catalog.py} (62%) diff --git a/.env b/.env deleted file mode 100644 index 7ef3d3a..0000000 --- a/.env +++ /dev/null @@ -1,12 +0,0 @@ -# Iceberg Catalog Configuration -ICESTAC_CATALOG_NAME=rest_catalog -ICESTAC_CATALOG_TYPE=rest -ICESTAC_CATALOG_URI=http://localhost:8181 -ICESTAC_WAREHOUSE_PATH=s3://warehouse/ - -# S3/MinIO Storage Configuration -# These settings allow PyIceberg to directly access MinIO for data file I/O -ICESTAC_S3_ENDPOINT=http://localhost:9000 -ICESTAC_S3_ACCESS_KEY_ID=admin -ICESTAC_S3_SECRET_ACCESS_KEY=password -ICESTAC_S3_PATH_STYLE_ACCESS=true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f6a5f8..0751afb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,10 +26,6 @@ jobs: version: "0.10.*" enable-cache: true - - name: Install dependencies - run: | - uv sync - - name: Run pre-commit run: uv run pre-commit run --all-files diff --git a/main.py b/main.py index 5c87703..d44c704 100644 --- a/main.py +++ b/main.py @@ -2,37 +2,30 @@ import rustac -from icestac.config import IcebergCatalogConfig -from icestac.constants import DEFAULT_NAMESPACE -from icestac.item_table import create_item_table -from icestac.load import load_items +from icestac.catalog import IcestacCatalog +from icestac.config import IcestacCatalogConfig from icestac.schema import get_schema_from_item async def run(): - config = IcebergCatalogConfig() - catalog = config.load_catalog() + config = IcestacCatalogConfig.model_validate({}) + catalog = IcestacCatalog.from_config(config) items = await rustac.search( "https://stac.maap-project.org", collections="icesat2-boreal-v3.1-agb", max_items=5, ) - + collection_id = "icesat2_boreal_v3_1_agb" for item in items: - item["collection"] = "icesat2_boreal_v3_1_agb" + item["collection"] = collection_id schema = get_schema_from_item(items[0]) - table = create_item_table( - arrow_schema=schema, - collection_id=items[0]["collection"], - catalog=catalog, - namespace=DEFAULT_NAMESPACE, - ) + catalog.create_item_table(arrow_schema=schema, collection_id=collection_id) - load_items( + catalog.load_items( + collection_id=collection_id, items=items, - table=table, method="upsert", ) diff --git a/src/icestac/catalog.py b/src/icestac/catalog.py new file mode 100644 index 0000000..eadd86c --- /dev/null +++ b/src/icestac/catalog.py @@ -0,0 +1,91 @@ +from dataclasses import dataclass +from typing import Any + +from arro3.core import Schema as ArrowSchema +from pyiceberg.catalog import Catalog, load_catalog +from pyiceberg.partitioning import PartitionField, PartitionSpec +from pyiceberg.table import Table +from pyiceberg.transforms import MonthTransform + +from icestac.config import IcestacCatalogConfig +from icestac.errors import InvalidCollectionIdError +from icestac.load import Method, load_items +from icestac.schema import convert_schema, validate_schema + + +def validate_collection_id(collection_id: str) -> None: + """Ensure collection id is valid for icestac schema""" + + if "." in collection_id: + raise InvalidCollectionIdError + + +@dataclass +class IcestacCatalog: + """Icestac client class for pyiceberg Catalog""" + + catalog: Catalog + namespace: str + + def __getattr__(self, name: str) -> Any: + return getattr(self.catalog, name) + + def __post_init__(self) -> None: + self.create_namespace_if_not_exists(self.namespace) + + @classmethod + def from_config(cls, config: IcestacCatalogConfig) -> "IcestacCatalog": + catalog = load_catalog(config.catalog_name, **config.get_catalog_properties()) + + return cls(catalog=catalog, namespace=config.namespace) + + def create_item_table( + self, + collection_id: str, + arrow_schema: ArrowSchema, + ) -> Table: + """ + Create an Iceberg table from a stac-geoparquet Arrow schema + + Converts the Arrow schema to an Iceberg schema with manually assigned field IDs, + then creates or loads the Iceberg table partitioned by datetime month. + + Args: + schema: arro3.core.Schema for the items in this collection + collection_id: the collection id for the items in this table + catalog: PyIceberg catalog instance + namespace: Namespace for the Iceberg table + + Returns: + PyIceberg Table instance + + """ + validate_collection_id(collection_id) + validate_schema(arrow_schema) + + # TODO: check if collection record is present in collections table + + iceberg_schema = convert_schema(arrow_schema) + + return self.create_table( + identifier=f"{self.namespace}.{collection_id}", + schema=iceberg_schema, + partition_spec=PartitionSpec( + # TODO: make temporal partitioning configurable + PartitionField( + source_id=iceberg_schema.find_field("datetime").field_id, + field_id=1000, + transform=MonthTransform(), + name="datetime_month", + ) + ), + ) + + def load_item_table(self, collection_id: str) -> Table: + """Load the item table for a collection""" + return self.load_table(identifier=f"{self.namespace}.{collection_id}") + + def load_items( + self, collection_id: str, items: list[dict[str, Any]], method: Method = "upsert" + ) -> None: + load_items(items, table=self.load_item_table(collection_id), method=method) diff --git a/src/icestac/config.py b/src/icestac/config.py index 1f661a7..c4a875e 100644 --- a/src/icestac/config.py +++ b/src/icestac/config.py @@ -1,179 +1,207 @@ """Settings module for icestac Iceberg catalog configuration.""" -from typing import Literal +from typing import Annotated, Literal -from pydantic import Field, model_validator +from pydantic import BaseModel, Field from pydantic_settings import BaseSettings, SettingsConfigDict -from pyiceberg.catalog import Catalog, load_catalog +from icestac.constants import DEFAULT_NAMESPACE -class IcebergCatalogConfig(BaseSettings): - """ - Pydantic settings for configuring an Apache Iceberg catalog. - Settings are loaded from environment variables with the ICESTAC_ prefix. - Supports multiple catalog types: rest, glue, hive, and sql. +class RestConfig(BaseModel): + """Authentication settings for REST catalogs.""" + + token: str | None = Field( + default=None, description="Bearer token for REST catalog authentication" + ) + credential: str | None = Field( + default=None, description="Credential for REST catalog authentication" + ) - Examples: - REST Catalog: - ICESTAC_CATALOG_NAME=my_catalog - ICESTAC_CATALOG_TYPE=rest - ICESTAC_CATALOG_URI=https://iceberg-rest.example.com - - AWS Glue Catalog: - ICESTAC_CATALOG_NAME=glue_catalog - ICESTAC_CATALOG_TYPE=glue - ICESTAC_WAREHOUSE_PATH=s3://my-bucket/warehouse/ - - SQL Catalog (SQLite): - ICESTAC_CATALOG_NAME=local_catalog - ICESTAC_CATALOG_TYPE=sql - ICESTAC_CATALOG_URI=sqlite:///path/to/catalog.db - ICESTAC_WAREHOUSE_PATH=/path/to/warehouse - """ - model_config = SettingsConfigDict( - env_prefix="ICESTAC_", - case_sensitive=False, - env_file=".env", - env_file_encoding="utf-8", - extra="allow", # Allow extra fields for catalog-specific properties +class S3Config(BaseModel): + """S3/MinIO storage settings.""" + + endpoint: str | None = Field( + default=None, + description="S3 endpoint URL (required for MinIO or custom S3-compatible storage)", + ) + access_key_id: str | None = Field(default=None, description="S3 access key ID") + secret_access_key: str | None = Field( + default=None, description="S3 secret access key" + ) + path_style_access: bool = Field( + default=True, description="Use path-style access for S3 (required for MinIO)" ) - # Core catalog settings + +class SqlConfig(BaseModel): + """Settings specific to SQL catalogs.""" + + echo: bool = Field(default=False, description="Enable SQL query logging") + + +class GlueConfig(BaseModel): + """Settings specific to AWS Glue catalogs.""" + + region: str | None = Field(default=None, description="AWS region for Glue catalog") + + +_SETTINGS_CONFIG = SettingsConfigDict( + env_prefix="ICESTAC_", + env_nested_delimiter="__", + case_sensitive=False, + env_file=".env", + env_file_encoding="utf-8", + extra="allow", +) + + +class _BaseCatalogConfig(BaseSettings): + """Base settings shared by all catalog types.""" + + model_config = _SETTINGS_CONFIG + + catalog_type: str # Narrowed to Literal in each subclass + catalog_name: str = Field( default="default", description="Name of the Iceberg catalog" ) - - catalog_type: Literal["rest", "glue", "hive", "sql"] = Field( - default="sql", description="Type of Iceberg catalog backend" + namespace: str = Field( + default=DEFAULT_NAMESPACE, description="Namespace within Iceberg catalog" ) + s3: S3Config = Field(default_factory=S3Config) - catalog_uri: str | None = Field( - default=None, - description="URI for the catalog (required for rest, hive, and sql catalogs)", - ) + def _get_base_properties(self) -> dict[str, str]: + """Build properties common to all catalog types.""" + properties: dict[str, str] = {"type": self.catalog_type} - warehouse_path: str | None = Field( - default=None, - description="Base path for the data warehouse (required for most catalog types)", - ) + if self.s3.endpoint: + properties["s3.endpoint"] = self.s3.endpoint + properties["s3.path-style-access"] = str(self.s3.path_style_access).lower() + if self.s3.access_key_id: + properties["s3.access-key-id"] = self.s3.access_key_id + if self.s3.secret_access_key: + properties["s3.secret-access-key"] = self.s3.secret_access_key - # AWS-specific settings - aws_region: str | None = Field( - default=None, description="AWS region for Glue catalog" - ) + for key, value in (self.model_extra or {}).items(): + if value is not None: + properties[key] = str(value) - # REST catalog authentication - rest_token: str | None = Field( - default=None, description="Bearer token for REST catalog authentication" - ) + return properties - rest_credential: str | None = Field( - default=None, description="Credential for REST catalog authentication" - ) - # SQL catalog settings - sql_echo: bool = Field( - default=False, description="Enable SQL query logging (for sql catalog type)" - ) +class SqlCatalogConfig(_BaseCatalogConfig): + """ + Configuration for SQL-based Iceberg catalogs (SQLite, PostgreSQL, etc.). - # S3/MinIO storage settings - s3_endpoint: str | None = Field( - default=None, - description="S3 endpoint URL (required for MinIO or custom S3-compatible storage)", + Examples: + ICESTAC_CATALOG_TYPE=sql + ICESTAC_CATALOG_URI=sqlite:///path/to/catalog.db + ICESTAC_WAREHOUSE_PATH=/path/to/warehouse + ICESTAC_SQL__ECHO=true + """ + + catalog_type: Literal["sql"] = "sql" + catalog_uri: str = Field( + description="URI for the SQL catalog (e.g. sqlite:///path/to/catalog.db)" ) + warehouse_path: str = Field(description="Base path for the data warehouse") + sql: SqlConfig = Field(default_factory=SqlConfig) - s3_access_key_id: str | None = Field(default=None, description="S3 access key ID") + def get_catalog_properties(self) -> dict[str, str]: + props = self._get_base_properties() + props.update( + { + "uri": self.catalog_uri, + "warehouse": self.warehouse_path, + "echo": str(self.sql.echo).lower(), + } + ) + return props + + +class RestCatalogConfig(_BaseCatalogConfig): + """ + Configuration for REST-based Iceberg catalogs. - s3_secret_access_key: str | None = Field( - default=None, description="S3 secret access key" - ) + Examples: + ICESTAC_CATALOG_TYPE=rest + ICESTAC_CATALOG_URI=https://iceberg-rest.example.com + ICESTAC_REST__TOKEN=my-token + """ - s3_path_style_access: bool = Field( - default=True, description="Use path-style access for S3 (required for MinIO)" + catalog_type: Literal["rest"] = "rest" + catalog_uri: str = Field(description="URI for the REST catalog") + warehouse_path: str | None = Field( + default=None, description="Optional warehouse path" ) + rest: RestConfig = Field(default_factory=RestConfig) - @model_validator(mode="after") - def validate_required_fields(self): - """Ensure required fields are provided based on catalog type.""" - # Validate catalog_uri requirement - if self.catalog_type in ["rest", "hive", "sql"] and not self.catalog_uri: - raise ValueError( - f"catalog_uri is required for catalog_type='{self.catalog_type}'" - ) + def get_catalog_properties(self) -> dict[str, str]: + props = self._get_base_properties() + props["uri"] = self.catalog_uri + if self.warehouse_path: + props["warehouse"] = self.warehouse_path + if self.rest.token: + props["token"] = self.rest.token + if self.rest.credential: + props["credential"] = self.rest.credential + return props - # Validate warehouse_path requirement - if self.catalog_type in ["sql", "hive"] and not self.warehouse_path: - raise ValueError( - f"warehouse_path is required for catalog_type='{self.catalog_type}'" - ) - return self +class GlueCatalogConfig(_BaseCatalogConfig): + """ + Configuration for AWS Glue Iceberg catalogs. - def get_catalog_properties(self) -> dict[str, str]: - """ - Generate the properties dict for PyIceberg catalog initialization. + Examples: + ICESTAC_CATALOG_TYPE=glue + ICESTAC_WAREHOUSE_PATH=s3://my-bucket/warehouse/ + ICESTAC_GLUE__REGION=us-east-1 + """ - Returns: - Dictionary of catalog properties suitable for pyiceberg.catalog.load_catalog() - """ - properties: dict[str, str] = { - "type": self.catalog_type, - } + catalog_type: Literal["glue"] = "glue" + catalog_uri: str | None = Field( + default=None, description="Optional URI for the Glue catalog" + ) + warehouse_path: str | None = Field( + default=None, description="S3 path for the data warehouse" + ) + glue: GlueConfig = Field(default_factory=GlueConfig) - # Add URI if provided + def get_catalog_properties(self) -> dict[str, str]: + props = self._get_base_properties() if self.catalog_uri: - properties["uri"] = self.catalog_uri - - # Add warehouse path if provided + props["uri"] = self.catalog_uri if self.warehouse_path: - properties["warehouse"] = self.warehouse_path - - # Add AWS region for Glue - if self.catalog_type == "glue" and self.aws_region: - properties["region"] = self.aws_region - - # Add REST authentication - if self.catalog_type == "rest": - if self.rest_token: - properties["token"] = self.rest_token - if self.rest_credential: - properties["credential"] = self.rest_credential - - # Add SQL-specific settings - if self.catalog_type == "sql": - properties["echo"] = str(self.sql_echo).lower() - - # Add S3/MinIO storage settings - if self.s3_endpoint: - properties["s3.endpoint"] = self.s3_endpoint - if self.s3_access_key_id: - properties["s3.access-key-id"] = self.s3_access_key_id - if self.s3_secret_access_key: - properties["s3.secret-access-key"] = self.s3_secret_access_key - # Always set path-style-access when S3 endpoint is configured - if self.s3_endpoint: - properties["s3.path-style-access"] = str(self.s3_path_style_access).lower() - - # Include any extra fields from environment (for catalog-specific properties) - for key, value in self.model_extra.items() if self.model_extra else []: - if value is not None: - properties[key] = str(value) + props["warehouse"] = self.warehouse_path + if self.glue.region: + props["region"] = self.glue.region + return props - return properties - def load_catalog(self) -> Catalog: - """ - Create and return a PyIceberg Catalog instance using the configured settings. +class HiveCatalogConfig(_BaseCatalogConfig): + """ + Configuration for Hive-based Iceberg catalogs. + + Examples: + ICESTAC_CATALOG_TYPE=hive + ICESTAC_CATALOG_URI=thrift://hive-metastore:9083 + ICESTAC_WAREHOUSE_PATH=s3://my-bucket/warehouse/ + """ + + catalog_type: Literal["hive"] = "hive" + catalog_uri: str = Field(description="URI for the Hive metastore") + warehouse_path: str = Field(description="Base path for the data warehouse") + + def get_catalog_properties(self) -> dict[str, str]: + props = self._get_base_properties() + props["uri"] = self.catalog_uri + props["warehouse"] = self.warehouse_path + return props - Returns: - PyIceberg Catalog instance configured with the specified properties - Example: - >>> settings = IcebergCatalogSettings() - >>> catalog = settings.load_catalog() - >>> catalog.list_namespaces() - """ - properties = self.get_catalog_properties() - return load_catalog(self.catalog_name, **properties) +IcestacCatalogConfig = Annotated[ + SqlCatalogConfig | RestCatalogConfig | GlueCatalogConfig | HiveCatalogConfig, + Field(discriminator="catalog_type"), +] diff --git a/src/icestac/item_table.py b/src/icestac/item_table.py deleted file mode 100644 index 5adc36f..0000000 --- a/src/icestac/item_table.py +++ /dev/null @@ -1,62 +0,0 @@ -from arro3.core import Schema as ArrowSchema -from pyiceberg.catalog import Catalog -from pyiceberg.partitioning import PartitionField, PartitionSpec -from pyiceberg.table import Table -from pyiceberg.transforms import MonthTransform - -from icestac.constants import DEFAULT_NAMESPACE -from icestac.errors import InvalidCollectionIdError -from icestac.schema import convert_schema, validate_schema - - -def validate_collection_id(collection_id: str) -> None: - """Ensure collection id is valid for icestac schema""" - - if "." in collection_id: - raise InvalidCollectionIdError - - -def create_item_table( - arrow_schema: ArrowSchema, - collection_id: str, - catalog: Catalog, - namespace: str = DEFAULT_NAMESPACE, -) -> Table: - """ - Create an Iceberg table from a stac-geoparquet Arrow schema - - Converts the Arrow schema to an Iceberg schema with manually assigned field IDs, - then creates or loads the Iceberg table partitioned by datetime month. - - Args: - schema: arro3.core.Schema for the items in this collection - collection_id: the collection id for the items in this table - catalog: PyIceberg catalog instance - namespace: Namespace for the Iceberg table - - Returns: - PyIceberg Table instance - - """ - validate_collection_id(collection_id) - validate_schema(arrow_schema) - - catalog.create_namespace_if_not_exists(namespace) - - # TODO: check if collection record is present in collections table - - iceberg_schema = convert_schema(arrow_schema) - - return catalog.create_table( - identifier=f"{namespace}.{collection_id}", - schema=iceberg_schema, - partition_spec=PartitionSpec( - # TODO: make temporal partitioning configurable - PartitionField( - source_id=iceberg_schema.find_field("datetime").field_id, - field_id=1000, - transform=MonthTransform(), - name="datetime_month", - ) - ), - ) diff --git a/src/icestac/lambda_handler.py b/src/icestac/lambda_handler.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/conftest.py b/tests/conftest.py index d8ab3c6..56308bb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,14 +1,16 @@ import gc import tempfile from pathlib import Path -from typing import Any +from typing import Any, Generator import pyarrow import pytest from pyarrow import Table -from pyiceberg.catalog.sql import SqlCatalog from rustac import to_arrow +from icestac.catalog import IcestacCatalog +from icestac.config import SqlCatalogConfig + @pytest.fixture def temp_warehouse(): @@ -18,29 +20,25 @@ def temp_warehouse(): @pytest.fixture -def test_catalog(temp_warehouse): - """Create an in-memory SQL catalog for testing.""" - catalog = SqlCatalog( - "test_catalog", - **{ - "uri": f"sqlite:///{temp_warehouse}/catalog.db", - "warehouse": f"file://{temp_warehouse}", - }, +def test_config(temp_warehouse: Path) -> SqlCatalogConfig: + return SqlCatalogConfig( + catalog_name="test_catalog", + catalog_uri=f"sqlite:///{temp_warehouse}/catalog.db", + warehouse_path=str(temp_warehouse), ) - # Create test namespace - catalog.create_namespace("test_namespace") + +@pytest.fixture +def test_catalog( + test_config: SqlCatalogConfig, +) -> Generator[IcestacCatalog, None, None]: + """Create an in-memory SQL catalog for testing.""" + catalog = IcestacCatalog.from_config(test_config) yield catalog gc.collect() - catalog.close() - - -@pytest.fixture -def test_namespace(): - """Provide a test namespace name.""" - return "test_namespace" + catalog.catalog.close() @pytest.fixture diff --git a/tests/test_item_table.py b/tests/test_catalog.py similarity index 62% rename from tests/test_item_table.py rename to tests/test_catalog.py index df0b31e..f47d473 100644 --- a/tests/test_item_table.py +++ b/tests/test_catalog.py @@ -2,26 +2,22 @@ import pyarrow import pytest -from pyiceberg.catalog import Catalog from pyiceberg.exceptions import TableAlreadyExistsError from rustac import to_arrow +from icestac.catalog import IcestacCatalog from icestac.errors import InvalidCollectionIdError -from icestac.item_table import create_item_table from icestac.schema import enforce_required_fields, get_schema_from_item def test_create_item_table( - test_catalog: Catalog, - test_namespace: str, + test_catalog: IcestacCatalog, sample_stac_items: list[dict[str, Any]], ) -> None: arrow_schema = get_schema_from_item(sample_stac_items[0]) - table = create_item_table( + table = test_catalog.create_item_table( arrow_schema=arrow_schema, collection_id=sample_stac_items[0]["collection"], - catalog=test_catalog, - namespace=test_namespace, ) assert table.schema().find_field("datetime") @@ -42,24 +38,44 @@ def test_create_item_table( assert result.column("id").to_pylist() == [item["id"] for item in sample_stac_items] with pytest.raises(TableAlreadyExistsError): - create_item_table( + test_catalog.create_item_table( arrow_schema=arrow_schema, collection_id=sample_stac_items[0]["collection"], - catalog=test_catalog, - namespace=test_namespace, ) def test_create_item_table_bad_collection_id( - test_catalog: Catalog, - test_namespace: str, + test_catalog: IcestacCatalog, sample_stac_items: list[dict[str, Any]], ) -> None: arrow_schema = get_schema_from_item(sample_stac_items[0]) with pytest.raises(InvalidCollectionIdError): - create_item_table( + test_catalog.create_item_table( arrow_schema=arrow_schema, collection_id="bad.collection", - catalog=test_catalog, - namespace=test_namespace, ) + + +def test_load_items( + test_catalog: IcestacCatalog, + sample_stac_items: list[dict[str, Any]], +) -> None: + collection_id = sample_stac_items[0]["collection"] + arrow_schema = get_schema_from_item(sample_stac_items[0]) + table = test_catalog.create_item_table( + arrow_schema=arrow_schema, + collection_id=collection_id, + ) + + test_catalog.load_items( + collection_id=sample_stac_items[0]["collection"], + items=sample_stac_items, + method="upsert", + ) + + table.refresh() + + # Verify records were inserted + result = table.scan().to_arrow() + assert len(result) == len(sample_stac_items) + assert result.column("id").to_pylist() == [item["id"] for item in sample_stac_items] diff --git a/tests/test_config.py b/tests/test_config.py index 919557d..87b4ef2 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -5,20 +5,18 @@ import pytest from pydantic import ValidationError -from icestac.config import IcebergCatalogConfig +from icestac.config import GlueCatalogConfig, RestCatalogConfig, SqlCatalogConfig -class TestIcebergCatalogConfig: +class TestIcestacCatalogConfig: """Test Iceberg catalog settings configuration.""" def test_default_settings(self, monkeypatch, tmp_path): - """Test default settings values with minimal valid configuration.""" - # Clear any existing environment variables + """Test default settings values with minimal valid SQL configuration.""" for key in os.environ.copy(): if key.startswith("ICESTAC_"): monkeypatch.delenv(key, raising=False) - # Set minimal required configuration for SQL catalog (the default) catalog_db = tmp_path / "catalog.db" warehouse_path = tmp_path / "warehouse" warehouse_path.mkdir() @@ -26,24 +24,21 @@ def test_default_settings(self, monkeypatch, tmp_path): monkeypatch.setenv("ICESTAC_CATALOG_URI", f"sqlite:///{catalog_db}") monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", str(warehouse_path)) - # Disable .env file reading to avoid pollution from project .env file - settings = IcebergCatalogConfig(_env_file=None) # ty: ignore[unknown-argument] + settings = SqlCatalogConfig.model_validate({}) - # Verify defaults are applied assert settings.catalog_name == "default" assert settings.catalog_type == "sql" - assert settings.sql_echo is False - assert settings.aws_region is None - assert settings.rest_token is None + assert settings.sql.echo is False def test_sql_catalog_properties(self, monkeypatch): """Test SQL catalog configuration.""" monkeypatch.setenv("ICESTAC_CATALOG_NAME", "test_catalog") - monkeypatch.setenv("ICESTAC_CATALOG_TYPE", "sql") monkeypatch.setenv("ICESTAC_CATALOG_URI", "sqlite:///catalog.db") monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", "/tmp/warehouse") - settings = IcebergCatalogConfig() + settings = SqlCatalogConfig.model_validate( + {}, + ) assert settings.catalog_name == "test_catalog" assert settings.catalog_type == "sql" @@ -59,14 +54,15 @@ def test_sql_catalog_properties(self, monkeypatch): def test_rest_catalog_properties(self, monkeypatch): """Test REST catalog configuration.""" monkeypatch.setenv("ICESTAC_CATALOG_NAME", "rest_catalog") - monkeypatch.setenv("ICESTAC_CATALOG_TYPE", "rest") monkeypatch.setenv("ICESTAC_CATALOG_URI", "https://rest.example.com") - monkeypatch.setenv("ICESTAC_REST_TOKEN", "my-token") + monkeypatch.setenv("ICESTAC_REST__TOKEN", "my-token") - settings = IcebergCatalogConfig() + settings = RestCatalogConfig.model_validate( + {}, + ) assert settings.catalog_type == "rest" - assert settings.rest_token == "my-token" + assert settings.rest.token == "my-token" properties = settings.get_catalog_properties() assert properties["type"] == "rest" @@ -76,14 +72,13 @@ def test_rest_catalog_properties(self, monkeypatch): def test_glue_catalog_properties(self, monkeypatch): """Test AWS Glue catalog configuration.""" monkeypatch.setenv("ICESTAC_CATALOG_NAME", "glue_catalog") - monkeypatch.setenv("ICESTAC_CATALOG_TYPE", "glue") monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", "s3://bucket/warehouse") - monkeypatch.setenv("ICESTAC_AWS_REGION", "us-west-2") + monkeypatch.setenv("ICESTAC_GLUE__REGION", "us-west-2") - settings = IcebergCatalogConfig() + settings = GlueCatalogConfig() assert settings.catalog_type == "glue" - assert settings.aws_region == "us-west-2" + assert settings.glue.region == "us-west-2" properties = settings.get_catalog_properties() assert properties["type"] == "glue" @@ -92,77 +87,58 @@ def test_glue_catalog_properties(self, monkeypatch): def test_sql_catalog_requires_uri(self, monkeypatch): """Test that SQL catalog requires a URI.""" - monkeypatch.setenv("ICESTAC_CATALOG_TYPE", "sql") monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", "/tmp/warehouse") with pytest.raises(ValidationError) as exc_info: - IcebergCatalogConfig(_env_file=None) # ty: ignore[unknown-argument] + SqlCatalogConfig.model_validate( + {}, + ) - assert "catalog_uri is required" in str(exc_info.value) + assert "catalog_uri" in str(exc_info.value) def test_sql_catalog_requires_warehouse_path(self, monkeypatch): """Test that SQL catalog requires a warehouse path.""" - monkeypatch.setenv("ICESTAC_CATALOG_TYPE", "sql") monkeypatch.setenv("ICESTAC_CATALOG_URI", "sqlite:///catalog.db") with pytest.raises(ValidationError) as exc_info: - IcebergCatalogConfig(_env_file=None) # ty: ignore[unknown-argument] + SqlCatalogConfig.model_validate( + {}, + ) - assert "warehouse_path is required" in str(exc_info.value) + assert "warehouse_path" in str(exc_info.value) def test_rest_catalog_requires_uri(self, monkeypatch): """Test that REST catalog requires a URI.""" - monkeypatch.setenv("ICESTAC_CATALOG_TYPE", "rest") - with pytest.raises(ValidationError) as exc_info: - IcebergCatalogConfig(_env_file=None) # ty: ignore[unknown-argument] + RestCatalogConfig.model_validate( + {}, + ) - assert "catalog_uri is required" in str(exc_info.value) + assert "catalog_uri" in str(exc_info.value) def test_case_insensitive_env_vars(self, monkeypatch): """Test that environment variables are case-insensitive.""" monkeypatch.setenv("icestac_catalog_name", "test") - monkeypatch.setenv("ICESTAC_CATALOG_TYPE", "sql") monkeypatch.setenv("icestac_catalog_uri", "sqlite:///catalog.db") monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", "/tmp/warehouse") - settings = IcebergCatalogConfig() + settings = SqlCatalogConfig.model_validate( + {}, + ) assert settings.catalog_name == "test" def test_sql_echo_enabled(self, monkeypatch): """Test SQL echo setting.""" - monkeypatch.setenv("ICESTAC_CATALOG_TYPE", "sql") monkeypatch.setenv("ICESTAC_CATALOG_URI", "sqlite:///catalog.db") monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", "/tmp/warehouse") - monkeypatch.setenv("ICESTAC_SQL_ECHO", "true") + monkeypatch.setenv("ICESTAC_SQL__ECHO", "true") - settings = IcebergCatalogConfig() + settings = SqlCatalogConfig.model_validate( + {}, + ) - assert settings.sql_echo is True + assert settings.sql.echo is True properties = settings.get_catalog_properties() assert properties["echo"] == "true" - - def test_load_catalog(self, tmp_path, monkeypatch): - """Test loading a PyIceberg catalog from settings.""" - catalog_db = tmp_path / "catalog.db" - warehouse_path = tmp_path / "warehouse" - warehouse_path.mkdir() - - monkeypatch.setenv("ICESTAC_CATALOG_NAME", "test_catalog") - monkeypatch.setenv("ICESTAC_CATALOG_TYPE", "sql") - monkeypatch.setenv("ICESTAC_CATALOG_URI", f"sqlite:///{catalog_db}") - monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", str(warehouse_path)) - - settings = IcebergCatalogConfig() - catalog = settings.load_catalog() - - # Verify catalog is created successfully - assert catalog is not None - assert catalog.name == "test_catalog" - - # Test basic catalog operations - catalog.create_namespace("test") - namespaces = catalog.list_namespaces() - assert ("test",) in namespaces diff --git a/tests/test_load.py b/tests/test_load.py index f8e6a42..b6a6595 100644 --- a/tests/test_load.py +++ b/tests/test_load.py @@ -1,26 +1,22 @@ from typing import Any import pytest -from pyiceberg.catalog import Catalog -from icestac.item_table import create_item_table +from icestac.catalog import IcestacCatalog from icestac.load import load_items from icestac.schema import get_schema_from_item def test_load_items_upsert_default( - test_catalog: Catalog, - test_namespace: str, + test_catalog: IcestacCatalog, sample_stac_items: list[dict[str, Any]], ) -> None: """Test loading items with default upsert method.""" # Create the table arrow_schema = get_schema_from_item(sample_stac_items[0]) - table = create_item_table( + table = test_catalog.create_item_table( arrow_schema=arrow_schema, collection_id=sample_stac_items[0]["collection"], - catalog=test_catalog, - namespace=test_namespace, ) # Load items (default method is upsert) @@ -35,18 +31,15 @@ def test_load_items_upsert_default( def test_load_items_upsert_explicit( - test_catalog: Catalog, - test_namespace: str, + test_catalog: IcestacCatalog, sample_stac_items: list[dict[str, Any]], ) -> None: """Test loading items with explicit upsert method.""" # Create the table arrow_schema = get_schema_from_item(sample_stac_items[0]) - table = create_item_table( + table = test_catalog.create_item_table( arrow_schema=arrow_schema, collection_id=sample_stac_items[0]["collection"], - catalog=test_catalog, - namespace=test_namespace, ) # Load items with explicit upsert method @@ -58,18 +51,15 @@ def test_load_items_upsert_explicit( def test_load_items_upsert_updates_existing( - test_catalog: Catalog, - test_namespace: str, + test_catalog: IcestacCatalog, sample_stac_items: list[dict[str, Any]], ) -> None: """Test that upsert updates existing records with same ID.""" # Create the table arrow_schema = get_schema_from_item(sample_stac_items[0]) - table = create_item_table( + table = test_catalog.create_item_table( arrow_schema=arrow_schema, collection_id=sample_stac_items[0]["collection"], - catalog=test_catalog, - namespace=test_namespace, ) # Load initial items @@ -96,18 +86,15 @@ def test_load_items_upsert_updates_existing( def test_load_items_append( - test_catalog: Catalog, - test_namespace: str, + test_catalog: IcestacCatalog, sample_stac_items: list[dict[str, Any]], ) -> None: """Test loading items with append method.""" # Create the table arrow_schema = get_schema_from_item(sample_stac_items[0]) - table = create_item_table( + table = test_catalog.create_item_table( arrow_schema=arrow_schema, collection_id=sample_stac_items[0]["collection"], - catalog=test_catalog, - namespace=test_namespace, ) # Load items with append method @@ -119,18 +106,15 @@ def test_load_items_append( def test_load_items_append_creates_duplicates( - test_catalog: Catalog, - test_namespace: str, + test_catalog: IcestacCatalog, sample_stac_items: list[dict[str, Any]], ) -> None: """Test that append creates duplicate records when IDs overlap.""" # Create the table arrow_schema = get_schema_from_item(sample_stac_items[0]) - table = create_item_table( + table = test_catalog.create_item_table( arrow_schema=arrow_schema, collection_id=sample_stac_items[0]["collection"], - catalog=test_catalog, - namespace=test_namespace, ) # Load items twice with append @@ -143,18 +127,15 @@ def test_load_items_append_creates_duplicates( def test_load_items_multiple_batches( - test_catalog: Catalog, - test_namespace: str, + test_catalog: IcestacCatalog, sample_stac_items: list[dict[str, Any]], ) -> None: """Test loading items in multiple batches with different methods.""" # Create the table arrow_schema = get_schema_from_item(sample_stac_items[0]) - table = create_item_table( + table = test_catalog.create_item_table( arrow_schema=arrow_schema, collection_id=sample_stac_items[0]["collection"], - catalog=test_catalog, - namespace=test_namespace, ) # Load first batch @@ -171,18 +152,15 @@ def test_load_items_multiple_batches( def test_load_items_single_item( - test_catalog: Catalog, - test_namespace: str, + test_catalog: IcestacCatalog, sample_stac_item: dict[str, Any], ) -> None: """Test loading a single item.""" # Create the table arrow_schema = get_schema_from_item(sample_stac_item) - table = create_item_table( + table = test_catalog.create_item_table( arrow_schema=arrow_schema, collection_id=sample_stac_item["collection"], - catalog=test_catalog, - namespace=test_namespace, ) # Load single item as a list @@ -195,18 +173,15 @@ def test_load_items_single_item( def test_load_items_different_schema( - test_catalog: Catalog, - test_namespace: str, + test_catalog: IcestacCatalog, sample_stac_item: dict[str, Any], ) -> None: # Create the table arrow_schema = get_schema_from_item(sample_stac_item) - table = create_item_table( + table = test_catalog.create_item_table( arrow_schema=arrow_schema, collection_id=sample_stac_item["collection"], - catalog=test_catalog, - namespace=test_namespace, ) # load an item From 17e3d1d23a0078c4794a5ff851003b8136b97f3f Mon Sep 17 00:00:00 2001 From: hrodmn Date: Sat, 28 Feb 2026 10:24:37 -0600 Subject: [PATCH 08/23] chore(ci): add codecov report --- .github/workflows/ci.yml | 8 +++- pyproject.toml | 3 +- tests/test_config.py | 82 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 90 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0751afb..755eb91 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,4 +30,10 @@ jobs: run: uv run pre-commit run --all-files - name: Run tests - run: uv run pytest + run: uv run pytest --cov-report=xml + + - name: Upload coverage to Codecov + if: matrix.python-version == '3.14' + uses: codecov/codecov-action@v5 + with: + files: coverage.xml diff --git a/pyproject.toml b/pyproject.toml index 54d5f6b..b509623 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,8 @@ dev = [ addopts = [ "-v", "--cov-config=pyproject.toml", - "--cov=src" + "--cov=src", + "--cov-report=term-missing" ] filterwarnings = [ "ignore::DeprecationWarning:pyiceberg.*", diff --git a/tests/test_config.py b/tests/test_config.py index 87b4ef2..7c38fc0 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -5,7 +5,12 @@ import pytest from pydantic import ValidationError -from icestac.config import GlueCatalogConfig, RestCatalogConfig, SqlCatalogConfig +from icestac.config import ( + GlueCatalogConfig, + HiveCatalogConfig, + RestCatalogConfig, + SqlCatalogConfig, +) class TestIcestacCatalogConfig: @@ -142,3 +147,78 @@ def test_sql_echo_enabled(self, monkeypatch): properties = settings.get_catalog_properties() assert properties["echo"] == "true" + + def test_s3_endpoint_properties(self, monkeypatch): + """Test that S3 endpoint and path-style access are included in properties.""" + monkeypatch.setenv("ICESTAC_CATALOG_URI", "sqlite:///catalog.db") + monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", "/tmp/warehouse") + monkeypatch.setenv("ICESTAC_S3__ENDPOINT", "http://minio:9000") + monkeypatch.setenv("ICESTAC_S3__PATH_STYLE_ACCESS", "true") + + settings = SqlCatalogConfig.model_validate({}) + properties = settings.get_catalog_properties() + + assert properties["s3.endpoint"] == "http://minio:9000" + assert properties["s3.path-style-access"] == "true" + + def test_s3_credentials_properties(self, monkeypatch): + """Test that S3 access key and secret are included in properties.""" + monkeypatch.setenv("ICESTAC_CATALOG_URI", "sqlite:///catalog.db") + monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", "/tmp/warehouse") + monkeypatch.setenv("ICESTAC_S3__ACCESS_KEY_ID", "mykey") + monkeypatch.setenv("ICESTAC_S3__SECRET_ACCESS_KEY", "mysecret") + + settings = SqlCatalogConfig.model_validate({}) + properties = settings.get_catalog_properties() + + assert properties["s3.access-key-id"] == "mykey" + assert properties["s3.secret-access-key"] == "mysecret" + + def test_extra_properties_passthrough(self, monkeypatch): + """Test that extra fields passed to the model are included in catalog properties.""" + monkeypatch.setenv("ICESTAC_CATALOG_URI", "sqlite:///catalog.db") + monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", "/tmp/warehouse") + + settings = SqlCatalogConfig.model_validate({"some_extra_key": "extra_value"}) + properties = settings.get_catalog_properties() + + assert properties["some_extra_key"] == "extra_value" + + def test_rest_catalog_with_warehouse_and_credential(self, monkeypatch): + """Test REST catalog with optional warehouse path and credential.""" + monkeypatch.setenv("ICESTAC_CATALOG_URI", "https://rest.example.com") + monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", "s3://bucket/warehouse") + monkeypatch.setenv("ICESTAC_REST__CREDENTIAL", "client_id:client_secret") + + settings = RestCatalogConfig.model_validate({}) + properties = settings.get_catalog_properties() + + assert properties["warehouse"] == "s3://bucket/warehouse" + assert properties["credential"] == "client_id:client_secret" + + def test_glue_catalog_with_uri(self, monkeypatch): + """Test AWS Glue catalog with optional catalog URI.""" + monkeypatch.setenv("ICESTAC_CATALOG_URI", "glue://my-catalog") + monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", "s3://bucket/warehouse") + + settings = GlueCatalogConfig.model_validate({}) + properties = settings.get_catalog_properties() + + assert properties["uri"] == "glue://my-catalog" + assert properties["warehouse"] == "s3://bucket/warehouse" + + def test_hive_catalog_properties(self, monkeypatch): + """Test Hive catalog configuration and properties.""" + monkeypatch.setenv("ICESTAC_CATALOG_URI", "thrift://hive-metastore:9083") + monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", "s3://bucket/warehouse") + + settings = HiveCatalogConfig.model_validate({}) + + assert settings.catalog_type == "hive" + assert settings.catalog_uri == "thrift://hive-metastore:9083" + assert settings.warehouse_path == "s3://bucket/warehouse" + + properties = settings.get_catalog_properties() + assert properties["type"] == "hive" + assert properties["uri"] == "thrift://hive-metastore:9083" + assert properties["warehouse"] == "s3://bucket/warehouse" From 3e9a22520f8f0a752d8ee944d5d49a65c908ad56 Mon Sep 17 00:00:00 2001 From: hrodmn Date: Sat, 28 Feb 2026 10:38:27 -0600 Subject: [PATCH 09/23] chore(ci): use codecov token --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 755eb91..6d3fa42 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,4 +36,5 @@ jobs: if: matrix.python-version == '3.14' uses: codecov/codecov-action@v5 with: + token: ${{ secrets.CODECOV_TOKEN }} files: coverage.xml From c63573c27a3c25900d60a6500d62d6968d5bd101 Mon Sep 17 00:00:00 2001 From: hrodmn Date: Sat, 28 Feb 2026 10:45:22 -0600 Subject: [PATCH 10/23] chore(ci): add --cov-branch --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6d3fa42..3e6c289 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: run: uv run pre-commit run --all-files - name: Run tests - run: uv run pytest --cov-report=xml + run: uv run pytest --cov-branch --cov-report=xml - name: Upload coverage to Codecov if: matrix.python-version == '3.14' From 179416b59fcbcf71b450949427dd561a9cd10a97 Mon Sep 17 00:00:00 2001 From: hrodmn Date: Sat, 28 Feb 2026 10:49:17 -0600 Subject: [PATCH 11/23] docs: use .env-local for interacting with docker network --- .env-local | 12 ++++++++++++ README.md | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 .env-local diff --git a/.env-local b/.env-local new file mode 100644 index 0000000..7ef3d3a --- /dev/null +++ b/.env-local @@ -0,0 +1,12 @@ +# Iceberg Catalog Configuration +ICESTAC_CATALOG_NAME=rest_catalog +ICESTAC_CATALOG_TYPE=rest +ICESTAC_CATALOG_URI=http://localhost:8181 +ICESTAC_WAREHOUSE_PATH=s3://warehouse/ + +# S3/MinIO Storage Configuration +# These settings allow PyIceberg to directly access MinIO for data file I/O +ICESTAC_S3_ENDPOINT=http://localhost:9000 +ICESTAC_S3_ACCESS_KEY_ID=admin +ICESTAC_S3_SECRET_ACCESS_KEY=password +ICESTAC_S3_PATH_STYLE_ACCESS=true diff --git a/README.md b/README.md index d9a6765..05541c2 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ uv run pytest ### Local Instance -**Environment Configuration** (`.env`): +**Environment Configuration** (`.env-local`): ```bash # REST catalog endpoint ICESTAC_CATALOG_NAME=rest_catalog From c2c57355e82379f14af27d23bdcffcc122f6b115 Mon Sep 17 00:00:00 2001 From: hrodmn Date: Tue, 3 Mar 2026 07:16:50 -0600 Subject: [PATCH 12/23] apply feedback from review, defer config to pyiceberg --- README.md | 55 +++++----- main.py | 5 +- pyproject.toml | 1 - src/icestac/catalog.py | 39 ++++--- src/icestac/config.py | 207 ------------------------------------- src/icestac/load.py | 4 +- src/icestac/schema.py | 128 +++++++++++------------ tests/conftest.py | 30 +++--- tests/test_catalog.py | 4 +- tests/test_config.py | 224 ----------------------------------------- tests/test_schema.py | 8 +- uv.lock | 25 ----- 12 files changed, 129 insertions(+), 601 deletions(-) delete mode 100644 src/icestac/config.py delete mode 100644 tests/test_config.py diff --git a/README.md b/README.md index 05541c2..b4bc7f9 100644 --- a/README.md +++ b/README.md @@ -17,21 +17,32 @@ uv run pytest ### Local Instance -**Environment Configuration** (`.env-local`): +**Catalog Configuration** (`.pyiceberg.yaml`): + +icestac delegates catalog configuration to PyIceberg. Create a `.pyiceberg.yaml` in your working directory (or `~/.pyiceberg.yaml` for a user-wide default): + +```yaml +catalog: + default: + type: rest + uri: http://localhost:8181 + warehouse: s3://warehouse/ + s3.endpoint: http://localhost:9000 + s3.access-key-id: admin + s3.secret-access-key: password + s3.path-style-access: "true" +``` + +Alternatively, configure via environment variables using PyIceberg's `PYICEBERG_CATALOG____` prefix: + ```bash -# REST catalog endpoint -ICESTAC_CATALOG_NAME=rest_catalog -ICESTAC_CATALOG_TYPE=rest -ICESTAC_CATALOG_URI=http://localhost:8181 -ICESTAC_WAREHOUSE_PATH=s3://warehouse/ - -# S3/MinIO storage for PyIceberg data file I/O -ICESTAC_S3_ENDPOINT=http://localhost:9000 -ICESTAC_S3_ACCESS_KEY_ID=admin -ICESTAC_S3_SECRET_ACCESS_KEY=password -ICESTAC_S3_PATH_STYLE_ACCESS=true +PYICEBERG_CATALOG__DEFAULT__TYPE=rest +PYICEBERG_CATALOG__DEFAULT__URI=http://localhost:8181 +PYICEBERG_CATALOG__DEFAULT__WAREHOUSE=s3://warehouse/ ``` +See the [PyIceberg configuration docs](https://py.iceberg.apache.org/configuration/) for the full list of catalog and S3 properties. + **Starting the local environment:** ```bash docker compose up @@ -117,22 +128,6 @@ Schema validation and enforcement: **`validate_schema(schema: Schema) -> None`** - Validates Arrow schema contains all required STAC fields -#### Config Module (`src/icestac/config.py`) - ✓ IMPLEMENTED - -**`IcebergCatalogConfig`** - Pydantic settings for catalog configuration -- Loads from environment variables with `ICESTAC_` prefix -- Supports catalog types: `rest`, `glue`, `hive`, `sql` -- Environment variables: - - Catalog: `CATALOG_NAME`, `CATALOG_TYPE`, `CATALOG_URI`, `WAREHOUSE_PATH`, `AWS_REGION` - - REST auth: `REST_TOKEN`, `REST_CREDENTIAL` - - SQL: `SQL_ECHO` - - S3/MinIO: `S3_ENDPOINT`, `S3_ACCESS_KEY_ID`, `S3_SECRET_ACCESS_KEY`, `S3_PATH_STYLE_ACCESS` - -**`get_catalog_properties() -> dict[str, str]`** -- Generates PyIceberg catalog properties from settings - -**`load_catalog() -> Catalog`** -- Factory method that creates PyIceberg Catalog instance #### Lambda Handler Module (`src/icestac/lambda_handler.py`) - NOT IMPLEMENTED @@ -144,7 +139,6 @@ Placeholder for AWS Lambda handler. - **`tests/conftest.py`**: Pytest fixtures for test catalog, sample STAC items, and Arrow tables - **`tests/test_item_table.py`**: Unit tests for `sanitize_collection_id` and `create_item_table` -- **`tests/test_config.py`**: Unit tests for catalog configuration and settings validation - **`tests/test_schema.py`**: Unit tests for schema validation and enforcement ### Dependencies @@ -154,7 +148,6 @@ Placeholder for AWS Lambda handler. - `pyiceberg[pyiceberg-core]>=0.10.0` - Iceberg table management - `rustac[arrow]>=0.9.3` - STAC to Arrow conversion with arro3 schemas - `stac-pydantic>=3.4.0` - STAC item validation -- `pydantic-settings>=2.12.0` - Environment-based configuration **Development** (dev dependency group): - `pytest>=9.0.2` - Testing framework @@ -241,7 +234,7 @@ infrastructure/ 2. **Partitioning**: Monthly partitioning by datetime field (hardcoded, to be made configurable) 3. **Schema conversion**: Manual field ID assignment to avoid pyiceberg limitations 4. **Schema validation**: Required STAC fields marked as non-nullable using Pydantic models -5. **Configuration**: Environment variables with `ICESTAC_` prefix using Pydantic settings +5. **Configuration**: Delegated to PyIceberg via `.pyiceberg.yaml` or `PYICEBERG_CATALOG__` environment variables 6. **Testing catalog**: In-memory SQL catalog with SQLite for unit tests 7. **STAC to Arrow conversion**: Use rustac library with arro3 schemas diff --git a/main.py b/main.py index d44c704..bcf3dca 100644 --- a/main.py +++ b/main.py @@ -1,15 +1,14 @@ import asyncio import rustac +from pyiceberg.catalog import load_catalog from icestac.catalog import IcestacCatalog -from icestac.config import IcestacCatalogConfig from icestac.schema import get_schema_from_item async def run(): - config = IcestacCatalogConfig.model_validate({}) - catalog = IcestacCatalog.from_config(config) + catalog = IcestacCatalog(catalog=load_catalog()) items = await rustac.search( "https://stac.maap-project.org", diff --git a/pyproject.toml b/pyproject.toml index b509623..ef132a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,6 @@ classifiers = [ requires-python = ">=3.11" dependencies = [ "pyarrow>=23.0.0", - "pydantic-settings>=2.12.0", "pyiceberg[pyiceberg-core]>=0.10.0", "rustac[arrow]>=0.9.3", "stac-pydantic>=3.4.0", diff --git a/src/icestac/catalog.py b/src/icestac/catalog.py index eadd86c..54e7786 100644 --- a/src/icestac/catalog.py +++ b/src/icestac/catalog.py @@ -1,23 +1,25 @@ +from __future__ import annotations + from dataclasses import dataclass from typing import Any from arro3.core import Schema as ArrowSchema -from pyiceberg.catalog import Catalog, load_catalog +from pyiceberg.catalog import Catalog from pyiceberg.partitioning import PartitionField, PartitionSpec from pyiceberg.table import Table from pyiceberg.transforms import MonthTransform -from icestac.config import IcestacCatalogConfig +from icestac.constants import DEFAULT_NAMESPACE from icestac.errors import InvalidCollectionIdError from icestac.load import Method, load_items -from icestac.schema import convert_schema, validate_schema +from icestac.schema import IcestacItem, convert_schema def validate_collection_id(collection_id: str) -> None: """Ensure collection id is valid for icestac schema""" if "." in collection_id: - raise InvalidCollectionIdError + raise InvalidCollectionIdError(collection_id) @dataclass @@ -25,19 +27,10 @@ class IcestacCatalog: """Icestac client class for pyiceberg Catalog""" catalog: Catalog - namespace: str - - def __getattr__(self, name: str) -> Any: - return getattr(self.catalog, name) + namespace: str = DEFAULT_NAMESPACE def __post_init__(self) -> None: - self.create_namespace_if_not_exists(self.namespace) - - @classmethod - def from_config(cls, config: IcestacCatalogConfig) -> "IcestacCatalog": - catalog = load_catalog(config.catalog_name, **config.get_catalog_properties()) - - return cls(catalog=catalog, namespace=config.namespace) + self.catalog.create_namespace_if_not_exists(self.namespace) def create_item_table( self, @@ -61,13 +54,13 @@ def create_item_table( """ validate_collection_id(collection_id) - validate_schema(arrow_schema) + IcestacItem.validate_schema(arrow_schema) # TODO: check if collection record is present in collections table iceberg_schema = convert_schema(arrow_schema) - return self.create_table( + return self.catalog.create_table( identifier=f"{self.namespace}.{collection_id}", schema=iceberg_schema, partition_spec=PartitionSpec( @@ -81,11 +74,13 @@ def create_item_table( ), ) - def load_item_table(self, collection_id: str) -> Table: - """Load the item table for a collection""" - return self.load_table(identifier=f"{self.namespace}.{collection_id}") - def load_items( self, collection_id: str, items: list[dict[str, Any]], method: Method = "upsert" ) -> None: - load_items(items, table=self.load_item_table(collection_id), method=method) + load_items( + items, + table=self.catalog.load_table( + identifier=f"{self.namespace}.{collection_id}" + ), + method=method, + ) diff --git a/src/icestac/config.py b/src/icestac/config.py deleted file mode 100644 index c4a875e..0000000 --- a/src/icestac/config.py +++ /dev/null @@ -1,207 +0,0 @@ -"""Settings module for icestac Iceberg catalog configuration.""" - -from typing import Annotated, Literal - -from pydantic import BaseModel, Field -from pydantic_settings import BaseSettings, SettingsConfigDict - -from icestac.constants import DEFAULT_NAMESPACE - - -class RestConfig(BaseModel): - """Authentication settings for REST catalogs.""" - - token: str | None = Field( - default=None, description="Bearer token for REST catalog authentication" - ) - credential: str | None = Field( - default=None, description="Credential for REST catalog authentication" - ) - - -class S3Config(BaseModel): - """S3/MinIO storage settings.""" - - endpoint: str | None = Field( - default=None, - description="S3 endpoint URL (required for MinIO or custom S3-compatible storage)", - ) - access_key_id: str | None = Field(default=None, description="S3 access key ID") - secret_access_key: str | None = Field( - default=None, description="S3 secret access key" - ) - path_style_access: bool = Field( - default=True, description="Use path-style access for S3 (required for MinIO)" - ) - - -class SqlConfig(BaseModel): - """Settings specific to SQL catalogs.""" - - echo: bool = Field(default=False, description="Enable SQL query logging") - - -class GlueConfig(BaseModel): - """Settings specific to AWS Glue catalogs.""" - - region: str | None = Field(default=None, description="AWS region for Glue catalog") - - -_SETTINGS_CONFIG = SettingsConfigDict( - env_prefix="ICESTAC_", - env_nested_delimiter="__", - case_sensitive=False, - env_file=".env", - env_file_encoding="utf-8", - extra="allow", -) - - -class _BaseCatalogConfig(BaseSettings): - """Base settings shared by all catalog types.""" - - model_config = _SETTINGS_CONFIG - - catalog_type: str # Narrowed to Literal in each subclass - - catalog_name: str = Field( - default="default", description="Name of the Iceberg catalog" - ) - namespace: str = Field( - default=DEFAULT_NAMESPACE, description="Namespace within Iceberg catalog" - ) - s3: S3Config = Field(default_factory=S3Config) - - def _get_base_properties(self) -> dict[str, str]: - """Build properties common to all catalog types.""" - properties: dict[str, str] = {"type": self.catalog_type} - - if self.s3.endpoint: - properties["s3.endpoint"] = self.s3.endpoint - properties["s3.path-style-access"] = str(self.s3.path_style_access).lower() - if self.s3.access_key_id: - properties["s3.access-key-id"] = self.s3.access_key_id - if self.s3.secret_access_key: - properties["s3.secret-access-key"] = self.s3.secret_access_key - - for key, value in (self.model_extra or {}).items(): - if value is not None: - properties[key] = str(value) - - return properties - - -class SqlCatalogConfig(_BaseCatalogConfig): - """ - Configuration for SQL-based Iceberg catalogs (SQLite, PostgreSQL, etc.). - - Examples: - ICESTAC_CATALOG_TYPE=sql - ICESTAC_CATALOG_URI=sqlite:///path/to/catalog.db - ICESTAC_WAREHOUSE_PATH=/path/to/warehouse - ICESTAC_SQL__ECHO=true - """ - - catalog_type: Literal["sql"] = "sql" - catalog_uri: str = Field( - description="URI for the SQL catalog (e.g. sqlite:///path/to/catalog.db)" - ) - warehouse_path: str = Field(description="Base path for the data warehouse") - sql: SqlConfig = Field(default_factory=SqlConfig) - - def get_catalog_properties(self) -> dict[str, str]: - props = self._get_base_properties() - props.update( - { - "uri": self.catalog_uri, - "warehouse": self.warehouse_path, - "echo": str(self.sql.echo).lower(), - } - ) - return props - - -class RestCatalogConfig(_BaseCatalogConfig): - """ - Configuration for REST-based Iceberg catalogs. - - Examples: - ICESTAC_CATALOG_TYPE=rest - ICESTAC_CATALOG_URI=https://iceberg-rest.example.com - ICESTAC_REST__TOKEN=my-token - """ - - catalog_type: Literal["rest"] = "rest" - catalog_uri: str = Field(description="URI for the REST catalog") - warehouse_path: str | None = Field( - default=None, description="Optional warehouse path" - ) - rest: RestConfig = Field(default_factory=RestConfig) - - def get_catalog_properties(self) -> dict[str, str]: - props = self._get_base_properties() - props["uri"] = self.catalog_uri - if self.warehouse_path: - props["warehouse"] = self.warehouse_path - if self.rest.token: - props["token"] = self.rest.token - if self.rest.credential: - props["credential"] = self.rest.credential - return props - - -class GlueCatalogConfig(_BaseCatalogConfig): - """ - Configuration for AWS Glue Iceberg catalogs. - - Examples: - ICESTAC_CATALOG_TYPE=glue - ICESTAC_WAREHOUSE_PATH=s3://my-bucket/warehouse/ - ICESTAC_GLUE__REGION=us-east-1 - """ - - catalog_type: Literal["glue"] = "glue" - catalog_uri: str | None = Field( - default=None, description="Optional URI for the Glue catalog" - ) - warehouse_path: str | None = Field( - default=None, description="S3 path for the data warehouse" - ) - glue: GlueConfig = Field(default_factory=GlueConfig) - - def get_catalog_properties(self) -> dict[str, str]: - props = self._get_base_properties() - if self.catalog_uri: - props["uri"] = self.catalog_uri - if self.warehouse_path: - props["warehouse"] = self.warehouse_path - if self.glue.region: - props["region"] = self.glue.region - return props - - -class HiveCatalogConfig(_BaseCatalogConfig): - """ - Configuration for Hive-based Iceberg catalogs. - - Examples: - ICESTAC_CATALOG_TYPE=hive - ICESTAC_CATALOG_URI=thrift://hive-metastore:9083 - ICESTAC_WAREHOUSE_PATH=s3://my-bucket/warehouse/ - """ - - catalog_type: Literal["hive"] = "hive" - catalog_uri: str = Field(description="URI for the Hive metastore") - warehouse_path: str = Field(description="Base path for the data warehouse") - - def get_catalog_properties(self) -> dict[str, str]: - props = self._get_base_properties() - props["uri"] = self.catalog_uri - props["warehouse"] = self.warehouse_path - return props - - -IcestacCatalogConfig = Annotated[ - SqlCatalogConfig | RestCatalogConfig | GlueCatalogConfig | HiveCatalogConfig, - Field(discriminator="catalog_type"), -] diff --git a/src/icestac/load.py b/src/icestac/load.py index 7202f87..d6e7998 100644 --- a/src/icestac/load.py +++ b/src/icestac/load.py @@ -4,7 +4,7 @@ from pyiceberg.table import Table from rustac import to_arrow -from icestac.schema import enforce_required_fields +from icestac.schema import IcestacItem Method = Literal["append", "upsert"] @@ -15,7 +15,7 @@ def load_items( method: Method = "upsert", ) -> None: arrow_data = to_arrow(items) - enforced_schema = enforce_required_fields(arrow_data.schema) + enforced_schema = IcestacItem.enforce_required_fields(arrow_data.schema) arrow_table = pyarrow.table(arrow_data).cast(pyarrow.schema(enforced_schema)) if method == "upsert": diff --git a/src/icestac/schema.py b/src/icestac/schema.py index eabc427..fdd619b 100644 --- a/src/icestac/schema.py +++ b/src/icestac/schema.py @@ -1,6 +1,7 @@ from typing import Any import pyarrow as pa +from arro3.core import Field from arro3.core import Schema as ArrowSchema from pyiceberg.io.pyarrow import _pyarrow_to_schema_without_ids from pyiceberg.schema import Schema as IcebergSchema @@ -12,88 +13,85 @@ class IcestacItem(Item): collection: str + @classmethod + def get_required_fields(cls) -> set[str]: + """ + Get the set of required field names from IcestacItem. -def get_schema_from_item(item: dict[str, Any]) -> ArrowSchema: - # validate stac item - _ = IcestacItem(**item) + Returns: + Set of required field names, with special handling for flattened properties + """ + required_fields = set() - return enforce_required_fields(to_arrow([item]).schema) + for field_name, field_info in cls.model_fields.items(): + if field_info.is_required(): + required_fields.add(field_name) + # Special handling: rustac flattens properties.datetime to just "datetime" + if "properties" in required_fields: + required_fields.remove("properties") + required_fields.add("datetime") -def get_required_fields() -> set[str]: - """ - Get the set of required field names from IcestacItem. + return required_fields - Returns: - Set of required field names, with special handling for flattened properties - """ - required_fields = set() + @classmethod + def enforce_required_fields(cls, schema: ArrowSchema) -> ArrowSchema: + """ + Ensure required STAC fields are marked as non-nullable in the Arrow schema. - for field_name, field_info in IcestacItem.model_fields.items(): - if field_info.is_required(): - required_fields.add(field_name) + Returns a schema with required fields marked as nullable=False. This ensures + the Iceberg table will enforce these fields as required. - # Special handling: rustac flattens properties.datetime to just "datetime" - if "properties" in required_fields: - required_fields.remove("properties") - required_fields.add("datetime") + Args: + schema: arro3.core.Schema from rustac - return required_fields + Returns: + arro3.core.Schema with required fields marked as non-nullable + """ + required_fields = cls.get_required_fields() + new_fields = [] -def enforce_required_fields(schema: ArrowSchema) -> ArrowSchema: - """ - Ensure required STAC fields are marked as non-nullable in the Arrow schema. + for field in schema: + if field.name in required_fields: + # Mark as non-nullable (required) + new_fields.append( + Field(name=field.name, type=field.type, nullable=False) + ) + else: + # Keep original nullable setting + new_fields.append(field) - Takes an arro3 Schema and returns a pyarrow Schema with required fields - marked as nullable=False. This ensures the Iceberg table will enforce - these fields as required. + return ArrowSchema(fields=new_fields) - Args: - schema: arro3.core.Schema from rustac + @classmethod + def validate_schema(cls, schema: ArrowSchema) -> None: + """ + Validate that an Arrow schema contains required STAC item fields. - Returns: - pyarrow.Schema with required fields marked as non-nullable - """ - required_fields = get_required_fields() + Checks for top-level required fields from IcestacItem. + Note: rustac flattens nested properties, so 'properties.datetime' + becomes 'datetime' in the Arrow schema. - # Convert arro3 schema to pyarrow schema and rebuild with correct nullable flags - pa_schema = pa.schema(schema) - new_fields = [] + Args: + schema: arro3.core.Schema to validate - for field in pa_schema: - if field.name in required_fields: - # Mark as non-nullable (required) - new_fields.append(pa.field(field.name, field.type, nullable=False)) - else: - # Keep original nullable setting - new_fields.append(field) + Raises: + ValueError: If required STAC fields are missing from the schema + """ + schema_fields = set(schema.names) + required_fields = cls.get_required_fields() + missing_fields = required_fields - schema_fields - return ArrowSchema.from_arrow(pa.schema(new_fields)) + if missing_fields: + raise ValueError( + f"Arrow schema is missing required STAC fields: {sorted(missing_fields)}" + ) -def validate_schema(schema: ArrowSchema) -> None: - """ - Validate that an Arrow schema contains required STAC item fields. - - Checks for top-level required fields from IcestacItem. - Note: rustac flattens nested properties, so 'properties.datetime' - becomes 'datetime' in the Arrow schema. - - Args: - schema: arro3.core.Schema to validate - - Raises: - ValueError: If required STAC fields are missing from the schema - """ - schema_fields = set(schema.names) - required_fields = get_required_fields() - missing_fields = required_fields - schema_fields +def get_schema_from_item(item: dict[str, Any]) -> ArrowSchema: - if missing_fields: - raise ValueError( - f"Arrow schema is missing required STAC fields: {sorted(missing_fields)}" - ) + return IcestacItem(**item).enforce_required_fields(to_arrow([item]).schema) def convert_schema(schema: ArrowSchema) -> IcebergSchema: @@ -101,7 +99,9 @@ def convert_schema(schema: ArrowSchema) -> IcebergSchema: Necessary because built-in converter functions do not assign field ids. """ - _schema = _pyarrow_to_schema_without_ids(pa.schema(enforce_required_fields(schema))) + _schema = _pyarrow_to_schema_without_ids( + pa.schema(IcestacItem.enforce_required_fields(schema)) + ) fields = [] for i, _field in enumerate(_schema.fields, start=1): diff --git a/tests/conftest.py b/tests/conftest.py index 56308bb..dd65468 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,8 +8,9 @@ from pyarrow import Table from rustac import to_arrow +from pyiceberg.catalog import load_catalog + from icestac.catalog import IcestacCatalog -from icestac.config import SqlCatalogConfig @pytest.fixture @@ -20,25 +21,22 @@ def temp_warehouse(): @pytest.fixture -def test_config(temp_warehouse: Path) -> SqlCatalogConfig: - return SqlCatalogConfig( - catalog_name="test_catalog", - catalog_uri=f"sqlite:///{temp_warehouse}/catalog.db", - warehouse_path=str(temp_warehouse), +def test_catalog(temp_warehouse: Path) -> Generator[IcestacCatalog, None, None]: + """Create a temporary SQL catalog for testing.""" + catalog = load_catalog( + "test_catalog", + **{ + "type": "sql", + "uri": f"sqlite:///{temp_warehouse}/catalog.db", + "warehouse": str(temp_warehouse), + }, ) + icestac_catalog = IcestacCatalog(catalog=catalog) - -@pytest.fixture -def test_catalog( - test_config: SqlCatalogConfig, -) -> Generator[IcestacCatalog, None, None]: - """Create an in-memory SQL catalog for testing.""" - catalog = IcestacCatalog.from_config(test_config) - - yield catalog + yield icestac_catalog gc.collect() - catalog.catalog.close() + icestac_catalog.catalog.close() @pytest.fixture diff --git a/tests/test_catalog.py b/tests/test_catalog.py index f47d473..331b59d 100644 --- a/tests/test_catalog.py +++ b/tests/test_catalog.py @@ -7,7 +7,7 @@ from icestac.catalog import IcestacCatalog from icestac.errors import InvalidCollectionIdError -from icestac.schema import enforce_required_fields, get_schema_from_item +from icestac.schema import IcestacItem, get_schema_from_item def test_create_item_table( @@ -24,7 +24,7 @@ def test_create_item_table( # Ensure data has required fields marked as non-nullable to match table schema arrow_data = to_arrow(sample_stac_items) - enforced_schema = enforce_required_fields(arrow_data.schema) + enforced_schema = IcestacItem.enforce_required_fields(arrow_data.schema) arrow_table = pyarrow.table(arrow_data).cast(pyarrow.schema(enforced_schema)) table.upsert( diff --git a/tests/test_config.py b/tests/test_config.py deleted file mode 100644 index 7c38fc0..0000000 --- a/tests/test_config.py +++ /dev/null @@ -1,224 +0,0 @@ -"""Tests for icestac config module.""" - -import os - -import pytest -from pydantic import ValidationError - -from icestac.config import ( - GlueCatalogConfig, - HiveCatalogConfig, - RestCatalogConfig, - SqlCatalogConfig, -) - - -class TestIcestacCatalogConfig: - """Test Iceberg catalog settings configuration.""" - - def test_default_settings(self, monkeypatch, tmp_path): - """Test default settings values with minimal valid SQL configuration.""" - for key in os.environ.copy(): - if key.startswith("ICESTAC_"): - monkeypatch.delenv(key, raising=False) - - catalog_db = tmp_path / "catalog.db" - warehouse_path = tmp_path / "warehouse" - warehouse_path.mkdir() - - monkeypatch.setenv("ICESTAC_CATALOG_URI", f"sqlite:///{catalog_db}") - monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", str(warehouse_path)) - - settings = SqlCatalogConfig.model_validate({}) - - assert settings.catalog_name == "default" - assert settings.catalog_type == "sql" - assert settings.sql.echo is False - - def test_sql_catalog_properties(self, monkeypatch): - """Test SQL catalog configuration.""" - monkeypatch.setenv("ICESTAC_CATALOG_NAME", "test_catalog") - monkeypatch.setenv("ICESTAC_CATALOG_URI", "sqlite:///catalog.db") - monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", "/tmp/warehouse") - - settings = SqlCatalogConfig.model_validate( - {}, - ) - - assert settings.catalog_name == "test_catalog" - assert settings.catalog_type == "sql" - assert settings.catalog_uri == "sqlite:///catalog.db" - assert settings.warehouse_path == "/tmp/warehouse" - - properties = settings.get_catalog_properties() - assert properties["type"] == "sql" - assert properties["uri"] == "sqlite:///catalog.db" - assert properties["warehouse"] == "/tmp/warehouse" - assert properties["echo"] == "false" - - def test_rest_catalog_properties(self, monkeypatch): - """Test REST catalog configuration.""" - monkeypatch.setenv("ICESTAC_CATALOG_NAME", "rest_catalog") - monkeypatch.setenv("ICESTAC_CATALOG_URI", "https://rest.example.com") - monkeypatch.setenv("ICESTAC_REST__TOKEN", "my-token") - - settings = RestCatalogConfig.model_validate( - {}, - ) - - assert settings.catalog_type == "rest" - assert settings.rest.token == "my-token" - - properties = settings.get_catalog_properties() - assert properties["type"] == "rest" - assert properties["uri"] == "https://rest.example.com" - assert properties["token"] == "my-token" - - def test_glue_catalog_properties(self, monkeypatch): - """Test AWS Glue catalog configuration.""" - monkeypatch.setenv("ICESTAC_CATALOG_NAME", "glue_catalog") - monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", "s3://bucket/warehouse") - monkeypatch.setenv("ICESTAC_GLUE__REGION", "us-west-2") - - settings = GlueCatalogConfig() - - assert settings.catalog_type == "glue" - assert settings.glue.region == "us-west-2" - - properties = settings.get_catalog_properties() - assert properties["type"] == "glue" - assert properties["warehouse"] == "s3://bucket/warehouse" - assert properties["region"] == "us-west-2" - - def test_sql_catalog_requires_uri(self, monkeypatch): - """Test that SQL catalog requires a URI.""" - monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", "/tmp/warehouse") - - with pytest.raises(ValidationError) as exc_info: - SqlCatalogConfig.model_validate( - {}, - ) - - assert "catalog_uri" in str(exc_info.value) - - def test_sql_catalog_requires_warehouse_path(self, monkeypatch): - """Test that SQL catalog requires a warehouse path.""" - monkeypatch.setenv("ICESTAC_CATALOG_URI", "sqlite:///catalog.db") - - with pytest.raises(ValidationError) as exc_info: - SqlCatalogConfig.model_validate( - {}, - ) - - assert "warehouse_path" in str(exc_info.value) - - def test_rest_catalog_requires_uri(self, monkeypatch): - """Test that REST catalog requires a URI.""" - with pytest.raises(ValidationError) as exc_info: - RestCatalogConfig.model_validate( - {}, - ) - - assert "catalog_uri" in str(exc_info.value) - - def test_case_insensitive_env_vars(self, monkeypatch): - """Test that environment variables are case-insensitive.""" - monkeypatch.setenv("icestac_catalog_name", "test") - monkeypatch.setenv("icestac_catalog_uri", "sqlite:///catalog.db") - monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", "/tmp/warehouse") - - settings = SqlCatalogConfig.model_validate( - {}, - ) - - assert settings.catalog_name == "test" - - def test_sql_echo_enabled(self, monkeypatch): - """Test SQL echo setting.""" - monkeypatch.setenv("ICESTAC_CATALOG_URI", "sqlite:///catalog.db") - monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", "/tmp/warehouse") - monkeypatch.setenv("ICESTAC_SQL__ECHO", "true") - - settings = SqlCatalogConfig.model_validate( - {}, - ) - - assert settings.sql.echo is True - - properties = settings.get_catalog_properties() - assert properties["echo"] == "true" - - def test_s3_endpoint_properties(self, monkeypatch): - """Test that S3 endpoint and path-style access are included in properties.""" - monkeypatch.setenv("ICESTAC_CATALOG_URI", "sqlite:///catalog.db") - monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", "/tmp/warehouse") - monkeypatch.setenv("ICESTAC_S3__ENDPOINT", "http://minio:9000") - monkeypatch.setenv("ICESTAC_S3__PATH_STYLE_ACCESS", "true") - - settings = SqlCatalogConfig.model_validate({}) - properties = settings.get_catalog_properties() - - assert properties["s3.endpoint"] == "http://minio:9000" - assert properties["s3.path-style-access"] == "true" - - def test_s3_credentials_properties(self, monkeypatch): - """Test that S3 access key and secret are included in properties.""" - monkeypatch.setenv("ICESTAC_CATALOG_URI", "sqlite:///catalog.db") - monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", "/tmp/warehouse") - monkeypatch.setenv("ICESTAC_S3__ACCESS_KEY_ID", "mykey") - monkeypatch.setenv("ICESTAC_S3__SECRET_ACCESS_KEY", "mysecret") - - settings = SqlCatalogConfig.model_validate({}) - properties = settings.get_catalog_properties() - - assert properties["s3.access-key-id"] == "mykey" - assert properties["s3.secret-access-key"] == "mysecret" - - def test_extra_properties_passthrough(self, monkeypatch): - """Test that extra fields passed to the model are included in catalog properties.""" - monkeypatch.setenv("ICESTAC_CATALOG_URI", "sqlite:///catalog.db") - monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", "/tmp/warehouse") - - settings = SqlCatalogConfig.model_validate({"some_extra_key": "extra_value"}) - properties = settings.get_catalog_properties() - - assert properties["some_extra_key"] == "extra_value" - - def test_rest_catalog_with_warehouse_and_credential(self, monkeypatch): - """Test REST catalog with optional warehouse path and credential.""" - monkeypatch.setenv("ICESTAC_CATALOG_URI", "https://rest.example.com") - monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", "s3://bucket/warehouse") - monkeypatch.setenv("ICESTAC_REST__CREDENTIAL", "client_id:client_secret") - - settings = RestCatalogConfig.model_validate({}) - properties = settings.get_catalog_properties() - - assert properties["warehouse"] == "s3://bucket/warehouse" - assert properties["credential"] == "client_id:client_secret" - - def test_glue_catalog_with_uri(self, monkeypatch): - """Test AWS Glue catalog with optional catalog URI.""" - monkeypatch.setenv("ICESTAC_CATALOG_URI", "glue://my-catalog") - monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", "s3://bucket/warehouse") - - settings = GlueCatalogConfig.model_validate({}) - properties = settings.get_catalog_properties() - - assert properties["uri"] == "glue://my-catalog" - assert properties["warehouse"] == "s3://bucket/warehouse" - - def test_hive_catalog_properties(self, monkeypatch): - """Test Hive catalog configuration and properties.""" - monkeypatch.setenv("ICESTAC_CATALOG_URI", "thrift://hive-metastore:9083") - monkeypatch.setenv("ICESTAC_WAREHOUSE_PATH", "s3://bucket/warehouse") - - settings = HiveCatalogConfig.model_validate({}) - - assert settings.catalog_type == "hive" - assert settings.catalog_uri == "thrift://hive-metastore:9083" - assert settings.warehouse_path == "s3://bucket/warehouse" - - properties = settings.get_catalog_properties() - assert properties["type"] == "hive" - assert properties["uri"] == "thrift://hive-metastore:9083" - assert properties["warehouse"] == "s3://bucket/warehouse" diff --git a/tests/test_schema.py b/tests/test_schema.py index ea47a96..c9909aa 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -5,7 +5,7 @@ from arro3.core import Schema from pydantic import ValidationError -from icestac.schema import get_schema_from_item, validate_schema +from icestac.schema import IcestacItem, get_schema_from_item def test_get_schema_from_item(sample_stac_item: dict[str, Any]) -> None: @@ -39,7 +39,7 @@ def test_validate_schema_valid(sample_stac_item: dict[str, Any]) -> None: schema = get_schema_from_item(sample_stac_item) # Should not raise - validate_schema(schema) + IcestacItem.validate_schema(schema) def test_validate_schema_missing_required_field() -> None: @@ -55,7 +55,7 @@ def test_validate_schema_missing_required_field() -> None: ) with pytest.raises(ValueError, match="missing required STAC fields.*'id'"): - validate_schema(schema) + IcestacItem.validate_schema(schema) def test_validate_schema_missing_datetime() -> None: @@ -75,4 +75,4 @@ def test_validate_schema_missing_datetime() -> None: ) with pytest.raises(ValueError, match="missing required STAC fields.*'datetime'"): - validate_schema(schema) + IcestacItem.validate_schema(schema) diff --git a/uv.lock b/uv.lock index 669d1cd..d78f9cb 100644 --- a/uv.lock +++ b/uv.lock @@ -466,7 +466,6 @@ version = "0.0.1" source = { editable = "." } dependencies = [ { name = "pyarrow" }, - { name = "pydantic-settings" }, { name = "pyiceberg", extra = ["pyiceberg-core"] }, { name = "rustac", extra = ["arrow"] }, { name = "stac-pydantic" }, @@ -488,7 +487,6 @@ dev = [ [package.metadata] requires-dist = [ { name = "pyarrow", specifier = ">=23.0.0" }, - { name = "pydantic-settings", specifier = ">=2.12.0" }, { name = "pyiceberg", extras = ["pyiceberg-core"], specifier = ">=0.10.0" }, { name = "rustac", extras = ["arrow"], specifier = ">=0.9.3" }, { name = "stac-pydantic", specifier = ">=3.4.0" }, @@ -899,20 +897,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/36/c7/cfc8e811f061c841d7990b0201912c3556bfeb99cdcb7ed24adc8d6f8704/pydantic_core-2.41.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51", size = 2145302, upload-time = "2025-11-04T13:43:46.64Z" }, ] -[[package]] -name = "pydantic-settings" -version = "2.12.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, - { name = "python-dotenv" }, - { name = "typing-inspection" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/43/4b/ac7e0aae12027748076d72a8764ff1c9d82ca75a7a52622e67ed3f765c54/pydantic_settings-2.12.0.tar.gz", hash = "sha256:005538ef951e3c2a68e1c08b292b5f2e71490def8589d4221b95dab00dafcfd0", size = 194184, upload-time = "2025-11-10T14:25:47.013Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c1/60/5d4751ba3f4a40a6891f24eec885f51afd78d208498268c734e256fb13c4/pydantic_settings-2.12.0-py3-none-any.whl", hash = "sha256:fddb9fd99a5b18da837b29710391e945b1e30c135477f484084ee513adb93809", size = 51880, upload-time = "2025-11-10T14:25:45.546Z" }, -] - [[package]] name = "pygments" version = "2.19.2" @@ -1086,15 +1070,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/06/54/82a6e2ef37f0f23dccac604b9585bdcbd0698604feb64807dcb72853693e/python_discovery-1.1.0-py3-none-any.whl", hash = "sha256:a162893b8809727f54594a99ad2179d2ede4bf953e12d4c7abc3cc9cdbd1437b", size = 30687, upload-time = "2026-02-26T09:42:48.548Z" }, ] -[[package]] -name = "python-dotenv" -version = "1.2.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f0/26/19cadc79a718c5edbec86fd4919a6b6d3f681039a2f6d66d14be94e75fb9/python_dotenv-1.2.1.tar.gz", hash = "sha256:42667e897e16ab0d66954af0e60a9caa94f0fd4ecf3aaf6d2d260eec1aa36ad6", size = 44221, upload-time = "2025-10-26T15:12:10.434Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/14/1b/a298b06749107c305e1fe0f814c6c74aea7b2f1e10989cb30f544a1b3253/python_dotenv-1.2.1-py3-none-any.whl", hash = "sha256:b81ee9561e9ca4004139c6cbba3a238c32b03e4894671e181b671e8cb8425d61", size = 21230, upload-time = "2025-10-26T15:12:09.109Z" }, -] - [[package]] name = "pyyaml" version = "6.0.3" From fbffdc33b3ba7c75cd4a4d99967f00b053fc4a8c Mon Sep 17 00:00:00 2001 From: hrodmn Date: Tue, 3 Mar 2026 12:37:34 -0600 Subject: [PATCH 13/23] docs: update local workflow in README --- .env-local | 12 --- .pyiceberg.yaml | 9 +++ README.md | 200 +++++++----------------------------------------- main.py | 22 ++++-- 4 files changed, 53 insertions(+), 190 deletions(-) delete mode 100644 .env-local create mode 100644 .pyiceberg.yaml diff --git a/.env-local b/.env-local deleted file mode 100644 index 7ef3d3a..0000000 --- a/.env-local +++ /dev/null @@ -1,12 +0,0 @@ -# Iceberg Catalog Configuration -ICESTAC_CATALOG_NAME=rest_catalog -ICESTAC_CATALOG_TYPE=rest -ICESTAC_CATALOG_URI=http://localhost:8181 -ICESTAC_WAREHOUSE_PATH=s3://warehouse/ - -# S3/MinIO Storage Configuration -# These settings allow PyIceberg to directly access MinIO for data file I/O -ICESTAC_S3_ENDPOINT=http://localhost:9000 -ICESTAC_S3_ACCESS_KEY_ID=admin -ICESTAC_S3_SECRET_ACCESS_KEY=password -ICESTAC_S3_PATH_STYLE_ACCESS=true diff --git a/.pyiceberg.yaml b/.pyiceberg.yaml new file mode 100644 index 0000000..c9a8c61 --- /dev/null +++ b/.pyiceberg.yaml @@ -0,0 +1,9 @@ +catalog: + default: + type: rest + uri: http://localhost:8181 + warehouse: s3://warehouse/ + s3.endpoint: http://localhost:9000 + s3.access-key-id: admin + s3.secret-access-key: password + s3.path-style-access: "true" diff --git a/README.md b/README.md index b4bc7f9..7d0ce36 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,20 @@ uv run pytest ### Local Instance -**Catalog Configuration** (`.pyiceberg.yaml`): +**1. Start the local environment:** -icestac delegates catalog configuration to PyIceberg. Create a `.pyiceberg.yaml` in your working directory (or `~/.pyiceberg.yaml` for a user-wide default): +```bash +docker compose up +``` + +This starts three services: +- **Iceberg REST Catalog** at `http://localhost:8181` +- **MinIO** (S3-compatible storage) at `http://localhost:9000` (API) and `http://localhost:9001` (Console) +- **MinIO Client** — initializes the `warehouse` bucket on startup + +**2. Configure catalog access:** + +A `.pyiceberg.yaml` is included in the repo with default credentials for the local Docker environment: ```yaml catalog: @@ -33,24 +44,21 @@ catalog: s3.path-style-access: "true" ``` -Alternatively, configure via environment variables using PyIceberg's `PYICEBERG_CATALOG____` prefix: +PyIceberg will pick this up automatically when running from the project directory. See the [PyIceberg configuration docs](https://py.iceberg.apache.org/configuration/) for other configuration options. -```bash -PYICEBERG_CATALOG__DEFAULT__TYPE=rest -PYICEBERG_CATALOG__DEFAULT__URI=http://localhost:8181 -PYICEBERG_CATALOG__DEFAULT__WAREHOUSE=s3://warehouse/ -``` +**3. Load sample items:** -See the [PyIceberg configuration docs](https://py.iceberg.apache.org/configuration/) for the full list of catalog and S3 properties. +`main.py` fetches 5 items from the `icesat2-boreal-v3.1-agb` collection on the MAAP STAC API and writes them to the local Iceberg catalog: -**Starting the local environment:** ```bash -docker compose up +uv run python main.py ``` -**Querying with DuckDB:** +This creates an `icestac.icesat2_boreal_v3_1_agb` table in the catalog and upserts the items. -After ingesting items (e.g. via `uv run python main.py`), you can query the Iceberg tables using DuckDB's `iceberg` extension. Tables live under the `icestac` namespace, named by the sanitized collection ID. +**4. Query with DuckDB:** + +After ingesting items, query the Iceberg tables using DuckDB's `iceberg` extension. Tables live under the `icestac` namespace. First, configure the extensions and MinIO credentials: @@ -72,178 +80,26 @@ CREATE OR REPLACE SECRET minio ( Query via the REST catalog: ```sql -ATTACH 'http://localhost:8181' AS catalog ( +ATTACH 'icestac' AS catalog ( TYPE ICEBERG, - WAREHOUSE 's3://warehouse/' + ENDPOINT 'http://localhost:8181', + AUTHORIZATION_TYPE 'none' ); SELECT id, datetime, collection, geometry FROM catalog.icestac.icesat2_boreal_v3_1_agb LIMIT 10; + +SELECT count(*) +FROM catalog.icestac.icesat2_boreal_v3_1_agb; ``` Or scan the table directly from its S3 path (no catalog required): ```sql SET unsafe_enable_version_guessing = true; -DESCRIBE SELECT * +SELECT * FROM iceberg_scan('s3://warehouse/icestac/icesat2_boreal_v3_1_agb') LIMIT 10; ``` -## Current Implementation Status - -### Core Library (`src/icestac/`) - -#### Item Table Module (`src/icestac/item_table.py`) - ✓ IMPLEMENTED - -Core functions for managing STAC item Iceberg tables: - -**`sanitize_collection_id(collection_id: str) -> str`** -- Converts STAC collection IDs to valid, deterministic Iceberg table names -- Uses lowercase + underscore normalization with 8-character hash suffix for uniqueness - -**`create_item_table(arrow_schema: ArrowSchema, collection_id: str, catalog: Catalog, namespace: str) -> Table`** -- Creates or loads Iceberg table from stac-geoparquet Arrow schema -- Converts Arrow schema to Iceberg schema with manual field ID assignment -- Creates table partitioned by datetime month using `MonthTransform` -- Validates schema for required STAC fields - -**Limitations:** -- Temporal partitioning is hardcoded to monthly (TODO: make configurable) -- No collection-level metadata management solution yet - -#### Schema Module (`src/icestac/schema.py`) - ✓ IMPLEMENTED - -Schema validation and enforcement: - -**`IcestacItem`** - Pydantic model extending stac-pydantic Item with required `collection` field - -**`get_schema_from_item(item: dict) -> Schema`** -- Validates STAC item and returns Arrow schema with enforced required fields - -**`enforce_required_fields(schema: Schema) -> Schema`** -- Marks required STAC fields as non-nullable in Arrow schema - -**`validate_schema(schema: Schema) -> None`** -- Validates Arrow schema contains all required STAC fields - - -#### Lambda Handler Module (`src/icestac/lambda_handler.py`) - NOT IMPLEMENTED - -Placeholder for AWS Lambda handler. - -### Testing Infrastructure (`tests/`) - -#### Test Coverage - ✓ IMPLEMENTED - -- **`tests/conftest.py`**: Pytest fixtures for test catalog, sample STAC items, and Arrow tables -- **`tests/test_item_table.py`**: Unit tests for `sanitize_collection_id` and `create_item_table` -- **`tests/test_schema.py`**: Unit tests for schema validation and enforcement - -### Dependencies - -**Core** (`pyproject.toml` dependencies): -- `pyarrow>=23.0.0` - Arrow table operations -- `pyiceberg[pyiceberg-core]>=0.10.0` - Iceberg table management -- `rustac[arrow]>=0.9.3` - STAC to Arrow conversion with arro3 schemas -- `stac-pydantic>=3.4.0` - STAC item validation - -**Development** (dev dependency group): -- `pytest>=9.0.2` - Testing framework -- `sqlalchemy>=2.0.46` - SQL catalog backend for tests - -**Deployment** (deploy dependency group): -- `aws-cdk-lib>=2.236.0` - AWS infrastructure as code - -**Still needed for Lambda handler:** -- `boto3` - Lambda/SNS/SQS/S3 interactions -- `aws-lambda-powertools` - Structured logging and tracing -- `moto` - AWS service mocking for tests - -## Next Steps - -### Immediate Priorities - -1. **Design Collection Metadata Management** - - Determine approach for storing and managing collection-level metadata - - Options: Separate metadata table, catalog namespace properties, or external store - - Should track: collection description, temporal extent, spatial extent, schema versions - -2. **Complete Lambda Handler** (`src/icestac/lambda_handler.py`) - - Implement SNS event parsing - - Add collection grouping logic - - Integrate `create_item_table` function and config module - - Add error handling and structured logging - - Write integration tests - -3. **Enhance Item Table Module** - - Make partitioning strategy configurable (currently hardcoded to monthly) - - Add support for schema evolution - - Add write statistics/metadata - -### Future Work: AWS Infrastructure (CDK) - -**Planned Stack Structure**: -``` -infrastructure/ -├── app.py -├── stacks/ - ├── stac_ingestion_stack.py # SNS → SQS → Lambda pipeline - └── iceberg_catalog_stack.py # Optional: Glue/DynamoDB catalog -``` - -**STAC Ingestion Stack Components**: -- SNS Topic for incoming STAC items -- SQS Queue with batching and DLQ -- Lambda Function with icestac library -- CloudWatch Alarms for monitoring - -**Additional Dependencies Needed**: -- `constructs` -- `aws-cdk.aws-lambda-python-alpha` (Python Lambda bundling) - -## Development Workflow - -### Local Testing Strategy - -**Implemented:** -- PyIceberg with SQL catalog (SQLite) for unit tests -- Pytest for test framework -- Docker Compose with MinIO and Iceberg REST catalog for local development - -**Planned:** -- Moto for mocking AWS services in Lambda handler tests -- Helper script to simulate SNS events locally -- Optional: LocalStack for complete AWS simulation - -### Local Development Environment - -**Docker Compose Services**: -- **Iceberg REST Catalog** - `localhost:8181` for metadata operations -- **MinIO** - S3-compatible storage at `localhost:9000` (API) and `localhost:9001` (Console) - - Credentials: `admin` / `password` - - Warehouse bucket: `s3://warehouse/` -- **MinIO Client (mc)** - Initializes warehouse bucket on startup - -## Key Design Decisions - -### Decided - -1. **Table naming**: Sanitized collection ID with 8-character hash suffix for uniqueness -2. **Partitioning**: Monthly partitioning by datetime field (hardcoded, to be made configurable) -3. **Schema conversion**: Manual field ID assignment to avoid pyiceberg limitations -4. **Schema validation**: Required STAC fields marked as non-nullable using Pydantic models -5. **Configuration**: Delegated to PyIceberg via `.pyiceberg.yaml` or `PYICEBERG_CATALOG__` environment variables -6. **Testing catalog**: In-memory SQL catalog with SQLite for unit tests -7. **STAC to Arrow conversion**: Use rustac library with arro3 schemas - -### To Be Decided - -1. **Collection metadata management**: How to store and query collection-level metadata (description, extents, schema versions)? -2. **Iceberg catalog type for production**: AWS Glue (managed AWS) vs REST (self-hosted/managed) vs SQL (RDS)? -3. **Configurable partitioning strategies**: Support daily, monthly, yearly, or custom partitioning? -4. **Schema evolution policy**: Strict or flexible? How to handle schema changes across items in same collection? -5. **Batch size**: How many STAC items per SQS batch for optimal performance? -6. **Error handling**: Retry strategy for failed items? DLQ processing? -7. **S3 bucket structure**: How to organize Iceberg table data and metadata? diff --git a/main.py b/main.py index bcf3dca..f9f05d7 100644 --- a/main.py +++ b/main.py @@ -1,4 +1,5 @@ import asyncio +import logging import rustac from pyiceberg.catalog import load_catalog @@ -6,15 +7,21 @@ from icestac.catalog import IcestacCatalog from icestac.schema import get_schema_from_item +logger = logging.getLogger(__name__) + +BATCH_SIZE = 1000 + async def run(): + logging.basicConfig(level=logging.INFO) catalog = IcestacCatalog(catalog=load_catalog()) items = await rustac.search( "https://stac.maap-project.org", collections="icesat2-boreal-v3.1-agb", - max_items=5, + limit=200, ) + collection_id = "icesat2_boreal_v3_1_agb" for item in items: item["collection"] = collection_id @@ -22,11 +29,14 @@ async def run(): schema = get_schema_from_item(items[0]) catalog.create_item_table(arrow_schema=schema, collection_id=collection_id) - catalog.load_items( - collection_id=collection_id, - items=items, - method="upsert", - ) + batches = [items[i : i + BATCH_SIZE] for i in range(0, len(items), BATCH_SIZE)] + for i, batch in enumerate(batches, start=1): + logger.info("Loading batch %d/%d (%d items)", i, len(batches), len(batch)) + catalog.load_items( + collection_id=collection_id, + items=batch, + method="upsert", + ) if __name__ == "__main__": From 0f29d778292788e78ab8582fd9ce1b35a3a768ac Mon Sep 17 00:00:00 2001 From: hrodmn Date: Tue, 19 May 2026 15:13:09 -0500 Subject: [PATCH 14/23] feat: add pathway to load items directly from arrow table --- .gitignore | 4 ++ main.py | 4 +- pyproject.toml | 3 ++ src/icestac/catalog.py | 5 +- src/icestac/load.py | 20 +++++--- src/icestac/schema.py | 32 ++++++++++-- tests/__init__.py | 0 tests/conftest.py | 28 +++++++--- tests/helpers.py | 15 ++++++ tests/test_catalog.py | 43 ++++++++-------- tests/test_load.py | 113 +++++++++++++++++++---------------------- tests/test_schema.py | 18 +++---- 12 files changed, 175 insertions(+), 110 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/helpers.py diff --git a/.gitignore b/.gitignore index f06d48f..e9cd161 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,7 @@ wheels/ .venv minio-data/ + +# dev docs +dev-docs/plans/ +dev-docs/brainstorms/ diff --git a/main.py b/main.py index f9f05d7..da64240 100644 --- a/main.py +++ b/main.py @@ -5,7 +5,7 @@ from pyiceberg.catalog import load_catalog from icestac.catalog import IcestacCatalog -from icestac.schema import get_schema_from_item +from icestac.schema import get_schema_from_items logger = logging.getLogger(__name__) @@ -26,7 +26,7 @@ async def run(): for item in items: item["collection"] = collection_id - schema = get_schema_from_item(items[0]) + schema = get_schema_from_items(items) catalog.create_item_table(arrow_schema=schema, collection_id=collection_id) batches = [items[i : i + BATCH_SIZE] for i in range(0, len(items), BATCH_SIZE)] diff --git a/pyproject.toml b/pyproject.toml index ef132a7..2849309 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,9 @@ dev = [ "ty>=0.0.18", ] +[tool.ty.rules] +all = "error" + [tool.pytest.ini_options] addopts = [ "-v", diff --git a/src/icestac/catalog.py b/src/icestac/catalog.py index 54e7786..e111861 100644 --- a/src/icestac/catalog.py +++ b/src/icestac/catalog.py @@ -1,7 +1,6 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Any from arro3.core import Schema as ArrowSchema from pyiceberg.catalog import Catalog @@ -12,7 +11,7 @@ from icestac.constants import DEFAULT_NAMESPACE from icestac.errors import InvalidCollectionIdError from icestac.load import Method, load_items -from icestac.schema import IcestacItem, convert_schema +from icestac.schema import IcestacItem, ItemsInput, convert_schema def validate_collection_id(collection_id: str) -> None: @@ -75,7 +74,7 @@ def create_item_table( ) def load_items( - self, collection_id: str, items: list[dict[str, Any]], method: Method = "upsert" + self, collection_id: str, items: ItemsInput, method: Method = "upsert" ) -> None: load_items( items, diff --git a/src/icestac/load.py b/src/icestac/load.py index d6e7998..954a0c2 100644 --- a/src/icestac/load.py +++ b/src/icestac/load.py @@ -1,22 +1,28 @@ -from typing import Any, Literal +from typing import Literal import pyarrow +import rustac +from arro3.core import Table as ArrowTable from pyiceberg.table import Table -from rustac import to_arrow -from icestac.schema import IcestacItem +from icestac.schema import IcestacItem, ItemsInput Method = Literal["append", "upsert"] def load_items( - items: list[dict[str, Any]], + items: ItemsInput, table: Table, method: Method = "upsert", ) -> None: - arrow_data = to_arrow(items) - enforced_schema = IcestacItem.enforce_required_fields(arrow_data.schema) - arrow_table = pyarrow.table(arrow_data).cast(pyarrow.schema(enforced_schema)) + if isinstance(items, dict): + items = [items] + + if not isinstance(items, ArrowTable): + items = rustac.to_arrow(items) + + enforced_schema = IcestacItem.enforce_required_fields(items.schema) + arrow_table = pyarrow.table(items).cast(pyarrow.schema(enforced_schema)) if method == "upsert": table.upsert( diff --git a/src/icestac/schema.py b/src/icestac/schema.py index fdd619b..727d2d8 100644 --- a/src/icestac/schema.py +++ b/src/icestac/schema.py @@ -1,14 +1,17 @@ from typing import Any import pyarrow as pa +import rustac from arro3.core import Field from arro3.core import Schema as ArrowSchema +from arro3.core import Table as ArrowTable from pyiceberg.io.pyarrow import _pyarrow_to_schema_without_ids from pyiceberg.schema import Schema as IcebergSchema from pyiceberg.types import NestedField -from rustac import to_arrow from stac_pydantic.item import Item +ItemsInput = ArrowTable | list[dict[str, Any]] | dict[str, Any] + class IcestacItem(Item): collection: str @@ -89,9 +92,32 @@ def validate_schema(cls, schema: ArrowSchema) -> None: ) -def get_schema_from_item(item: dict[str, Any]) -> ArrowSchema: +def _first_item_from_arrow(items: ArrowTable) -> dict[str, Any]: + table = pa.table(items) + + if len(table) == 0: + raise ValueError("Cannot validate an empty Arrow table") + + feature_collection = rustac.from_arrow(table.slice(0, 1)) + + return feature_collection["features"][0] + + +def get_schema_from_items(items: ItemsInput) -> ArrowSchema: + if isinstance(items, dict): + item = items + items = [items] + elif isinstance(items, list): + item = items[0] + elif isinstance(items, ArrowTable): + item = _first_item_from_arrow(items) + + IcestacItem.model_validate(item) + + if not isinstance(items, ArrowTable): + items = rustac.to_arrow(items) - return IcestacItem(**item).enforce_required_fields(to_arrow([item]).schema) + return IcestacItem.enforce_required_fields(items.schema) def convert_schema(schema: ArrowSchema) -> IcebergSchema: diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py index dd65468..d2be113 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,14 +3,13 @@ from pathlib import Path from typing import Any, Generator -import pyarrow import pytest -from pyarrow import Table -from rustac import to_arrow - +import rustac +from arro3.core import Table as ArrowTable from pyiceberg.catalog import load_catalog from icestac.catalog import IcestacCatalog +from icestac.schema import ItemsInput @pytest.fixture @@ -92,5 +91,22 @@ def sample_stac_items(sample_stac_item) -> list[dict[str, Any]]: @pytest.fixture -def sample_item_arrow_table(sample_stac_items) -> Table: - return pyarrow.table(to_arrow(sample_stac_items)) +def sample_stac_item_arrow_table(sample_stac_items) -> ArrowTable: + return rustac.to_arrow(sample_stac_items) + + +@pytest.fixture +def sample_item_arrow_table(sample_stac_item_arrow_table: ArrowTable) -> ArrowTable: + """Backward-compatible alias for the Arrow-backed STAC items fixture.""" + return sample_stac_item_arrow_table + + +@pytest.fixture( + params=[ + pytest.param("sample_stac_item", id="single-item"), + pytest.param("sample_stac_items", id="item-list"), + pytest.param("sample_stac_item_arrow_table", id="arrow-table"), + ] +) +def items(request) -> ItemsInput: + return request.getfixturevalue(request.param) diff --git a/tests/helpers.py b/tests/helpers.py new file mode 100644 index 0000000..35777b9 --- /dev/null +++ b/tests/helpers.py @@ -0,0 +1,15 @@ +from typing import Any + +import pyarrow as pa +import rustac +from arro3.core import Table as ArrowTable + +from icestac.schema import ItemsInput + + +def items_to_list(items: ItemsInput) -> list[dict[str, Any]]: + if isinstance(items, dict): + return [items] + if isinstance(items, ArrowTable): + return rustac.from_arrow(pa.table(items))["features"] + return items diff --git a/tests/test_catalog.py b/tests/test_catalog.py index 331b59d..7567e43 100644 --- a/tests/test_catalog.py +++ b/tests/test_catalog.py @@ -1,29 +1,31 @@ -from typing import Any - import pyarrow import pytest +from arro3.core import Table as ArrowTable from pyiceberg.exceptions import TableAlreadyExistsError from rustac import to_arrow from icestac.catalog import IcestacCatalog from icestac.errors import InvalidCollectionIdError -from icestac.schema import IcestacItem, get_schema_from_item +from icestac.schema import IcestacItem, ItemsInput, get_schema_from_items +from tests.helpers import items_to_list def test_create_item_table( test_catalog: IcestacCatalog, - sample_stac_items: list[dict[str, Any]], + test_collection_id: str, + items: ItemsInput, ) -> None: - arrow_schema = get_schema_from_item(sample_stac_items[0]) + expected_items = items_to_list(items) + arrow_schema = get_schema_from_items(items) table = test_catalog.create_item_table( arrow_schema=arrow_schema, - collection_id=sample_stac_items[0]["collection"], + collection_id=test_collection_id, ) assert table.schema().find_field("datetime") # Ensure data has required fields marked as non-nullable to match table schema - arrow_data = to_arrow(sample_stac_items) + arrow_data = to_arrow(expected_items) if not isinstance(items, ArrowTable) else items enforced_schema = IcestacItem.enforce_required_fields(arrow_data.schema) arrow_table = pyarrow.table(arrow_data).cast(pyarrow.schema(enforced_schema)) @@ -34,21 +36,21 @@ def test_create_item_table( # Verify records were inserted result = table.scan().to_arrow() - assert len(result) == len(sample_stac_items) - assert result.column("id").to_pylist() == [item["id"] for item in sample_stac_items] + assert len(result) == len(expected_items) + assert result.column("id").to_pylist() == [item["id"] for item in expected_items] with pytest.raises(TableAlreadyExistsError): test_catalog.create_item_table( arrow_schema=arrow_schema, - collection_id=sample_stac_items[0]["collection"], + collection_id=test_collection_id, ) def test_create_item_table_bad_collection_id( test_catalog: IcestacCatalog, - sample_stac_items: list[dict[str, Any]], + items: ItemsInput, ) -> None: - arrow_schema = get_schema_from_item(sample_stac_items[0]) + arrow_schema = get_schema_from_items(items) with pytest.raises(InvalidCollectionIdError): test_catalog.create_item_table( arrow_schema=arrow_schema, @@ -58,18 +60,19 @@ def test_create_item_table_bad_collection_id( def test_load_items( test_catalog: IcestacCatalog, - sample_stac_items: list[dict[str, Any]], + test_collection_id: str, + items: ItemsInput, ) -> None: - collection_id = sample_stac_items[0]["collection"] - arrow_schema = get_schema_from_item(sample_stac_items[0]) + expected_items = items_to_list(items) + arrow_schema = get_schema_from_items(items) table = test_catalog.create_item_table( arrow_schema=arrow_schema, - collection_id=collection_id, + collection_id=test_collection_id, ) test_catalog.load_items( - collection_id=sample_stac_items[0]["collection"], - items=sample_stac_items, + collection_id=test_collection_id, + items=items, method="upsert", ) @@ -77,5 +80,5 @@ def test_load_items( # Verify records were inserted result = table.scan().to_arrow() - assert len(result) == len(sample_stac_items) - assert result.column("id").to_pylist() == [item["id"] for item in sample_stac_items] + assert len(result) == len(expected_items) + assert result.column("id").to_pylist() == [item["id"] for item in expected_items] diff --git a/tests/test_load.py b/tests/test_load.py index b6a6595..00a2001 100644 --- a/tests/test_load.py +++ b/tests/test_load.py @@ -4,70 +4,78 @@ from icestac.catalog import IcestacCatalog from icestac.load import load_items -from icestac.schema import get_schema_from_item +from icestac.schema import ItemsInput, get_schema_from_items +from tests.helpers import items_to_list def test_load_items_upsert_default( test_catalog: IcestacCatalog, - sample_stac_items: list[dict[str, Any]], + test_collection_id: str, + items: ItemsInput, ) -> None: """Test loading items with default upsert method.""" + expected_items = items_to_list(items) + # Create the table - arrow_schema = get_schema_from_item(sample_stac_items[0]) + arrow_schema = get_schema_from_items(items) table = test_catalog.create_item_table( arrow_schema=arrow_schema, - collection_id=sample_stac_items[0]["collection"], + collection_id=test_collection_id, ) # Load items (default method is upsert) - load_items(sample_stac_items, table) + load_items(items, table) # Verify records were inserted result = table.scan().to_arrow() - assert len(result) == len(sample_stac_items) + assert len(result) == len(expected_items) assert sorted(result.column("id").to_pylist()) == sorted( - [item["id"] for item in sample_stac_items] + item["id"] for item in expected_items ) def test_load_items_upsert_explicit( test_catalog: IcestacCatalog, - sample_stac_items: list[dict[str, Any]], + items: ItemsInput, ) -> None: """Test loading items with explicit upsert method.""" + expected_items = items_to_list(items) + # Create the table - arrow_schema = get_schema_from_item(sample_stac_items[0]) + arrow_schema = get_schema_from_items(items) table = test_catalog.create_item_table( arrow_schema=arrow_schema, - collection_id=sample_stac_items[0]["collection"], + collection_id=expected_items[0]["collection"], ) # Load items with explicit upsert method - load_items(sample_stac_items, table, method="upsert") + load_items(items, table, method="upsert") # Verify records were inserted result = table.scan().to_arrow() - assert len(result) == len(sample_stac_items) + assert len(result) == len(expected_items) def test_load_items_upsert_updates_existing( test_catalog: IcestacCatalog, - sample_stac_items: list[dict[str, Any]], + items: ItemsInput, ) -> None: """Test that upsert updates existing records with same ID.""" + expected_items = items_to_list(items) + # Create the table - arrow_schema = get_schema_from_item(sample_stac_items[0]) + arrow_schema = get_schema_from_items(items) table = test_catalog.create_item_table( arrow_schema=arrow_schema, - collection_id=sample_stac_items[0]["collection"], + collection_id=expected_items[0]["collection"], ) # Load initial items - load_items(sample_stac_items, table, method="upsert") + load_items(items, table, method="upsert") # Modify items (same IDs but different data) modified_items = [] - for item in sample_stac_items: + for item in expected_items: modified_item = item.copy() modified_item["properties"] = item["properties"].copy() modified_item["properties"]["title"] = f"Updated {item['properties']['title']}" @@ -76,9 +84,9 @@ def test_load_items_upsert_updates_existing( # Load modified items with upsert load_items(modified_items, table, method="upsert") - # Verify only 3 records exist (not 6) and they have updated titles + # Verify only the original record count exists and they have updated titles result = table.scan().to_arrow() - assert len(result) == len(sample_stac_items) + assert len(result) == len(expected_items) # Check that titles were updated titles = result.column("title").to_pylist() @@ -87,98 +95,83 @@ def test_load_items_upsert_updates_existing( def test_load_items_append( test_catalog: IcestacCatalog, - sample_stac_items: list[dict[str, Any]], + items: ItemsInput, ) -> None: """Test loading items with append method.""" + expected_items = items_to_list(items) + # Create the table - arrow_schema = get_schema_from_item(sample_stac_items[0]) + arrow_schema = get_schema_from_items(items) table = test_catalog.create_item_table( arrow_schema=arrow_schema, - collection_id=sample_stac_items[0]["collection"], + collection_id=expected_items[0]["collection"], ) # Load items with append method - load_items(sample_stac_items, table, method="append") + load_items(items, table, method="append") # Verify records were inserted result = table.scan().to_arrow() - assert len(result) == len(sample_stac_items) + assert len(result) == len(expected_items) def test_load_items_append_creates_duplicates( test_catalog: IcestacCatalog, - sample_stac_items: list[dict[str, Any]], + items: ItemsInput, ) -> None: """Test that append creates duplicate records when IDs overlap.""" + expected_items = items_to_list(items) + # Create the table - arrow_schema = get_schema_from_item(sample_stac_items[0]) + arrow_schema = get_schema_from_items(items) table = test_catalog.create_item_table( arrow_schema=arrow_schema, - collection_id=sample_stac_items[0]["collection"], + collection_id=expected_items[0]["collection"], ) # Load items twice with append - load_items(sample_stac_items, table, method="append") - load_items(sample_stac_items, table, method="append") + load_items(items, table, method="append") + load_items(items, table, method="append") # Verify we have double the records (append doesn't deduplicate) result = table.scan().to_arrow() - assert len(result) == len(sample_stac_items) * 2 + assert len(result) == len(expected_items) * 2 def test_load_items_multiple_batches( test_catalog: IcestacCatalog, - sample_stac_items: list[dict[str, Any]], + items: ItemsInput, ) -> None: """Test loading items in multiple batches with different methods.""" + expected_items = items_to_list(items) + # Create the table - arrow_schema = get_schema_from_item(sample_stac_items[0]) + arrow_schema = get_schema_from_items(items) table = test_catalog.create_item_table( arrow_schema=arrow_schema, - collection_id=sample_stac_items[0]["collection"], + collection_id=expected_items[0]["collection"], ) # Load first batch - load_items(sample_stac_items[:2], table, method="upsert") + load_items(expected_items[:2], table, method="upsert") result = table.scan().to_arrow() - assert len(result) == 2 + assert len(result) == min(2, len(expected_items)) # Load second batch - load_items(sample_stac_items[2:], table, method="upsert") + if len(expected_items) > 2: + load_items(expected_items[2:], table, method="upsert") result = table.scan().to_arrow() - assert len(result) == 3 - - -def test_load_items_single_item( - test_catalog: IcestacCatalog, - sample_stac_item: dict[str, Any], -) -> None: - """Test loading a single item.""" - # Create the table - arrow_schema = get_schema_from_item(sample_stac_item) - table = test_catalog.create_item_table( - arrow_schema=arrow_schema, - collection_id=sample_stac_item["collection"], - ) - - # Load single item as a list - load_items([sample_stac_item], table) - - # Verify record was inserted - result = table.scan().to_arrow() - assert len(result) == 1 - assert result.column("id").to_pylist()[0] == sample_stac_item["id"] + assert len(result) == len(expected_items) def test_load_items_different_schema( test_catalog: IcestacCatalog, sample_stac_item: dict[str, Any], ) -> None: - # Create the table - arrow_schema = get_schema_from_item(sample_stac_item) + arrow_schema = get_schema_from_items(sample_stac_item) table = test_catalog.create_item_table( arrow_schema=arrow_schema, collection_id=sample_stac_item["collection"], diff --git a/tests/test_schema.py b/tests/test_schema.py index c9909aa..165531e 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -5,12 +5,12 @@ from arro3.core import Schema from pydantic import ValidationError -from icestac.schema import IcestacItem, get_schema_from_item +from icestac.schema import IcestacItem, get_schema_from_items -def test_get_schema_from_item(sample_stac_item: dict[str, Any]) -> None: +def test_get_schema_from_items(sample_stac_item: dict[str, Any]) -> None: """Test that we can extract an Arrow schema from a STAC item.""" - schema = get_schema_from_item(sample_stac_item) + schema = get_schema_from_items(sample_stac_item) assert isinstance(schema, Schema) assert "id" in schema.names @@ -18,25 +18,25 @@ def test_get_schema_from_item(sample_stac_item: dict[str, Any]) -> None: assert "collection" in schema.names -def test_get_schema_from_item_validates(sample_stac_item: dict[str, Any]) -> None: - """Test that get_schema_from_item validates the STAC item.""" +def test_get_schema_from_items_validates(sample_stac_item: dict[str, Any]) -> None: + """Test that get_schema_from_items validates the STAC item.""" invalid_item = {"not": "a stac item"} with pytest.raises(ValidationError): - get_schema_from_item(invalid_item) + get_schema_from_items(invalid_item) -def test_get_schema_from_item_no_collection(sample_stac_item: dict[str, Any]) -> None: +def test_get_schema_from_items_no_collection(sample_stac_item: dict[str, Any]) -> None: """Test that missing collection field raises ValueError.""" _ = sample_stac_item.pop("collection") with pytest.raises(ValidationError): - _ = get_schema_from_item(sample_stac_item) + _ = get_schema_from_items(sample_stac_item) def test_validate_schema_valid(sample_stac_item: dict[str, Any]) -> None: """Test that a valid STAC schema passes validation.""" - schema = get_schema_from_item(sample_stac_item) + schema = get_schema_from_items(sample_stac_item) # Should not raise IcestacItem.validate_schema(schema) From 80fa4a49903a7321009e183dffdfab3d9d3d54a1 Mon Sep 17 00:00:00 2001 From: hrodmn Date: Tue, 19 May 2026 15:28:36 -0500 Subject: [PATCH 15/23] chore: fix type checks --- src/icestac/load.py | 5 +++-- src/icestac/schema.py | 6 +++--- tests/helpers.py | 5 +++-- tests/test_catalog.py | 4 +++- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/icestac/load.py b/src/icestac/load.py index 954a0c2..2cc33f3 100644 --- a/src/icestac/load.py +++ b/src/icestac/load.py @@ -1,4 +1,4 @@ -from typing import Literal +from typing import Any, Literal, cast import pyarrow import rustac @@ -16,7 +16,8 @@ def load_items( method: Method = "upsert", ) -> None: if isinstance(items, dict): - items = [items] + item = cast(dict[str, Any], items) + items = [item] if not isinstance(items, ArrowTable): items = rustac.to_arrow(items) diff --git a/src/icestac/schema.py b/src/icestac/schema.py index 727d2d8..8931bb3 100644 --- a/src/icestac/schema.py +++ b/src/icestac/schema.py @@ -1,4 +1,4 @@ -from typing import Any +from typing import Any, cast import pyarrow as pa import rustac @@ -105,8 +105,8 @@ def _first_item_from_arrow(items: ArrowTable) -> dict[str, Any]: def get_schema_from_items(items: ItemsInput) -> ArrowSchema: if isinstance(items, dict): - item = items - items = [items] + item = cast(dict[str, Any], items) + items = [item] elif isinstance(items, list): item = items[0] elif isinstance(items, ArrowTable): diff --git a/tests/helpers.py b/tests/helpers.py index 35777b9..72f48e4 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,4 +1,4 @@ -from typing import Any +from typing import Any, cast import pyarrow as pa import rustac @@ -9,7 +9,8 @@ def items_to_list(items: ItemsInput) -> list[dict[str, Any]]: if isinstance(items, dict): - return [items] + item = cast(dict[str, Any], items) + return [item] if isinstance(items, ArrowTable): return rustac.from_arrow(pa.table(items))["features"] return items diff --git a/tests/test_catalog.py b/tests/test_catalog.py index 7567e43..a787e95 100644 --- a/tests/test_catalog.py +++ b/tests/test_catalog.py @@ -25,7 +25,9 @@ def test_create_item_table( assert table.schema().find_field("datetime") # Ensure data has required fields marked as non-nullable to match table schema - arrow_data = to_arrow(expected_items) if not isinstance(items, ArrowTable) else items + arrow_data = ( + to_arrow(expected_items) if not isinstance(items, ArrowTable) else items + ) enforced_schema = IcestacItem.enforce_required_fields(arrow_data.schema) arrow_table = pyarrow.table(arrow_data).cast(pyarrow.schema(enforced_schema)) From 6859e4a19fafcadbe2d5e511ad8c076a48d54708 Mon Sep 17 00:00:00 2001 From: hrodmn Date: Tue, 19 May 2026 15:52:31 -0500 Subject: [PATCH 16/23] chore: update project metadata and README --- README.md | 66 +++++++++++++++++++++++++++++++++++++++++++++++--- main.py | 7 +++++- pyproject.toml | 2 +- uv.lock | 9 ++----- 4 files changed, 72 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 7d0ce36..81e96f5 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,13 @@ ## Overview -This project creates a Python library (`icestac`) that uses rustac to convert STAC item collections to arrow tables and writes them to Apache Iceberg tables. +This project creates a Python library (`icestac`) that uses rustac to convert STAC item collections to Arrow tables and writes them to Apache Iceberg tables. + +At a high level, the library is split into three pieces: + +- `src/icestac/schema.py` validates STAC items with `stac-pydantic`, derives an Arrow schema from incoming items, and converts that schema to an Iceberg schema with stable field IDs. +- `src/icestac/catalog.py` wraps a PyIceberg catalog and creates per-collection item tables in the `icestac` namespace. +- `src/icestac/load.py` turns STAC items into Arrow data and writes them to Iceberg with either `append` or `upsert` semantics. The goal is a stac-geoparquet-backed system that can be used to maintain a **STAC Catalog** with many collections and support real-time ingestion. It will include an event-driven AWS pipeline for ingesting STAC items into an Iceberg catalog via SNS/SQS and Lambda. @@ -48,13 +54,67 @@ PyIceberg will pick this up automatically when running from the project director **3. Load sample items:** -`main.py` fetches 5 items from the `icesat2-boreal-v3.1-agb` collection on the MAAP STAC API and writes them to the local Iceberg catalog: +`main.py` is the best example of the current ingestion workflow: ```bash uv run python main.py ``` -This creates an `icestac.icesat2_boreal_v3_1_agb` table in the catalog and upserts the items. +First, load the default PyIceberg catalog from `.pyiceberg.yaml` and wrap it with `IcestacCatalog`: + +```python +catalog = IcestacCatalog(catalog=load_catalog()) +``` + +That gives `icestac` a catalog client that knows how to create and load item tables in the `icestac` namespace. + +Next, fetch a collection of STAC items from a STAC API: + +```python +items = await rustac.search( + "https://stac.maap-project.org", + collections="icesat2-boreal-v3.1-agb", + max_items=200, +) +``` + +Here `rustac.search(...)` pulls pages of 200 items from the MAAP STAC API. In a real application, those items could also come from a webhook, a queue, or another ingestion step. + +Then normalize the collection id into something that will work as an Iceberg table name and write that value onto each item: + +```python +collection_id = "icesat2_boreal_v3_1_agb" +for item in items: + item["collection"] = collection_id +``` + +The sample uses an Iceberg-safe table id with underscores. It also ensures every item carries the collection value that will be stored in the table. + +Once the items are in hand, derive the schema and create the Iceberg table: + +```python +schema = get_schema_from_items(items) +catalog.create_item_table(arrow_schema=schema, collection_id=collection_id) +``` + +`get_schema_from_items(...)` validates the items as STAC, derives an Arrow schema, and marks required STAC fields as non-nullable. `create_item_table(...)` converts that Arrow schema to an Iceberg schema and creates `icestac.icesat2_boreal_v3_1_agb`, currently partitioned by `datetime` month. + +Finally, split the items into batches and upsert them into Iceberg: + +```python +batches = [items[i : i + BATCH_SIZE] for i in range(0, len(items), BATCH_SIZE)] +for i, batch in enumerate(batches, start=1): + logger.info("Loading batch %d/%d (%d items)", i, len(batches), len(batch)) + catalog.load_items( + collection_id=collection_id, + items=batch, + method="upsert", + ) +``` + +`catalog.load_items(...)` converts each batch to Arrow and writes it to Iceberg. In `upsert` mode, the table uses STAC `id` as the join key, so rerunning the workflow updates existing items instead of blindly appending duplicates. + +That is the core `icestac` usage pattern today: generate items, set the collection id, derive a schema, create the collection table, and then append or upsert batches into Iceberg. **4. Query with DuckDB:** diff --git a/main.py b/main.py index da64240..27cd5c0 100644 --- a/main.py +++ b/main.py @@ -3,6 +3,7 @@ import rustac from pyiceberg.catalog import load_catalog +from pyiceberg.exceptions import TableAlreadyExistsError from icestac.catalog import IcestacCatalog from icestac.schema import get_schema_from_items @@ -27,7 +28,11 @@ async def run(): item["collection"] = collection_id schema = get_schema_from_items(items) - catalog.create_item_table(arrow_schema=schema, collection_id=collection_id) + + try: + catalog.create_item_table(arrow_schema=schema, collection_id=collection_id) + except TableAlreadyExistsError: + logger.warning(f"{collection_id} table already exists... skipping") batches = [items[i : i + BATCH_SIZE] for i in range(0, len(items), BATCH_SIZE)] for i, batch in enumerate(batches, start=1): diff --git a/pyproject.toml b/pyproject.toml index 2849309..40d6efe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "icestac" -version = "0.0.1" +version = "0.0.0" description = "Manage STAC metadata in Apache Iceberg" readme = "README.md" authors = [ diff --git a/uv.lock b/uv.lock index d78f9cb..e042fb7 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.11" [[package]] @@ -417,7 +417,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/e8/2e1462c8fdbe0f210feb5ac7ad2d9029af8be3bf45bd9fa39765f821642f/greenlet-3.3.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:5fd23b9bc6d37b563211c6abbb1b3cab27db385a4449af5c32e932f93017080c", size = 274974, upload-time = "2026-01-23T15:31:02.891Z" }, { url = "https://files.pythonhosted.org/packages/7e/a8/530a401419a6b302af59f67aaf0b9ba1015855ea7e56c036b5928793c5bd/greenlet-3.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09f51496a0bfbaa9d74d36a52d2580d1ef5ed4fdfcff0a73730abfbbbe1403dd", size = 577175, upload-time = "2026-01-23T16:00:56.213Z" }, { url = "https://files.pythonhosted.org/packages/8e/89/7e812bb9c05e1aaef9b597ac1d0962b9021d2c6269354966451e885c4e6b/greenlet-3.3.1-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb0feb07fe6e6a74615ee62a880007d976cf739b6669cce95daa7373d4fc69c5", size = 590401, upload-time = "2026-01-23T16:05:26.365Z" }, - { url = "https://files.pythonhosted.org/packages/70/ae/e2d5f0e59b94a2269b68a629173263fa40b63da32f5c231307c349315871/greenlet-3.3.1-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:67ea3fc73c8cd92f42467a72b75e8f05ed51a0e9b1d15398c913416f2dafd49f", size = 601161, upload-time = "2026-01-23T16:15:53.456Z" }, { url = "https://files.pythonhosted.org/packages/5c/ae/8d472e1f5ac5efe55c563f3eabb38c98a44b832602e12910750a7c025802/greenlet-3.3.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:39eda9ba259cc9801da05351eaa8576e9aa83eb9411e8f0c299e05d712a210f2", size = 590272, upload-time = "2026-01-23T15:32:49.411Z" }, { url = "https://files.pythonhosted.org/packages/a8/51/0fde34bebfcadc833550717eade64e35ec8738e6b097d5d248274a01258b/greenlet-3.3.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e2e7e882f83149f0a71ac822ebf156d902e7a5d22c9045e3e0d1daf59cee2cc9", size = 1550729, upload-time = "2026-01-23T16:04:20.867Z" }, { url = "https://files.pythonhosted.org/packages/16/c9/2fb47bee83b25b119d5a35d580807bb8b92480a54b68fef009a02945629f/greenlet-3.3.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:80aa4d79eb5564f2e0a6144fcc744b5a37c56c4a92d60920720e99210d88db0f", size = 1615552, upload-time = "2026-01-23T15:33:45.743Z" }, @@ -426,7 +425,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f9/c8/9d76a66421d1ae24340dfae7e79c313957f6e3195c144d2c73333b5bfe34/greenlet-3.3.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:7e806ca53acf6d15a888405880766ec84721aa4181261cd11a457dfe9a7a4975", size = 276443, upload-time = "2026-01-23T15:30:10.066Z" }, { url = "https://files.pythonhosted.org/packages/81/99/401ff34bb3c032d1f10477d199724f5e5f6fbfb59816ad1455c79c1eb8e7/greenlet-3.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d842c94b9155f1c9b3058036c24ffb8ff78b428414a19792b2380be9cecf4f36", size = 597359, upload-time = "2026-01-23T16:00:57.394Z" }, { url = "https://files.pythonhosted.org/packages/2b/bc/4dcc0871ed557792d304f50be0f7487a14e017952ec689effe2180a6ff35/greenlet-3.3.1-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:20fedaadd422fa02695f82093f9a98bad3dab5fcda793c658b945fcde2ab27ba", size = 607805, upload-time = "2026-01-23T16:05:28.068Z" }, - { url = "https://files.pythonhosted.org/packages/3b/cd/7a7ca57588dac3389e97f7c9521cb6641fd8b6602faf1eaa4188384757df/greenlet-3.3.1-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c620051669fd04ac6b60ebc70478210119c56e2d5d5df848baec4312e260e4ca", size = 622363, upload-time = "2026-01-23T16:15:54.754Z" }, { url = "https://files.pythonhosted.org/packages/cf/05/821587cf19e2ce1f2b24945d890b164401e5085f9d09cbd969b0c193cd20/greenlet-3.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:14194f5f4305800ff329cbf02c5fcc88f01886cadd29941b807668a45f0d2336", size = 609947, upload-time = "2026-01-23T15:32:51.004Z" }, { url = "https://files.pythonhosted.org/packages/a4/52/ee8c46ed9f8babaa93a19e577f26e3d28a519feac6350ed6f25f1afee7e9/greenlet-3.3.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7b2fe4150a0cf59f847a67db8c155ac36aed89080a6a639e9f16df5d6c6096f1", size = 1567487, upload-time = "2026-01-23T16:04:22.125Z" }, { url = "https://files.pythonhosted.org/packages/8f/7c/456a74f07029597626f3a6db71b273a3632aecb9afafeeca452cfa633197/greenlet-3.3.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:49f4ad195d45f4a66a0eb9c1ba4832bb380570d361912fa3554746830d332149", size = 1636087, upload-time = "2026-01-23T15:33:47.486Z" }, @@ -435,7 +433,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/ab/d26750f2b7242c2b90ea2ad71de70cfcd73a948a49513188a0fc0d6fc15a/greenlet-3.3.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:7ab327905cabb0622adca5971e488064e35115430cec2c35a50fd36e72a315b3", size = 275205, upload-time = "2026-01-23T15:30:24.556Z" }, { url = "https://files.pythonhosted.org/packages/10/d3/be7d19e8fad7c5a78eeefb2d896a08cd4643e1e90c605c4be3b46264998f/greenlet-3.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:65be2f026ca6a176f88fb935ee23c18333ccea97048076aef4db1ef5bc0713ac", size = 599284, upload-time = "2026-01-23T16:00:58.584Z" }, { url = "https://files.pythonhosted.org/packages/ae/21/fe703aaa056fdb0f17e5afd4b5c80195bbdab701208918938bd15b00d39b/greenlet-3.3.1-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7a3ae05b3d225b4155bda56b072ceb09d05e974bc74be6c3fc15463cf69f33fd", size = 610274, upload-time = "2026-01-23T16:05:29.312Z" }, - { url = "https://files.pythonhosted.org/packages/06/00/95df0b6a935103c0452dad2203f5be8377e551b8466a29650c4c5a5af6cc/greenlet-3.3.1-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:12184c61e5d64268a160226fb4818af4df02cfead8379d7f8b99a56c3a54ff3e", size = 624375, upload-time = "2026-01-23T16:15:55.915Z" }, { url = "https://files.pythonhosted.org/packages/cb/86/5c6ab23bb3c28c21ed6bebad006515cfe08b04613eb105ca0041fecca852/greenlet-3.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6423481193bbbe871313de5fd06a082f2649e7ce6e08015d2a76c1e9186ca5b3", size = 612904, upload-time = "2026-01-23T15:32:52.317Z" }, { url = "https://files.pythonhosted.org/packages/c2/f3/7949994264e22639e40718c2daf6f6df5169bf48fb038c008a489ec53a50/greenlet-3.3.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:33a956fe78bbbda82bfc95e128d61129b32d66bcf0a20a1f0c08aa4839ffa951", size = 1567316, upload-time = "2026-01-23T16:04:23.316Z" }, { url = "https://files.pythonhosted.org/packages/8d/6e/d73c94d13b6465e9f7cd6231c68abde838bb22408596c05d9059830b7872/greenlet-3.3.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4b065d3284be43728dd280f6f9a13990b56470b81be20375a207cdc814a983f2", size = 1636549, upload-time = "2026-01-23T15:33:48.643Z" }, @@ -444,7 +441,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ae/fb/011c7c717213182caf78084a9bea51c8590b0afda98001f69d9f853a495b/greenlet-3.3.1-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:bd59acd8529b372775cd0fcbc5f420ae20681c5b045ce25bd453ed8455ab99b5", size = 275737, upload-time = "2026-01-23T15:32:16.889Z" }, { url = "https://files.pythonhosted.org/packages/41/2e/a3a417d620363fdbb08a48b1dd582956a46a61bf8fd27ee8164f9dfe87c2/greenlet-3.3.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b31c05dd84ef6871dd47120386aed35323c944d86c3d91a17c4b8d23df62f15b", size = 646422, upload-time = "2026-01-23T16:01:00.354Z" }, { url = "https://files.pythonhosted.org/packages/b4/09/c6c4a0db47defafd2d6bab8ddfe47ad19963b4e30f5bed84d75328059f8c/greenlet-3.3.1-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:02925a0bfffc41e542c70aa14c7eda3593e4d7e274bfcccca1827e6c0875902e", size = 658219, upload-time = "2026-01-23T16:05:30.956Z" }, - { url = "https://files.pythonhosted.org/packages/e2/89/b95f2ddcc5f3c2bc09c8ee8d77be312df7f9e7175703ab780f2014a0e781/greenlet-3.3.1-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3e0f3878ca3a3ff63ab4ea478585942b53df66ddde327b59ecb191b19dbbd62d", size = 671455, upload-time = "2026-01-23T16:15:57.232Z" }, { url = "https://files.pythonhosted.org/packages/80/38/9d42d60dffb04b45f03dbab9430898352dba277758640751dc5cc316c521/greenlet-3.3.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:34a729e2e4e4ffe9ae2408d5ecaf12f944853f40ad724929b7585bca808a9d6f", size = 660237, upload-time = "2026-01-23T15:32:53.967Z" }, { url = "https://files.pythonhosted.org/packages/96/61/373c30b7197f9e756e4c81ae90a8d55dc3598c17673f91f4d31c3c689c3f/greenlet-3.3.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:aec9ab04e82918e623415947921dea15851b152b822661cce3f8e4393c3df683", size = 1615261, upload-time = "2026-01-23T16:04:25.066Z" }, { url = "https://files.pythonhosted.org/packages/fd/d3/ca534310343f5945316f9451e953dcd89b36fe7a19de652a1dc5a0eeef3f/greenlet-3.3.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:71c767cf281a80d02b6c1bdc41c9468e1f5a494fb11bc8688c360524e273d7b1", size = 1683719, upload-time = "2026-01-23T15:33:50.61Z" }, @@ -453,7 +449,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/28/24/cbbec49bacdcc9ec652a81d3efef7b59f326697e7edf6ed775a5e08e54c2/greenlet-3.3.1-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:3e63252943c921b90abb035ebe9de832c436401d9c45f262d80e2d06cc659242", size = 282706, upload-time = "2026-01-23T15:33:05.525Z" }, { url = "https://files.pythonhosted.org/packages/86/2e/4f2b9323c144c4fe8842a4e0d92121465485c3c2c5b9e9b30a52e80f523f/greenlet-3.3.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:76e39058e68eb125de10c92524573924e827927df5d3891fbc97bd55764a8774", size = 651209, upload-time = "2026-01-23T16:01:01.517Z" }, { url = "https://files.pythonhosted.org/packages/d9/87/50ca60e515f5bb55a2fbc5f0c9b5b156de7d2fc51a0a69abc9d23914a237/greenlet-3.3.1-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9f9d5e7a9310b7a2f416dd13d2e3fd8b42d803968ea580b7c0f322ccb389b97", size = 654300, upload-time = "2026-01-23T16:05:32.199Z" }, - { url = "https://files.pythonhosted.org/packages/7c/25/c51a63f3f463171e09cb586eb64db0861eb06667ab01a7968371a24c4f3b/greenlet-3.3.1-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4b9721549a95db96689458a1e0ae32412ca18776ed004463df3a9299c1b257ab", size = 662574, upload-time = "2026-01-23T16:15:58.364Z" }, { url = "https://files.pythonhosted.org/packages/1d/94/74310866dfa2b73dd08659a3d18762f83985ad3281901ba0ee9a815194fb/greenlet-3.3.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:92497c78adf3ac703b57f1e3813c2d874f27f71a178f9ea5887855da413cd6d2", size = 653842, upload-time = "2026-01-23T15:32:55.671Z" }, { url = "https://files.pythonhosted.org/packages/97/43/8bf0ffa3d498eeee4c58c212a3905dd6146c01c8dc0b0a046481ca29b18c/greenlet-3.3.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ed6b402bc74d6557a705e197d47f9063733091ed6357b3de33619d8a8d93ac53", size = 1614917, upload-time = "2026-01-23T16:04:26.276Z" }, { url = "https://files.pythonhosted.org/packages/89/90/a3be7a5f378fc6e84abe4dcfb2ba32b07786861172e502388b4c90000d1b/greenlet-3.3.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:59913f1e5ada20fde795ba906916aea25d442abcc0593fba7e26c92b7ad76249", size = 1676092, upload-time = "2026-01-23T15:33:52.176Z" }, @@ -462,7 +457,7 @@ wheels = [ [[package]] name = "icestac" -version = "0.0.1" +version = "0.0.0" source = { editable = "." } dependencies = [ { name = "pyarrow" }, From 5a538c726cc8a875be380ef89b79842f494663b3 Mon Sep 17 00:00:00 2001 From: hrodmn Date: Thu, 20 Aug 2026 20:59:03 -0500 Subject: [PATCH 17/23] fix: enforce item table collection and schema invariants Preserve nullable STAC fields and Arrow metadata, and reject invalid or cross-collection loads. Align the HLS demo and documentation with the one-table-per-collection contract. --- .gitignore | 4 ++ README.md | 103 ++++++++++++++++++----------------------- main.py | 98 ++++++++++++++++++++++++++++++--------- pyproject.toml | 4 +- src/icestac/catalog.py | 20 +++----- src/icestac/load.py | 13 +++++- src/icestac/schema.py | 55 +++++++++------------- tests/test_catalog.py | 20 ++++++++ tests/test_load.py | 36 ++++++++++++++ tests/test_schema.py | 1 + uv.lock | 94 +++++++++++++++++++++++++++++++------ 11 files changed, 306 insertions(+), 142 deletions(-) diff --git a/.gitignore b/.gitignore index e9cd161..0e08b1d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ build/ dist/ wheels/ *.egg-info +.coverage +coverage.xml # Virtual environments .venv @@ -14,3 +16,5 @@ minio-data/ # dev docs dev-docs/plans/ dev-docs/brainstorms/ + +data/ diff --git a/README.md b/README.md index 81e96f5..634cec6 100644 --- a/README.md +++ b/README.md @@ -2,15 +2,20 @@ ## Overview -This project creates a Python library (`icestac`) that uses rustac to convert STAC item collections to Arrow tables and writes them to Apache Iceberg tables. +`icestac` uses rustac's flattened Arrow representation of STAC Items and writes it to Apache Iceberg. -At a high level, the library is split into three pieces: +The library is split into three pieces: -- `src/icestac/schema.py` validates STAC items with `stac-pydantic`, derives an Arrow schema from incoming items, and converts that schema to an Iceberg schema with stable field IDs. -- `src/icestac/catalog.py` wraps a PyIceberg catalog and creates per-collection item tables in the `icestac` namespace. -- `src/icestac/load.py` turns STAC items into Arrow data and writes them to Iceberg with either `append` or `upsert` semantics. +- `src/icestac/schema.py` validates STAC inputs, preserves Arrow metadata, and converts Arrow schemas to Iceberg schemas with field IDs. +- `src/icestac/catalog.py` wraps a PyIceberg catalog and creates one item table per collection in the `icestac` namespace. +- `src/icestac/load.py` accepts STAC dictionaries or `arro3.core.Table` data and writes it with `append` or `upsert` semantics. -The goal is a stac-geoparquet-backed system that can be used to maintain a **STAC Catalog** with many collections and support real-time ingestion. It will include an event-driven AWS pipeline for ingesting STAC items into an Iceberg catalog via SNS/SQS and Lambda. +This is an early foundation, not a released storage specification. Current boundaries are intentional: + +- A table name matches its items' collection ID. Periods are unsupported because Iceberg uses them as namespace delimiters; `icestac` does not silently rewrite IDs. +- Tables are partitioned by `datetime` month. Other temporal or spatial layouts are deferred until there are concrete query patterns. +- Table schemas do not evolve automatically. Loading a new shape fails until the Iceberg schema is updated separately. +- Geometry is stored as WKB, but PyIceberg does not currently emit the GeoParquet and STAC GeoParquet file metadata required to claim compliance with those specifications. ## Development @@ -60,61 +65,20 @@ PyIceberg will pick this up automatically when running from the project director uv run python main.py ``` -First, load the default PyIceberg catalog from `.pyiceberg.yaml` and wrap it with `IcestacCatalog`: - -```python -catalog = IcestacCatalog(catalog=load_catalog()) -``` +The script downloads four months of HLS STAC GeoParquet from public S3, reads each file directly as Arrow, and loads it through `IcestacCatalog`. -That gives `icestac` a catalog client that knows how to create and load item tables in the `icestac` namespace. +The source collection ID, `HLSS30_2.0`, contains a period and cannot be used unchanged as an Iceberg identifier. The demo explicitly migrates the dataset to `HLSS30_2_0` by replacing every item's collection value before creating the matching table. This is a dataset decision, not automatic library slugification. -Next, fetch a collection of STAC items from a STAC API: - -```python -items = await rustac.search( - "https://stac.maap-project.org", - collections="icesat2-boreal-v3.1-agb", - max_items=200, -) -``` - -Here `rustac.search(...)` pulls pages of 200 items from the MAAP STAC API. In a real application, those items could also come from a webhook, a queue, or another ingestion step. - -Then normalize the collection id into something that will work as an Iceberg table name and write that value onto each item: - -```python -collection_id = "icesat2_boreal_v3_1_agb" -for item in items: - item["collection"] = collection_id -``` - -The sample uses an Iceberg-safe table id with underscores. It also ensures every item carries the collection value that will be stored in the table. - -Once the items are in hand, derive the schema and create the Iceberg table: +The core API remains explicit: ```python +catalog = IcestacCatalog(catalog=load_catalog()) schema = get_schema_from_items(items) -catalog.create_item_table(arrow_schema=schema, collection_id=collection_id) +catalog.create_item_table(collection_id=collection_id, arrow_schema=schema) +catalog.load_items(collection_id=collection_id, items=items) ``` -`get_schema_from_items(...)` validates the items as STAC, derives an Arrow schema, and marks required STAC fields as non-nullable. `create_item_table(...)` converts that Arrow schema to an Iceberg schema and creates `icestac.icesat2_boreal_v3_1_agb`, currently partitioned by `datetime` month. - -Finally, split the items into batches and upsert them into Iceberg: - -```python -batches = [items[i : i + BATCH_SIZE] for i in range(0, len(items), BATCH_SIZE)] -for i, batch in enumerate(batches, start=1): - logger.info("Loading batch %d/%d (%d items)", i, len(batches), len(batch)) - catalog.load_items( - collection_id=collection_id, - items=batch, - method="upsert", - ) -``` - -`catalog.load_items(...)` converts each batch to Arrow and writes it to Iceberg. In `upsert` mode, the table uses STAC `id` as the join key, so rerunning the workflow updates existing items instead of blindly appending duplicates. - -That is the core `icestac` usage pattern today: generate items, set the collection id, derive a schema, create the collection table, and then append or upsert batches into Iceberg. +`get_schema_from_items(...)` accepts one STAC dictionary, a list of dictionaries, or an `arro3.core.Table`. `load_items(...)` accepts the same inputs, checks that every row belongs to the target collection, and upserts on STAC `id` by default. **4. Query with DuckDB:** @@ -147,11 +111,13 @@ ATTACH 'icestac' AS catalog ( ); SELECT id, datetime, collection, geometry -FROM catalog.icestac.icesat2_boreal_v3_1_agb +FROM catalog.icestac.HLSS30_2_0 LIMIT 10; SELECT count(*) -FROM catalog.icestac.icesat2_boreal_v3_1_agb; +FROM catalog.icestac.HLSS30_2_0; + +DESCRIBE SELECT bbox, geometry FROM catalog.icestac.HLSS30_2_0; ``` Or scan the table directly from its S3 path (no catalog required): @@ -159,7 +125,30 @@ Or scan the table directly from its S3 path (no catalog required): ```sql SET unsafe_enable_version_guessing = true; SELECT * -FROM iceberg_scan('s3://warehouse/icestac/icesat2_boreal_v3_1_agb') +FROM iceberg_scan('s3://warehouse/icestac/HLSS30_2_0') LIMIT 10; ``` +## Delete a Table + +To remove a table from the local Iceberg REST catalog, call `drop_table(...)` on the underlying PyIceberg catalog: + +```python +from pyiceberg.catalog import load_catalog +from icestac.catalog import IcestacCatalog + +catalog = IcestacCatalog(catalog=load_catalog()) +catalog.catalog.drop_table( + ("icestac", "HLSS30_2_0"), + purge_requested=True, +) +``` + +`purge_requested=True` asks the REST catalog to delete the underlying table data as well as the catalog entry. + +If the metadata entry is removed but table files remain in MinIO, delete the warehouse path manually: + +```bash +docker compose exec mc mc rm --recursive --force minio/warehouse/icestac/HLSS30_2_0 +``` + diff --git a/main.py b/main.py index 27cd5c0..130b428 100644 --- a/main.py +++ b/main.py @@ -1,45 +1,101 @@ import asyncio import logging -import rustac +import pyarrow +from arro3.core import Table as ArrowTable +from obstore.store import LocalStore, S3Store from pyiceberg.catalog import load_catalog from pyiceberg.exceptions import TableAlreadyExistsError +from rustac import DuckdbClient from icestac.catalog import IcestacCatalog from icestac.schema import get_schema_from_items -logger = logging.getLogger(__name__) +logger = logging.getLogger("icestac-demo") -BATCH_SIZE = 1000 +HLS_STAC_GEOPARQUET_BUCKET = "nasa-maap-data-store" +HLS_STAC_GEOPARQUET_PREFIX = "file-staging/nasa-map/hls-stac-geoparquet-archive/v2" +HLS_STAC_GEOPARQUET_PATH_FMT = ( + "{collection}/year={year}/month={month}/{collection}-{year}-{month}.parquet" +) -async def run(): +async def copy_hls_stac_geoparquet(path: str, store: LocalStore) -> None: + """Copy one public HLS STAC GeoParquet file into a local store.""" + hls_stac_store = S3Store( + bucket=HLS_STAC_GEOPARQUET_BUCKET, + prefix=HLS_STAC_GEOPARQUET_PREFIX, + region="us-west-2", + skip_signature=True, + ) + + resp = await hls_stac_store.get_async(path) + await store.put_async(path, resp) + + +async def run() -> None: + """Load several months of HLS STAC GeoParquet into local Iceberg.""" logging.basicConfig(level=logging.INFO) catalog = IcestacCatalog(catalog=load_catalog()) - items = await rustac.search( - "https://stac.maap-project.org", - collections="icesat2-boreal-v3.1-agb", - limit=200, - ) + duckdb_client = DuckdbClient() + duckdb_client.execute("SET TimeZone = 'UTC';") + local_store = LocalStore("data") - collection_id = "icesat2_boreal_v3_1_agb" - for item in items: - item["collection"] = collection_id + source_collection_id = "HLSS30_2.0" + collection_id = "HLSS30_2_0" + table_exists = False - schema = get_schema_from_items(items) + for month in ["1", "2", "3", "4"]: + logger.info("processing 2026-%s", month) + stac_geoparquet_path = HLS_STAC_GEOPARQUET_PATH_FMT.format( + collection=source_collection_id, + year="2026", + month=month, + ) + + try: + _ = local_store.head(stac_geoparquet_path) + except FileNotFoundError: + logger.info("downloading %s", stac_geoparquet_path) + await copy_hls_stac_geoparquet( + path=stac_geoparquet_path, + store=local_store, + ) + + logger.info("loading items as arrow table") + items = duckdb_client.search_to_arrow(href=f"data/{stac_geoparquet_path}") + + if not items: + raise ValueError("No items found") + + items_table = pyarrow.table(items) + collection_index = items_table.schema.get_field_index("collection") + collection_field = items_table.schema.field(collection_index) + items = ArrowTable.from_arrow( + items_table.set_column( + collection_index, + collection_field, + pyarrow.array( + [collection_id] * len(items_table), type=collection_field.type + ), + ) + ) + schema = get_schema_from_items(items) - try: - catalog.create_item_table(arrow_schema=schema, collection_id=collection_id) - except TableAlreadyExistsError: - logger.warning(f"{collection_id} table already exists... skipping") + if not table_exists: + try: + catalog.create_item_table( + arrow_schema=schema, collection_id=collection_id + ) + except TableAlreadyExistsError: + logger.warning("%s table already exists; using it", collection_id) + table_exists = True - batches = [items[i : i + BATCH_SIZE] for i in range(0, len(items), BATCH_SIZE)] - for i, batch in enumerate(batches, start=1): - logger.info("Loading batch %d/%d (%d items)", i, len(batches), len(batch)) + logger.info("loading items into icestac catalog") catalog.load_items( collection_id=collection_id, - items=batch, + items=items, method="upsert", ) diff --git a/pyproject.toml b/pyproject.toml index 40d6efe..7cba726 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,9 +24,6 @@ dependencies = [ "stac-pydantic>=3.4.0", ] -[project.scripts] -icestac = "icestac:main" - [build-system] requires = ["hatchling"] build-backend = "hatchling.build" @@ -36,6 +33,7 @@ deploy = [ "aws-cdk-lib>=2.236.0", ] dev = [ + "obstore>=0.9.4", "pre-commit>=4.5.1", "pytest>=9.0.2", "pytest-cov>=7.0.0", diff --git a/src/icestac/catalog.py b/src/icestac/catalog.py index e111861..b96c94f 100644 --- a/src/icestac/catalog.py +++ b/src/icestac/catalog.py @@ -15,7 +15,7 @@ def validate_collection_id(collection_id: str) -> None: - """Ensure collection id is valid for icestac schema""" + """Ensure a collection ID can be used as an Iceberg table name.""" if "." in collection_id: raise InvalidCollectionIdError(collection_id) @@ -23,7 +23,7 @@ def validate_collection_id(collection_id: str) -> None: @dataclass class IcestacCatalog: - """Icestac client class for pyiceberg Catalog""" + """Manage collection item tables through a PyIceberg catalog.""" catalog: Catalog namespace: str = DEFAULT_NAMESPACE @@ -36,21 +36,14 @@ def create_item_table( collection_id: str, arrow_schema: ArrowSchema, ) -> Table: - """ - Create an Iceberg table from a stac-geoparquet Arrow schema - - Converts the Arrow schema to an Iceberg schema with manually assigned field IDs, - then creates or loads the Iceberg table partitioned by datetime month. + """Create a monthly partitioned Iceberg item table for a collection. Args: - schema: arro3.core.Schema for the items in this collection - collection_id: the collection id for the items in this table - catalog: PyIceberg catalog instance - namespace: Namespace for the Iceberg table + collection_id: Collection ID, used unchanged as the table name. + arrow_schema: Arrow schema for the collection's items. Returns: - PyIceberg Table instance - + The created PyIceberg table. """ validate_collection_id(collection_id) IcestacItem.validate_schema(arrow_schema) @@ -76,6 +69,7 @@ def create_item_table( def load_items( self, collection_id: str, items: ItemsInput, method: Method = "upsert" ) -> None: + """Load items into the table matching their collection ID.""" load_items( items, table=self.catalog.load_table( diff --git a/src/icestac/load.py b/src/icestac/load.py index 2cc33f3..a389be1 100644 --- a/src/icestac/load.py +++ b/src/icestac/load.py @@ -15,6 +15,10 @@ def load_items( table: Table, method: Method = "upsert", ) -> None: + """Load STAC items into their collection's Iceberg table.""" + if method not in ("append", "upsert"): + raise ValueError(f"Unsupported load method: {method}") + if isinstance(items, dict): item = cast(dict[str, Any], items) items = [item] @@ -24,11 +28,18 @@ def load_items( enforced_schema = IcestacItem.enforce_required_fields(items.schema) arrow_table = pyarrow.table(items).cast(pyarrow.schema(enforced_schema)) + collection_ids = set(arrow_table.column("collection").unique().to_pylist()) + expected_collection_id = table.name()[-1] + if collection_ids != {expected_collection_id}: + raise ValueError( + f"Items for {expected_collection_id!r} contain collection ids " + f"{sorted(map(str, collection_ids))}" + ) if method == "upsert": table.upsert( df=arrow_table, join_cols=["id"], ) - elif method == "append": + else: table.append(df=arrow_table) diff --git a/src/icestac/schema.py b/src/icestac/schema.py index 8931bb3..821d9f3 100644 --- a/src/icestac/schema.py +++ b/src/icestac/schema.py @@ -1,8 +1,8 @@ -from typing import Any, cast +from types import NoneType +from typing import Any, cast, get_args import pyarrow as pa import rustac -from arro3.core import Field from arro3.core import Schema as ArrowSchema from arro3.core import Table as ArrowTable from pyiceberg.io.pyarrow import _pyarrow_to_schema_without_ids @@ -38,34 +38,25 @@ def get_required_fields(cls) -> set[str]: return required_fields @classmethod - def enforce_required_fields(cls, schema: ArrowSchema) -> ArrowSchema: - """ - Ensure required STAC fields are marked as non-nullable in the Arrow schema. - - Returns a schema with required fields marked as nullable=False. This ensures - the Iceberg table will enforce these fields as required. - - Args: - schema: arro3.core.Schema from rustac - - Returns: - arro3.core.Schema with required fields marked as non-nullable - """ - required_fields = cls.get_required_fields() - - new_fields = [] - - for field in schema: - if field.name in required_fields: - # Mark as non-nullable (required) - new_fields.append( - Field(name=field.name, type=field.type, nullable=False) - ) - else: - # Keep original nullable setting - new_fields.append(field) + def get_non_nullable_fields(cls) -> set[str]: + """Get STAC fields whose values cannot be null.""" + fields = { + name + for name, info in cls.model_fields.items() + if info.is_required() and NoneType not in get_args(info.annotation) + } + fields.discard("properties") + return fields - return ArrowSchema(fields=new_fields) + @classmethod + def enforce_required_fields(cls, schema: ArrowSchema) -> ArrowSchema: + """Mark non-null STAC fields as non-nullable without losing metadata.""" + non_nullable_fields = cls.get_non_nullable_fields() + fields = [ + field.with_nullable(False) if field.name in non_nullable_fields else field + for field in schema + ] + return ArrowSchema(fields=fields, metadata=schema.metadata) @classmethod def validate_schema(cls, schema: ArrowSchema) -> None: @@ -104,6 +95,7 @@ def _first_item_from_arrow(items: ArrowTable) -> dict[str, Any]: def get_schema_from_items(items: ItemsInput) -> ArrowSchema: + """Derive an enforced Arrow schema from STAC dictionaries or Arrow data.""" if isinstance(items, dict): item = cast(dict[str, Any], items) items = [item] @@ -121,10 +113,7 @@ def get_schema_from_items(items: ItemsInput) -> ArrowSchema: def convert_schema(schema: ArrowSchema) -> IcebergSchema: - """Convert the arrow schema to an iceberg schema with field ids - - Necessary because built-in converter functions do not assign field ids. - """ + """Convert an Arrow schema to an Iceberg schema with field IDs.""" _schema = _pyarrow_to_schema_without_ids( pa.schema(IcestacItem.enforce_required_fields(schema)) ) diff --git a/tests/test_catalog.py b/tests/test_catalog.py index a787e95..9888137 100644 --- a/tests/test_catalog.py +++ b/tests/test_catalog.py @@ -1,3 +1,5 @@ +from typing import Any + import pyarrow import pytest from arro3.core import Table as ArrowTable @@ -60,6 +62,24 @@ def test_create_item_table_bad_collection_id( ) +def test_load_items_rejects_a_different_collection( + test_catalog: IcestacCatalog, + test_collection_id: str, + sample_stac_item: dict[str, Any], +) -> None: + arrow_schema = get_schema_from_items(sample_stac_item) + table = test_catalog.create_item_table( + arrow_schema=arrow_schema, + collection_id=test_collection_id, + ) + sample_stac_item["collection"] = "different-collection" + + with pytest.raises(ValueError, match="different-collection"): + test_catalog.load_items(test_collection_id, sample_stac_item) + + assert len(table.scan().to_arrow()) == 0 + + def test_load_items( test_catalog: IcestacCatalog, test_collection_id: str, diff --git a/tests/test_load.py b/tests/test_load.py index 00a2001..cb73cf0 100644 --- a/tests/test_load.py +++ b/tests/test_load.py @@ -1,3 +1,4 @@ +from copy import deepcopy from typing import Any import pytest @@ -166,6 +167,41 @@ def test_load_items_multiple_batches( assert len(result) == len(expected_items) +def test_load_items_rejects_invalid_method( + test_catalog: IcestacCatalog, + sample_stac_item: dict[str, Any], +) -> None: + arrow_schema = get_schema_from_items(sample_stac_item) + table = test_catalog.create_item_table( + arrow_schema=arrow_schema, + collection_id=sample_stac_item["collection"], + ) + + with pytest.raises(ValueError, match="Unsupported load method"): + load_items(sample_stac_item, table, method="insert") # type: ignore[arg-type] + + +def test_load_items_supports_interval_datetime( + test_catalog: IcestacCatalog, + sample_stac_item: dict[str, Any], +) -> None: + interval_item = deepcopy(sample_stac_item) + interval_item["properties"] = { + "datetime": None, + "start_datetime": "2024-01-01T00:00:00Z", + "end_datetime": "2024-01-02T00:00:00Z", + } + arrow_schema = get_schema_from_items(interval_item) + table = test_catalog.create_item_table( + arrow_schema=arrow_schema, + collection_id=interval_item["collection"], + ) + + load_items(interval_item, table) + + assert table.scan().to_arrow().column("id").to_pylist() == [interval_item["id"]] + + def test_load_items_different_schema( test_catalog: IcestacCatalog, sample_stac_item: dict[str, Any], diff --git a/tests/test_schema.py b/tests/test_schema.py index 165531e..2657b58 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -16,6 +16,7 @@ def test_get_schema_from_items(sample_stac_item: dict[str, Any]) -> None: assert "id" in schema.names assert "datetime" in schema.names assert "collection" in schema.names + assert schema.field("geometry").metadata[b"ARROW:extension:name"] == b"geoarrow.wkb" def test_get_schema_from_items_validates(sample_stac_item: dict[str, Any]) -> None: diff --git a/uv.lock b/uv.lock index e042fb7..6db6ae2 100644 --- a/uv.lock +++ b/uv.lock @@ -471,6 +471,7 @@ deploy = [ { name = "aws-cdk-lib" }, ] dev = [ + { name = "obstore" }, { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-cov" }, @@ -490,6 +491,7 @@ requires-dist = [ [package.metadata.requires-dev] deploy = [{ name = "aws-cdk-lib", specifier = ">=2.236.0" }] dev = [ + { name = "obstore", specifier = ">=0.9.4" }, { name = "pre-commit", specifier = ">=4.5.1" }, { name = "pytest", specifier = ">=9.0.2" }, { name = "pytest-cov", specifier = ">=7.0.0" }, @@ -678,6 +680,68 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, ] +[[package]] +name = "obstore" +version = "0.9.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d3/7a/3a37b0bf0da898478029fcc511a0d2a7252689b1f29e46db7ae74a219c74/obstore-0.9.4.tar.gz", hash = "sha256:e2b93f1372c59da2c7e74122fc6dc4b713d84fd4528b5b500ef7f548425496b5", size = 124167, upload-time = "2026-04-22T19:51:05.261Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/25/4449a0066796b91e282d7604a66387bba399b14752598c748ea9557c4c32/obstore-0.9.4-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:0d17cd04e7f22960050a85f8daa6e274d693e8fb3b97b81eeaa293c6f9e62eb4", size = 4090743, upload-time = "2026-04-22T19:49:26.461Z" }, + { url = "https://files.pythonhosted.org/packages/93/91/639fe5f5644593b9f4bea66f8f29c7bfd4de3b3381fb74b4f7df678f505f/obstore-0.9.4-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:d4beec92710fb8826fb357baf28fb79a91ee07dcdfe73777207aa762164aaa35", size = 3876313, upload-time = "2026-04-22T19:49:28.107Z" }, + { url = "https://files.pythonhosted.org/packages/ce/71/d6675f845ebe1e3927f2dce6a2a4d5a393359274762ee00c5e6855d5f468/obstore-0.9.4-cp311-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d523c8c365ab60afb8d232614a00a92bea439a9f5c55b92486c23a47af038a1e", size = 4029950, upload-time = "2026-04-22T19:49:30.279Z" }, + { url = "https://files.pythonhosted.org/packages/0e/3a/5915a173f5c6a95f9ec186a7e29b0ce6a23bd9b04c2b0b29a351dbe2baf6/obstore-0.9.4-cp311-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee0483619088337ee365cb344fceee337e2670ec4de2a1da92ac7f6b2220f18e", size = 4129455, upload-time = "2026-04-22T19:49:31.934Z" }, + { url = "https://files.pythonhosted.org/packages/b5/a9/63c31d2d436c06c4d39ed5cb154fe54202b303854532ec09537c4ce0755b/obstore-0.9.4-cp311-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:83da348bf0a7dd84e5839c0cd54d79dcd08e0729c394e566f73a605b93b9e998", size = 4416727, upload-time = "2026-04-22T19:49:34.016Z" }, + { url = "https://files.pythonhosted.org/packages/7f/fa/23c5c6db02be0e13abcbe01c1ca94c5f7876e8c58e74cb9ac2b57b068866/obstore-0.9.4-cp311-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f282a17200bcc37b8d7a1d02a146ed41812eb6e76fd0a4c9a154f02da1b8031f", size = 4311520, upload-time = "2026-04-22T19:49:35.905Z" }, + { url = "https://files.pythonhosted.org/packages/86/f0/49f6b02dab9c05e3fd79d6129e4d9e7e9874d6e5e05369ca3b3b80a48aaa/obstore-0.9.4-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d29dcfceaa0a205ded2263d29a2a3aa206819d549e0325c1f2106f79e2658584", size = 4220536, upload-time = "2026-04-22T19:49:38.343Z" }, + { url = "https://files.pythonhosted.org/packages/50/ab/d0bfd6d68422e7d8f2204d91736c7e62767e0576ad749da442a71e7773b2/obstore-0.9.4-cp311-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:caecb912723ab8e9da8da26def249d66da4318959df2bafc0a55af64f3255902", size = 4105099, upload-time = "2026-04-22T19:49:40.384Z" }, + { url = "https://files.pythonhosted.org/packages/66/3b/f595d0ee354f9daa69438991f8818602f34bc59498c8468456a02d45fb27/obstore-0.9.4-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c1c06fec8837595a2829b5f7536d0d01e940ce10b07ad2a8594fec1cfd0b7d5", size = 4294206, upload-time = "2026-04-22T19:49:42.016Z" }, + { url = "https://files.pythonhosted.org/packages/60/54/3c5af2d59258aaa9e5bef05320658ea6e9b1f3897a3a977bf7f54a0b6ec1/obstore-0.9.4-cp311-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:c132795a789ec5ade31bf4d5b55ed321fb41d9749e9145520bf19063e1da5f7b", size = 4265047, upload-time = "2026-04-22T19:49:43.983Z" }, + { url = "https://files.pythonhosted.org/packages/fb/af/a8ba1feb81b9833b253147839da40405ec6bfa51feb3abfe909c800208a5/obstore-0.9.4-cp311-abi3-musllinux_1_2_i686.whl", hash = "sha256:c6e342360a5d0ae71486bc5f8311778aa144ec1a905c23593f8ef57b5bceae24", size = 4255361, upload-time = "2026-04-22T19:49:45.864Z" }, + { url = "https://files.pythonhosted.org/packages/15/f7/3ccc0288111e057f8ba3d99bee14f95d9e9bb00acaf6e9700e0eb4cd82c3/obstore-0.9.4-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:aeb6f7e7e862550f5020a10692ef6f02d5ba4912dba08942eb59bb7d73f93fe0", size = 4439378, upload-time = "2026-04-22T19:49:47.581Z" }, + { url = "https://files.pythonhosted.org/packages/c4/b1/3ac8b5772743c60064f3c7e02d27f346dbb58feaa99a49ee09798d1cfb00/obstore-0.9.4-cp311-abi3-win_amd64.whl", hash = "sha256:a58ef942292841f99d69ac11d19d05544c835447c8c09dacbfb7409c6374c4a1", size = 4191594, upload-time = "2026-04-22T19:49:49.308Z" }, + { url = "https://files.pythonhosted.org/packages/9d/81/8f6b6509f8df603261cdb5ddb521c49891457775669c6ad857812bf4a7c1/obstore-0.9.4-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:fff17f59390ed307afcd1fb18c56076c1f911dd9f5c2636b7d7133c4d07f8c3f", size = 4071300, upload-time = "2026-04-22T19:49:51.386Z" }, + { url = "https://files.pythonhosted.org/packages/ab/fe/0c74ddf3ab9b24ef356925bfb613bc7846f869220361a784b63f754d8563/obstore-0.9.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4527c4c7889f1bd1f1952017d74774870e14e199d6b50b9e72f291f9498d898c", size = 3870593, upload-time = "2026-04-22T19:49:53.481Z" }, + { url = "https://files.pythonhosted.org/packages/73/fa/260ec94f9a7b4f4c8afbdd016710bed0736615488d3ac0c5620f9179bfcd/obstore-0.9.4-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a57c2016e3e569de35050f95c679ffe61813c4e3cb6d6028c4c3f57231021eb4", size = 4023990, upload-time = "2026-04-22T19:49:55.644Z" }, + { url = "https://files.pythonhosted.org/packages/8d/84/5b8e2b9607fb93c96a39a4cfa6d37bd3049ebf7265d0e9f8afa938bf32fe/obstore-0.9.4-cp313-cp313t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bd5327cee4fb3578b51beb1c92915cc3a05ffe794be40f50bd68d27e97d78c5c", size = 4119971, upload-time = "2026-04-22T19:49:57.745Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2b/e6c093acb7e62009d5b1678d82839903287c29d4a6e1dfbea8fbf41313d5/obstore-0.9.4-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12b1e6105eafe02d8973dbeb2d274eeac2271c67f1126ffa16f18ddea8dd5443", size = 4407147, upload-time = "2026-04-22T19:49:59.928Z" }, + { url = "https://files.pythonhosted.org/packages/ce/3d/5c93a9adee8f045b89d5f21b337f53667499db770bda129f805723ab14e4/obstore-0.9.4-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0b0d378248fda4e36652808d73eaaeb7e67154427e6c724248c9b0b9b03e70a6", size = 4312215, upload-time = "2026-04-22T19:50:01.534Z" }, + { url = "https://files.pythonhosted.org/packages/9a/de/507f60b4e6a8c0cad9f93a51a7b28132c9db49e20aadbcd542fa2abc57c4/obstore-0.9.4-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a78fb77c346abd2bcdfa071d7166be2bdc38c28573ae5a230746df6158a5593e", size = 4216936, upload-time = "2026-04-22T19:50:03.244Z" }, + { url = "https://files.pythonhosted.org/packages/7b/ff/612bd5f8258349bfe9e8c349d184b5ea3333038d4cce0d003eefafb2160c/obstore-0.9.4-cp313-cp313t-manylinux_2_24_aarch64.whl", hash = "sha256:f4e5a6dfe6877fb599868d560d6fcf4d7416cadbdf3bd947254b53830c2f11c0", size = 4105091, upload-time = "2026-04-22T19:50:05.038Z" }, + { url = "https://files.pythonhosted.org/packages/e3/73/b083b99e7bc0b529bee7b4437cafd7cc7d9f59c10995a48b6c26447fdf7f/obstore-0.9.4-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:f8114a2b84268c991232d89b105d9239299b6afb56e4941a61c09f3a89033022", size = 4292570, upload-time = "2026-04-22T19:50:06.823Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cd/3c4555f98db9a49432bc0afa68bfc33dd47bdfa3699c915b4b0e887577e3/obstore-0.9.4-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:9d7b959f5f74532a142fb449c0bef5814dfe3fa5c43c31ac4284a15221a75aaf", size = 4261946, upload-time = "2026-04-22T19:50:08.789Z" }, + { url = "https://files.pythonhosted.org/packages/96/f8/bdc66df3d0dfdcfb3931a585a7fb3b74336619baf6d3540b1425b424232b/obstore-0.9.4-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:a8e9101fc2659dd938e7ae06512075bc0a8f02ab28d2ee438d6fca8b4f3bdfba", size = 4245595, upload-time = "2026-04-22T19:50:10.765Z" }, + { url = "https://files.pythonhosted.org/packages/d7/22/1aa58ea676293e5b888391c8433ff6ab8f66622aae30427287f9daac6d46/obstore-0.9.4-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:538384255545b5c575497fcab26389c8f01707402b6ddcdd73b769b66311635d", size = 4436599, upload-time = "2026-04-22T19:50:12.585Z" }, + { url = "https://files.pythonhosted.org/packages/1d/9e/b52f2c97be27952d488cf1980af0c635f9947003e5744e3e1dc6252f0040/obstore-0.9.4-cp313-cp313t-win_amd64.whl", hash = "sha256:eef1c772657bb1293adad0d671ca1ff1e1dcae84ec4dfbf1a34e47c2a1f134ac", size = 4180463, upload-time = "2026-04-22T19:50:14.288Z" }, + { url = "https://files.pythonhosted.org/packages/19/76/c53583f95c6811057abd3116756dca46785318d564a0e99c207cbb2d8938/obstore-0.9.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:e009e7437770c85beae4c32cb79f662f0a9922676ef127e943d107a5c082d38d", size = 4071302, upload-time = "2026-04-22T19:50:15.967Z" }, + { url = "https://files.pythonhosted.org/packages/2f/23/ac3b9c05a09b3d5f178ed6f288c5d6913df8f7386059590194e0fee65d15/obstore-0.9.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ac5f3ad314bd4592fe484b79c229518be7bb5f6218bed33c20742026d5caf860", size = 3870813, upload-time = "2026-04-22T19:50:17.62Z" }, + { url = "https://files.pythonhosted.org/packages/f7/ec/c3458e0f24d2d1a4f185f541905b07e51c91b3fec589b1600c77d511e585/obstore-0.9.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:db79d5ebc4177360565ffcec4abd49930cf052cdbeb94e3a3ece2e2d08f087d0", size = 4024237, upload-time = "2026-04-22T19:50:19.81Z" }, + { url = "https://files.pythonhosted.org/packages/a7/eb/6cf468a200e491fdc6c04075e2fbbac1707bbecd243f0f56ae1e75d052ed/obstore-0.9.4-cp314-cp314t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:05b565d89c3115fb74385852dd628e12f6645a1bba97523dceae016b538a3f33", size = 4119635, upload-time = "2026-04-22T19:50:21.605Z" }, + { url = "https://files.pythonhosted.org/packages/81/fb/b44d002767fa5af95ab4ca8e16c3a9057fc11f13de03f498b99adf0c4e50/obstore-0.9.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7dfc4fc98403d8fbb316eb04257c8122b6f1dda37e80869491fdacf60a815e4c", size = 4406906, upload-time = "2026-04-22T19:50:23.654Z" }, + { url = "https://files.pythonhosted.org/packages/4b/18/9a75ad5082cd581c4a55f0e62bedf4b030a8b53824976fc1f030eff225b3/obstore-0.9.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c69af620fd3d06a8cfb62d25faf1adb6ccc97cc572f47ee04dddcde5a5e5444e", size = 4311826, upload-time = "2026-04-22T19:50:25.458Z" }, + { url = "https://files.pythonhosted.org/packages/8d/03/b0f945b31f40364a7ed4dbc5677abc66331fcf478732f4d643e17e56bb13/obstore-0.9.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa78c0230e0b9d49b25ed18980e1751331ddfe05782d6ce97579a9ccda8229ea", size = 4217086, upload-time = "2026-04-22T19:50:27.266Z" }, + { url = "https://files.pythonhosted.org/packages/1a/26/bdd85264c806802086f21d73cc7c95a5baca5feeeac4bce8acb97142163f/obstore-0.9.4-cp314-cp314t-manylinux_2_24_aarch64.whl", hash = "sha256:c828719f0bb310a9cf0e0f08cb62a0b8cc550138617cb03ac897900aec9d3d47", size = 4105560, upload-time = "2026-04-22T19:50:29.324Z" }, + { url = "https://files.pythonhosted.org/packages/6a/36/4a4a6a398e5f145edd1886388ebe5e6f6bbaf74950a5dea1a6ceae63e6b5/obstore-0.9.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:49a0455519f284b6bc2e0694298114926aff1d1f3d5d344e9163e03b446826cc", size = 4292582, upload-time = "2026-04-22T19:50:31.028Z" }, + { url = "https://files.pythonhosted.org/packages/39/4c/9caa197cd2eba726e9a5285db34027049b9527a23e1a7e08479678ad6a4a/obstore-0.9.4-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cf437309fc0fe852591ae50405300490229f876ea06574651fd753ca3fd23f25", size = 4261613, upload-time = "2026-04-22T19:50:32.868Z" }, + { url = "https://files.pythonhosted.org/packages/b8/94/a3fbe6fb3ee1c57fd4943ddbb21848eea3925b77e0789614c857d86b795e/obstore-0.9.4-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:d83dbd20b6a5d42e35794ef64046de39040854829ec4f1eb2f6dfb54df48cc3d", size = 4245638, upload-time = "2026-04-22T19:50:35.009Z" }, + { url = "https://files.pythonhosted.org/packages/56/a7/d18e168f318327d63512dfa7cf3b5e89ed9bfba6d6a8917ad7d4700b8657/obstore-0.9.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5a0c337f37f30a2d66555d69bf3abd840457a279c57ede93bd02e014721ed364", size = 4437226, upload-time = "2026-04-22T19:50:36.635Z" }, + { url = "https://files.pythonhosted.org/packages/f4/ce/66aadd155db1e273c6ec2236c0fb904666d10c2e3b791b40624c272e586c/obstore-0.9.4-cp314-cp314t-win_amd64.whl", hash = "sha256:24e37a1c713c95a964e119f8ef879415a495432162e74e80ed29d645aeeca114", size = 4180746, upload-time = "2026-04-22T19:50:38.396Z" }, + { url = "https://files.pythonhosted.org/packages/f5/01/cc2446a87e051ce567817a4b61ca3a9c62297a4c5a0fba4ec6659123fe24/obstore-0.9.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c7bb85a892d5f1ac5a24170fc26068eec5e4cc46b11689af5058c033e494c1af", size = 4086409, upload-time = "2026-04-22T19:50:40.204Z" }, + { url = "https://files.pythonhosted.org/packages/f9/f6/41cd2a14d90ae81e26b511c04dfedb0fec17dff5bc64e174acf1c3335208/obstore-0.9.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:704d83e5d94e3c9b8c84d33a9d302e9c5110cdbeba3ff27c859e864b03e44fc1", size = 3878348, upload-time = "2026-04-22T19:50:42.382Z" }, + { url = "https://files.pythonhosted.org/packages/36/7b/88a30f98c96d138836024057be18aedc8bb669ee1bf69d9809a49ad2a806/obstore-0.9.4-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7f1b3626429fc223ed4b7355c444587d0c328a0126d81812f4f54984526fbf66", size = 4028849, upload-time = "2026-04-22T19:50:44.418Z" }, + { url = "https://files.pythonhosted.org/packages/28/c6/d0d3c08eac256f380dfdf12e07ec870b8f58ea85f6accea03c65e78ee7f7/obstore-0.9.4-pp311-pypy311_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78000bda795dccc6f48b2e743be3a92ed1e2933b974439f3dbb3549f9038668b", size = 4124212, upload-time = "2026-04-22T19:50:46.424Z" }, + { url = "https://files.pythonhosted.org/packages/e6/3d/89838eb1d7ae54edeeac41d1aa962361e24988505836c5e2753ce7fe750e/obstore-0.9.4-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:31590b64afe19d13f1b668b35900519810881636cea05366f3790b96d5881a6a", size = 4411073, upload-time = "2026-04-22T19:50:48.502Z" }, + { url = "https://files.pythonhosted.org/packages/60/bb/70db25327b13b412ee4f33731942d2828789260ace057c173d8692be3a00/obstore-0.9.4-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4f42cde6f3424ef02e9bfe50ca21d1b0ae6c313b12515a45d5c2bb3e6de9bd20", size = 4312441, upload-time = "2026-04-22T19:50:50.423Z" }, + { url = "https://files.pythonhosted.org/packages/55/84/76decc415ce07ca61b63b6ece3848fcc83785193c7fb984c4e45287fa781/obstore-0.9.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c228b545925510bef514ff2f954231a985ab1ef21c02e7c7ac448b3cc55c6377", size = 4214735, upload-time = "2026-04-22T19:50:52.12Z" }, + { url = "https://files.pythonhosted.org/packages/bb/52/0d8ca76cfc483a2ec74dde4c67ef609024b1b6f72e89cdb9ce8c4e4ec2f0/obstore-0.9.4-pp311-pypy311_pp73-manylinux_2_24_aarch64.whl", hash = "sha256:b99f5a681ad4dafe7ce21fbfff225a96522ce5a79c0f9042b57db31a530ab216", size = 4106235, upload-time = "2026-04-22T19:50:54.567Z" }, + { url = "https://files.pythonhosted.org/packages/18/58/dbe535e31b3fd7e3e227408a43a35233b2a48a38311b29619113262fd87e/obstore-0.9.4-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:8e63a7bdf69efdb49b2081daf11144e226eafa20606d3993e97ea0729c5c5cb7", size = 4293068, upload-time = "2026-04-22T19:50:56.51Z" }, + { url = "https://files.pythonhosted.org/packages/6e/d3/5f3f57f44f29a89c01a7f5c1ae7a657c960ad1fd86221bae04a27e5e3af2/obstore-0.9.4-pp311-pypy311_pp73-musllinux_1_2_armv7l.whl", hash = "sha256:2acaa282058d2f1bcffe1ec4ba7cd74746fc498fa400d5887579fdb155d16f39", size = 4265141, upload-time = "2026-04-22T19:50:58.583Z" }, + { url = "https://files.pythonhosted.org/packages/d8/5e/45572a456b99d83335fee623b92c79bfa89a8bf22284f065895ab0847dd3/obstore-0.9.4-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:1fc897945bd6d0eb0de1044c71bec8c961fa4f176453d28421c80a11e37f00ae", size = 4252425, upload-time = "2026-04-22T19:51:00.842Z" }, + { url = "https://files.pythonhosted.org/packages/54/e5/447801d3c962ba386875928ccaee83044829821c6437fc0eb526bfb5fb2d/obstore-0.9.4-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:7c39e3575796a712cbf437197404975d7d5e3f046f9bd6580a76be7f46b2ade6", size = 4435460, upload-time = "2026-04-22T19:51:02.718Z" }, +] + [[package]] name = "packaging" version = "26.0" @@ -1175,20 +1239,22 @@ wheels = [ [[package]] name = "rustac" -version = "0.9.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b1/dc/0c0618f576119fe1ac7b5b03a968a4a825411b0460f748039210e98c1dcd/rustac-0.9.3.tar.gz", hash = "sha256:427dc5325617d6c57f504318bc9f703763dc40df66d35a888d407d0ebc0f8c9f", size = 737820, upload-time = "2026-01-06T12:50:32.929Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/49/ef/d9698e162ffcbc47f172162d8f37a82f56c8afb7e3d48d10583952c6c29a/rustac-0.9.3-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:d3cc8bed80370b54450377a53ebc33980ae59ec4e5fc37aea01ff29dc9133400", size = 25960779, upload-time = "2026-01-06T12:50:19.666Z" }, - { url = "https://files.pythonhosted.org/packages/c2/62/6bebac90e854f008cf4b5788698d8c568fa4184fd77cb3b3f9f45cdac546/rustac-0.9.3-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:2f9cf5f1fe536bd88c1fe2ac26d9d22a55e5143ac76a8af6b1ee8534b99dd9b7", size = 24090602, upload-time = "2026-01-06T12:50:17.183Z" }, - { url = "https://files.pythonhosted.org/packages/0f/2f/07edacb1b1a82b08cdaf5691442076f5852e40979859b116eae8802acbc3/rustac-0.9.3-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e1f6f94eaf0fd93d5f5f0687b73b43a9ada0e176eb6b4c8d279829892ccf9109", size = 28102224, upload-time = "2026-01-06T12:50:05.228Z" }, - { url = "https://files.pythonhosted.org/packages/ec/46/555e3fefe7f5775885e0d3765f10e035d0e540cc4253a493f29fda9da14b/rustac-0.9.3-cp311-abi3-manylinux_2_28_armv7l.whl", hash = "sha256:be326379c3e2599e1e02e02ff2c56fc3b324a9afd30bed1f0abfb03b33f3e6d1", size = 26388084, upload-time = "2026-01-06T12:50:08.089Z" }, - { url = "https://files.pythonhosted.org/packages/82/1e/1c6e8ab14ef6066991232e9338ae42cb824376d22291e3d3c9741274eaca/rustac-0.9.3-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:288216ca0f96136afb8422e24728e19be535e6e34192b25763bfc374c98d72a9", size = 34572191, upload-time = "2026-01-06T12:50:10.682Z" }, - { url = "https://files.pythonhosted.org/packages/46/0f/e41b599cd4f81e62b14b06b5c45b68ea52fb644f9e99d7791df5bd55bfc7/rustac-0.9.3-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:df941bdbaedaf6bf0b51976c923642c9bab55ed5a8e25a56bc49b02a11887f28", size = 31255045, upload-time = "2026-01-06T12:50:13.232Z" }, - { url = "https://files.pythonhosted.org/packages/a3/08/479539289c0e0de5095a8fd42308aa0669c3413f7590c5907137fd1a1f48/rustac-0.9.3-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:8e8355fa226d109be2be7aed485034eb69851fb6b59a9ed47af9379356b75825", size = 34678348, upload-time = "2026-01-06T12:50:22.632Z" }, - { url = "https://files.pythonhosted.org/packages/14/19/53c5cbb4b7daa46abbb4f3db01f9fba619b73c20600eab9fc10ddcaed19b/rustac-0.9.3-cp311-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:1a02d268580e7c6595563a2c3b3c8f12ee15411071ebcbd09f61f0325b64d46e", size = 34008454, upload-time = "2026-01-06T12:50:25.328Z" }, - { url = "https://files.pythonhosted.org/packages/cc/28/6606216351812a1bd9043dd6284216b3065af0a0f3d7deb00333ad4561db/rustac-0.9.3-cp311-abi3-musllinux_1_2_i686.whl", hash = "sha256:4b162fc0051dd440deb1bdfa4a9d3ea6726d0b3355f11c630a7528780178679a", size = 39073017, upload-time = "2026-01-06T12:50:27.863Z" }, - { url = "https://files.pythonhosted.org/packages/56/7e/e3390934aa0a85fb7044fe8ca9c53868b55c0c2e996236e074b7c51ff429/rustac-0.9.3-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d3f3bb74f49acfbbce42be113dab300e98226b763974f7bbe94835bb90e7dcd6", size = 37084098, upload-time = "2026-01-06T12:50:30.696Z" }, +version = "0.9.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/00/a6/bdaedfe610cb82b94e7cbd66ec19658476963f72fa711c6babb167aee9d5/rustac-0.9.11.tar.gz", hash = "sha256:f124c74602fee38e23db15715173d1c7561b321029b393881d577e36f1556cf4", size = 746298, upload-time = "2026-05-05T20:48:19.08Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/57/6f/24c0049f5e47766260357e3c1ee3ff60fbb757b3f07b49c55dd88196f767/rustac-0.9.11-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:9a4b53fcda85a650007e886096e8f3d3ede36f4ea0123277817aef87bbff87ff", size = 27782230, upload-time = "2026-05-05T20:48:04.514Z" }, + { url = "https://files.pythonhosted.org/packages/04/fa/50c8992a649466dd2038098fc892153a0a76e11d13dd835cadf474cceda6/rustac-0.9.11-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:4a9ff5c397f9cf4671e13082e269fce10a4775a9f99c41a947ec110a4bedc2e8", size = 26041292, upload-time = "2026-05-05T20:48:01.816Z" }, + { url = "https://files.pythonhosted.org/packages/c6/51/0cfd2e0e7a59c0e9ba524d7981e8c8da7c57bf130487c715c61792cd9286/rustac-0.9.11-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f4e7e069aee94760f61ce236d288618893e3f3032c49c6965c1c12370bd17ebb", size = 27840135, upload-time = "2026-05-05T20:47:49.624Z" }, + { url = "https://files.pythonhosted.org/packages/c2/ce/f1db347f8f6f46bb5cde1a01b0cfeed70101d6e71b8245d070a886fc6f10/rustac-0.9.11-cp311-abi3-manylinux_2_28_armv7l.whl", hash = "sha256:e5ae59d77d3c627caef8ac05d1a3596ca75bbc16925532de053e2284d0c107d3", size = 26549383, upload-time = "2026-05-05T20:47:53.131Z" }, + { url = "https://files.pythonhosted.org/packages/8b/fe/58ae432dc5139847cacd068c8bdc382c0597b1c1b9cf758d2f5bf130d1e0/rustac-0.9.11-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:586e30f05b2244ff34187f1c6bcbabec3dd765e8fb2c51fa975fe646e7309305", size = 34499849, upload-time = "2026-05-05T20:47:56.087Z" }, + { url = "https://files.pythonhosted.org/packages/dc/05/17b7de22db8eb438cc114ac43c62323de612d5740648cf6a524fa2d2b9a9/rustac-0.9.11-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:29c8ca1e528430c47ea255a22e6efaf2df9fd743a71c6cc64ca82bdd86e770f9", size = 31236192, upload-time = "2026-05-05T20:47:58.925Z" }, + { url = "https://files.pythonhosted.org/packages/af/af/260d931ea816318582bf6b69b07d5fc4cf9a80620d2db87bf08f242259cb/rustac-0.9.11-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:97d6d87c20a54fe8d70265775094d9076a6b9c3b467caedd102a900c4727c365", size = 34364193, upload-time = "2026-05-05T20:48:07.52Z" }, + { url = "https://files.pythonhosted.org/packages/75/e4/4537c827c9e9446b39f9af144aed81d07fc077b867cf221b2abde9021eff/rustac-0.9.11-cp311-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:d338c6f564a9b91c476de7d643f93457fafd7c56d59ea1abf6d1732537304afa", size = 34294279, upload-time = "2026-05-05T20:48:10.884Z" }, + { url = "https://files.pythonhosted.org/packages/b7/fc/b9efadb41cdbdfc98d3f1feb773a7ef92f8e3f43c0bace6ebe5068200d5c/rustac-0.9.11-cp311-abi3-musllinux_1_2_i686.whl", hash = "sha256:e41c527bde84d8a8e85464cf9fe91f927566008ce6bbae175541f872606fd3fa", size = 39261522, upload-time = "2026-05-05T20:48:13.901Z" }, + { url = "https://files.pythonhosted.org/packages/74/e2/1bbfb9c477c488ef05f27a578208f941673b3559b01706486b5e462840c3/rustac-0.9.11-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e4a646cb5f4031506693a29ce02b82c8210c61424f24b919e26aa67f40290766", size = 37038505, upload-time = "2026-05-05T20:48:16.771Z" }, + { url = "https://files.pythonhosted.org/packages/78/fb/4c565c6236fec63694eeec9d23b5bd8198c18c4aa0b6c9b0d58259165c99/rustac-0.9.11-cp311-abi3-win32.whl", hash = "sha256:b68bc535af00ac655c22fec160ec4b6432a7410dc8c74ebfca5cb1a35b74afe6", size = 22884238, upload-time = "2026-05-05T20:48:23.548Z" }, + { url = "https://files.pythonhosted.org/packages/35/c5/2e81b9085dc6ef7e6edc508a01066b2ec0263e5cbdac78ac088ea40c098e/rustac-0.9.11-cp311-abi3-win_amd64.whl", hash = "sha256:7296ce31158ca657759ab0925004590afc0fe49697c0ea94243f5b9951eb9bac", size = 26525856, upload-time = "2026-05-05T20:48:20.833Z" }, ] [package.optional-dependencies] From 3b61625b251db235071aed717024e040181fe291 Mon Sep 17 00:00:00 2001 From: hrodmn Date: Fri, 21 Aug 2026 05:03:57 -0500 Subject: [PATCH 18/23] feat: update API to allow users to provide PartitionSpec and SortOrder --- README.md | 62 +++++++++++++++++++-- main.py | 11 ++-- src/icestac/catalog.py | 38 ++++++------- src/icestac/schema.py | 28 +++------- tests/test_catalog.py | 121 +++++++++++++++++++++++++++++++++++++++-- tests/test_load.py | 20 +++---- tests/test_schema.py | 32 ++++++++++- 7 files changed, 246 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index 634cec6..9acecc4 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ The library is split into three pieces: This is an early foundation, not a released storage specification. Current boundaries are intentional: - A table name matches its items' collection ID. Periods are unsupported because Iceberg uses them as namespace delimiters; `icestac` does not silently rewrite IDs. -- Tables are partitioned by `datetime` month. Other temporal or spatial layouts are deferred until there are concrete query patterns. +- Tables default to partitioning by `datetime` month, but callers can replace that layout with native PyIceberg partition and sort objects. Built-in spatial layouts remain deferred until there are concrete query patterns. - Table schemas do not evolve automatically. Loading a new shape fails until the Iceberg schema is updated separately. - Geometry is stored as WKB, but PyIceberg does not currently emit the GeoParquet and STAC GeoParquet file metadata required to claim compliance with those specifications. @@ -72,13 +72,67 @@ The source collection ID, `HLSS30_2.0`, contains a period and cannot be used unc The core API remains explicit: ```python +from pyiceberg.catalog import load_catalog + +from icestac.catalog import IcestacCatalog +from icestac.schema import convert_schema, get_schema_from_items + catalog = IcestacCatalog(catalog=load_catalog()) -schema = get_schema_from_items(items) -catalog.create_item_table(collection_id=collection_id, arrow_schema=schema) +arrow_schema = get_schema_from_items(items) +iceberg_schema = convert_schema(arrow_schema) +catalog.create_item_table(collection_id=collection_id, iceberg_schema=iceberg_schema) catalog.load_items(collection_id=collection_id, items=items) ``` -`get_schema_from_items(...)` accepts one STAC dictionary, a list of dictionaries, or an `arro3.core.Table`. `load_items(...)` accepts the same inputs, checks that every row belongs to the target collection, and upserts on STAC `id` by default. +`get_schema_from_items(...)` accepts one STAC dictionary, a list of dictionaries, or an `arro3.core.Table`. `convert_schema(...)` validates the item schema and returns the Iceberg schema with the field IDs used to configure table layout. `load_items(...)` accepts the same item inputs, checks that every row belongs to the target collection, and upserts on STAC `id` by default. + +Omitting layout configuration creates the monthly `datetime` partition and no sort order. To use a custom layout, build native PyIceberg objects from the prepared schema: + +```python +from pyiceberg.partitioning import PartitionField, PartitionSpec +from pyiceberg.table.sorting import SortField, SortOrder +from pyiceberg.transforms import IdentityTransform + +source_id = iceberg_schema.find_field("title").field_id +catalog.create_item_table( + collection_id=collection_id, + iceberg_schema=iceberg_schema, + partition_spec=PartitionSpec( + PartitionField( + source_id=source_id, + field_id=1000, + transform=IdentityTransform(), + name="title", + ) + ), + sort_order=SortOrder(SortField(source_id=source_id)), +) +``` + +A supplied partition specification replaces the monthly default; pass `PartitionSpec()` for an unpartitioned table. Sorting is optional. PyIceberg validates partition and sort references, so build both against the same prepared schema passed to `create_item_table(...)`. + +STAC interval items can have a null `datetime` when they include `start_datetime` and `end_datetime`. PyIceberg writes these items to the valid `datetime_month=null` partition. For a collection of interval items, override the default to partition by the start month: + +```python +from pyiceberg.partitioning import PartitionField, PartitionSpec +from pyiceberg.transforms import MonthTransform + +start_datetime_id = iceberg_schema.find_field("start_datetime").field_id +catalog.create_item_table( + collection_id=collection_id, + iceberg_schema=iceberg_schema, + partition_spec=PartitionSpec( + PartitionField( + source_id=start_datetime_id, + field_id=1000, + transform=MonthTransform(), + name="start_datetime_month", + ) + ), +) +``` + +Iceberg partition transforms use one source field, so mixed point and interval collections need a derived timestamp column to support a per-row fallback. **4. Query with DuckDB:** diff --git a/main.py b/main.py index 130b428..dad5a30 100644 --- a/main.py +++ b/main.py @@ -9,7 +9,7 @@ from rustac import DuckdbClient from icestac.catalog import IcestacCatalog -from icestac.schema import get_schema_from_items +from icestac.schema import convert_schema, get_schema_from_items logger = logging.getLogger("icestac-demo") @@ -46,12 +46,12 @@ async def run() -> None: collection_id = "HLSS30_2_0" table_exists = False - for month in ["1", "2", "3", "4"]: + for month in range(1, 9, 1): logger.info("processing 2026-%s", month) stac_geoparquet_path = HLS_STAC_GEOPARQUET_PATH_FMT.format( collection=source_collection_id, year="2026", - month=month, + month=str(month), ) try: @@ -81,12 +81,13 @@ async def run() -> None: ), ) ) - schema = get_schema_from_items(items) + arrow_schema = get_schema_from_items(items) if not table_exists: try: catalog.create_item_table( - arrow_schema=schema, collection_id=collection_id + iceberg_schema=convert_schema(arrow_schema), + collection_id=collection_id, ) except TableAlreadyExistsError: logger.warning("%s table already exists; using it", collection_id) diff --git a/src/icestac/catalog.py b/src/icestac/catalog.py index b96c94f..ac15e47 100644 --- a/src/icestac/catalog.py +++ b/src/icestac/catalog.py @@ -2,16 +2,17 @@ from dataclasses import dataclass -from arro3.core import Schema as ArrowSchema from pyiceberg.catalog import Catalog from pyiceberg.partitioning import PartitionField, PartitionSpec +from pyiceberg.schema import Schema as IcebergSchema from pyiceberg.table import Table +from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder from pyiceberg.transforms import MonthTransform from icestac.constants import DEFAULT_NAMESPACE from icestac.errors import InvalidCollectionIdError from icestac.load import Method, load_items -from icestac.schema import IcestacItem, ItemsInput, convert_schema +from icestac.schema import IcestacItem, ItemsInput def validate_collection_id(collection_id: str) -> None: @@ -34,36 +35,31 @@ def __post_init__(self) -> None: def create_item_table( self, collection_id: str, - arrow_schema: ArrowSchema, + iceberg_schema: IcebergSchema, + partition_spec: PartitionSpec | None = None, + sort_order: SortOrder = UNSORTED_SORT_ORDER, ) -> Table: - """Create a monthly partitioned Iceberg item table for a collection. - - Args: - collection_id: Collection ID, used unchanged as the table name. - arrow_schema: Arrow schema for the collection's items. - - Returns: - The created PyIceberg table. - """ + """Create an Iceberg item table for a collection.""" validate_collection_id(collection_id) - IcestacItem.validate_schema(arrow_schema) + IcestacItem.validate_schema(iceberg_schema) # TODO: check if collection record is present in collections table - iceberg_schema = convert_schema(arrow_schema) - - return self.catalog.create_table( - identifier=f"{self.namespace}.{collection_id}", - schema=iceberg_schema, - partition_spec=PartitionSpec( - # TODO: make temporal partitioning configurable + if partition_spec is None: + partition_spec = PartitionSpec( PartitionField( source_id=iceberg_schema.find_field("datetime").field_id, field_id=1000, transform=MonthTransform(), name="datetime_month", ) - ), + ) + + return self.catalog.create_table( + identifier=f"{self.namespace}.{collection_id}", + schema=iceberg_schema, + partition_spec=partition_spec, + sort_order=sort_order, ) def load_items( diff --git a/src/icestac/schema.py b/src/icestac/schema.py index 821d9f3..38be8a3 100644 --- a/src/icestac/schema.py +++ b/src/icestac/schema.py @@ -59,27 +59,16 @@ def enforce_required_fields(cls, schema: ArrowSchema) -> ArrowSchema: return ArrowSchema(fields=fields, metadata=schema.metadata) @classmethod - def validate_schema(cls, schema: ArrowSchema) -> None: - """ - Validate that an Arrow schema contains required STAC item fields. - - Checks for top-level required fields from IcestacItem. - Note: rustac flattens nested properties, so 'properties.datetime' - becomes 'datetime' in the Arrow schema. - - Args: - schema: arro3.core.Schema to validate - - Raises: - ValueError: If required STAC fields are missing from the schema - """ - schema_fields = set(schema.names) - required_fields = cls.get_required_fields() - missing_fields = required_fields - schema_fields + def validate_schema(cls, schema: ArrowSchema | IcebergSchema) -> None: + """Validate that a schema contains the required STAC item fields.""" + schema_fields = set( + schema.column_names if isinstance(schema, IcebergSchema) else schema.names + ) + missing_fields = cls.get_required_fields() - schema_fields if missing_fields: raise ValueError( - f"Arrow schema is missing required STAC fields: {sorted(missing_fields)}" + f"Schema is missing required STAC fields: {sorted(missing_fields)}" ) @@ -113,7 +102,8 @@ def get_schema_from_items(items: ItemsInput) -> ArrowSchema: def convert_schema(schema: ArrowSchema) -> IcebergSchema: - """Convert an Arrow schema to an Iceberg schema with field IDs.""" + """Validate and convert an Arrow item schema to Iceberg with field IDs.""" + IcestacItem.validate_schema(schema) _schema = _pyarrow_to_schema_without_ids( pa.schema(IcestacItem.enforce_required_fields(schema)) ) diff --git a/tests/test_catalog.py b/tests/test_catalog.py index 9888137..61922bc 100644 --- a/tests/test_catalog.py +++ b/tests/test_catalog.py @@ -4,11 +4,20 @@ import pytest from arro3.core import Table as ArrowTable from pyiceberg.exceptions import TableAlreadyExistsError +from pyiceberg.partitioning import PartitionField, PartitionSpec +from pyiceberg.schema import Schema as IcebergSchema +from pyiceberg.table.sorting import SortField, SortOrder +from pyiceberg.transforms import IdentityTransform, MonthTransform from rustac import to_arrow from icestac.catalog import IcestacCatalog from icestac.errors import InvalidCollectionIdError -from icestac.schema import IcestacItem, ItemsInput, get_schema_from_items +from icestac.schema import ( + IcestacItem, + ItemsInput, + convert_schema, + get_schema_from_items, +) from tests.helpers import items_to_list @@ -20,11 +29,14 @@ def test_create_item_table( expected_items = items_to_list(items) arrow_schema = get_schema_from_items(items) table = test_catalog.create_item_table( - arrow_schema=arrow_schema, + iceberg_schema=convert_schema(arrow_schema), collection_id=test_collection_id, ) assert table.schema().find_field("datetime") + assert len(table.spec().fields) == 1 + assert isinstance(table.spec().fields[0].transform, MonthTransform) + assert not table.sort_order().fields # Ensure data has required fields marked as non-nullable to match table schema arrow_data = ( @@ -45,7 +57,7 @@ def test_create_item_table( with pytest.raises(TableAlreadyExistsError): test_catalog.create_item_table( - arrow_schema=arrow_schema, + iceberg_schema=convert_schema(arrow_schema), collection_id=test_collection_id, ) @@ -57,11 +69,108 @@ def test_create_item_table_bad_collection_id( arrow_schema = get_schema_from_items(items) with pytest.raises(InvalidCollectionIdError): test_catalog.create_item_table( - arrow_schema=arrow_schema, + iceberg_schema=convert_schema(arrow_schema), collection_id="bad.collection", ) +def test_create_item_table_custom_layout( + test_catalog: IcestacCatalog, + test_collection_id: str, + sample_stac_item: dict[str, Any], +) -> None: + iceberg_schema = convert_schema(get_schema_from_items(sample_stac_item)) + title_id = iceberg_schema.find_field("title").field_id + + table = test_catalog.create_item_table( + collection_id=test_collection_id, + iceberg_schema=iceberg_schema, + partition_spec=PartitionSpec( + PartitionField( + source_id=title_id, + field_id=1000, + transform=IdentityTransform(), + name="title", + ) + ), + sort_order=SortOrder(SortField(source_id=title_id)), + ) + + assert [field.name for field in table.spec().fields] == ["title"] + assert isinstance(table.spec().fields[0].transform, IdentityTransform) + assert ( + table.spec().fields[0].source_id == table.schema().find_field("title").field_id + ) + assert ( + table.sort_order().fields[0].source_id + == table.schema().find_field("title").field_id + ) + + +def test_create_item_table_unpartitioned( + test_catalog: IcestacCatalog, + test_collection_id: str, + sample_stac_item: dict[str, Any], +) -> None: + table = test_catalog.create_item_table( + collection_id=test_collection_id, + iceberg_schema=convert_schema(get_schema_from_items(sample_stac_item)), + partition_spec=PartitionSpec(), + ) + + assert not table.spec().fields + + +def test_create_item_table_rejects_invalid_schema( + test_catalog: IcestacCatalog, + test_collection_id: str, + sample_stac_item: dict[str, Any], +) -> None: + iceberg_schema = convert_schema(get_schema_from_items(sample_stac_item)) + missing_id = IcebergSchema( + *(field for field in iceberg_schema.fields if field.name != "id") + ) + + with pytest.raises(ValueError, match="missing required STAC fields.*'id'"): + test_catalog.create_item_table( + collection_id=test_collection_id, + iceberg_schema=missing_id, + ) + + +def test_create_item_table_rejects_unknown_partition_source( + test_catalog: IcestacCatalog, + test_collection_id: str, + sample_stac_item: dict[str, Any], +) -> None: + with pytest.raises(ValueError): + test_catalog.create_item_table( + collection_id=test_collection_id, + iceberg_schema=convert_schema(get_schema_from_items(sample_stac_item)), + partition_spec=PartitionSpec( + PartitionField( + source_id=9999, + field_id=1000, + transform=IdentityTransform(), + name="missing", + ) + ), + ) + + +def test_create_item_table_rejects_unknown_sort_source( + test_catalog: IcestacCatalog, + test_collection_id: str, + sample_stac_item: dict[str, Any], +) -> None: + with pytest.raises(ValueError): + test_catalog.create_item_table( + collection_id=test_collection_id, + iceberg_schema=convert_schema(get_schema_from_items(sample_stac_item)), + sort_order=SortOrder(SortField(source_id=9999)), + ) + + def test_load_items_rejects_a_different_collection( test_catalog: IcestacCatalog, test_collection_id: str, @@ -69,7 +178,7 @@ def test_load_items_rejects_a_different_collection( ) -> None: arrow_schema = get_schema_from_items(sample_stac_item) table = test_catalog.create_item_table( - arrow_schema=arrow_schema, + iceberg_schema=convert_schema(arrow_schema), collection_id=test_collection_id, ) sample_stac_item["collection"] = "different-collection" @@ -88,7 +197,7 @@ def test_load_items( expected_items = items_to_list(items) arrow_schema = get_schema_from_items(items) table = test_catalog.create_item_table( - arrow_schema=arrow_schema, + iceberg_schema=convert_schema(arrow_schema), collection_id=test_collection_id, ) diff --git a/tests/test_load.py b/tests/test_load.py index cb73cf0..d12570e 100644 --- a/tests/test_load.py +++ b/tests/test_load.py @@ -5,7 +5,7 @@ from icestac.catalog import IcestacCatalog from icestac.load import load_items -from icestac.schema import ItemsInput, get_schema_from_items +from icestac.schema import ItemsInput, convert_schema, get_schema_from_items from tests.helpers import items_to_list @@ -20,7 +20,7 @@ def test_load_items_upsert_default( # Create the table arrow_schema = get_schema_from_items(items) table = test_catalog.create_item_table( - arrow_schema=arrow_schema, + iceberg_schema=convert_schema(arrow_schema), collection_id=test_collection_id, ) @@ -45,7 +45,7 @@ def test_load_items_upsert_explicit( # Create the table arrow_schema = get_schema_from_items(items) table = test_catalog.create_item_table( - arrow_schema=arrow_schema, + iceberg_schema=convert_schema(arrow_schema), collection_id=expected_items[0]["collection"], ) @@ -67,7 +67,7 @@ def test_load_items_upsert_updates_existing( # Create the table arrow_schema = get_schema_from_items(items) table = test_catalog.create_item_table( - arrow_schema=arrow_schema, + iceberg_schema=convert_schema(arrow_schema), collection_id=expected_items[0]["collection"], ) @@ -104,7 +104,7 @@ def test_load_items_append( # Create the table arrow_schema = get_schema_from_items(items) table = test_catalog.create_item_table( - arrow_schema=arrow_schema, + iceberg_schema=convert_schema(arrow_schema), collection_id=expected_items[0]["collection"], ) @@ -126,7 +126,7 @@ def test_load_items_append_creates_duplicates( # Create the table arrow_schema = get_schema_from_items(items) table = test_catalog.create_item_table( - arrow_schema=arrow_schema, + iceberg_schema=convert_schema(arrow_schema), collection_id=expected_items[0]["collection"], ) @@ -149,7 +149,7 @@ def test_load_items_multiple_batches( # Create the table arrow_schema = get_schema_from_items(items) table = test_catalog.create_item_table( - arrow_schema=arrow_schema, + iceberg_schema=convert_schema(arrow_schema), collection_id=expected_items[0]["collection"], ) @@ -173,7 +173,7 @@ def test_load_items_rejects_invalid_method( ) -> None: arrow_schema = get_schema_from_items(sample_stac_item) table = test_catalog.create_item_table( - arrow_schema=arrow_schema, + iceberg_schema=convert_schema(arrow_schema), collection_id=sample_stac_item["collection"], ) @@ -193,7 +193,7 @@ def test_load_items_supports_interval_datetime( } arrow_schema = get_schema_from_items(interval_item) table = test_catalog.create_item_table( - arrow_schema=arrow_schema, + iceberg_schema=convert_schema(arrow_schema), collection_id=interval_item["collection"], ) @@ -209,7 +209,7 @@ def test_load_items_different_schema( # Create the table arrow_schema = get_schema_from_items(sample_stac_item) table = test_catalog.create_item_table( - arrow_schema=arrow_schema, + iceberg_schema=convert_schema(arrow_schema), collection_id=sample_stac_item["collection"], ) diff --git a/tests/test_schema.py b/tests/test_schema.py index 2657b58..fa4ff3b 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -5,7 +5,7 @@ from arro3.core import Schema from pydantic import ValidationError -from icestac.schema import IcestacItem, get_schema_from_items +from icestac.schema import IcestacItem, convert_schema, get_schema_from_items def test_get_schema_from_items(sample_stac_item: dict[str, Any]) -> None: @@ -43,6 +43,36 @@ def test_validate_schema_valid(sample_stac_item: dict[str, Any]) -> None: IcestacItem.validate_schema(schema) +def test_convert_schema_prepares_item_schema( + sample_stac_item: dict[str, Any], +) -> None: + """Test that conversion preserves item fields and assigns source IDs.""" + iceberg_schema = convert_schema(get_schema_from_items(sample_stac_item)) + + assert iceberg_schema.find_field("title").field_id > 0 + assert iceberg_schema.find_field("id").required + assert str(iceberg_schema.find_field("geometry").field_type) == "binary" + + +def test_convert_schema_validates_required_fields() -> None: + """Test that conversion rejects an Arrow schema missing STAC fields.""" + with pytest.raises(ValueError, match="missing required STAC fields.*'id'"): + convert_schema(pa.schema([("type", pa.string())])) + + +def test_validate_prepared_schema_missing_required_field( + sample_stac_item: dict[str, Any], +) -> None: + """Test that required-field validation accepts prepared Iceberg schemas.""" + iceberg_schema = convert_schema(get_schema_from_items(sample_stac_item)) + missing_id = type(iceberg_schema)( + *(field for field in iceberg_schema.fields if field.name != "id") + ) + + with pytest.raises(ValueError, match="missing required STAC fields.*'id'"): + IcestacItem.validate_schema(missing_id) + + def test_validate_schema_missing_required_field() -> None: """Test that a schema missing required fields raises ValueError.""" # Create a schema missing the required 'id' field From fdba514356d686aa6c0c7962f1fc27422508d50f Mon Sep 17 00:00:00 2001 From: hrodmn Date: Fri, 21 Aug 2026 05:52:25 -0500 Subject: [PATCH 19/23] feat: add evolve_schema boolean to load_items --- main.py | 41 ++++++++++++++++++++++++--------- src/icestac/catalog.py | 9 ++++++-- src/icestac/load.py | 22 +++++++++++------- tests/test_catalog.py | 24 ++++++++++++++++++++ tests/test_load.py | 51 +++++++++++++++++++++++++++++++++++++----- 5 files changed, 120 insertions(+), 27 deletions(-) diff --git a/main.py b/main.py index dad5a30..e6b1645 100644 --- a/main.py +++ b/main.py @@ -35,7 +35,11 @@ async def copy_hls_stac_geoparquet(path: str, store: LocalStore) -> None: async def run() -> None: """Load several months of HLS STAC GeoParquet into local Iceberg.""" - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s:%(name)s:%(message)s", + datefmt="%Y-%m-%dT%H:%M:%S%z", + ) catalog = IcestacCatalog(catalog=load_catalog()) duckdb_client = DuckdbClient() @@ -47,7 +51,7 @@ async def run() -> None: table_exists = False for month in range(1, 9, 1): - logger.info("processing 2026-%s", month) + month_logger = logger.getChild(f"2026-{month}") stac_geoparquet_path = HLS_STAC_GEOPARQUET_PATH_FMT.format( collection=source_collection_id, year="2026", @@ -57,13 +61,13 @@ async def run() -> None: try: _ = local_store.head(stac_geoparquet_path) except FileNotFoundError: - logger.info("downloading %s", stac_geoparquet_path) + month_logger.info("downloading %s", stac_geoparquet_path) await copy_hls_stac_geoparquet( path=stac_geoparquet_path, store=local_store, ) - logger.info("loading items as arrow table") + month_logger.info("loading items as arrow table") items = duckdb_client.search_to_arrow(href=f"data/{stac_geoparquet_path}") if not items: @@ -90,15 +94,30 @@ async def run() -> None: collection_id=collection_id, ) except TableAlreadyExistsError: - logger.warning("%s table already exists; using it", collection_id) + month_logger.warning("%s table already exists; using it", collection_id) table_exists = True - logger.info("loading items into icestac catalog") - catalog.load_items( - collection_id=collection_id, - items=items, - method="upsert", - ) + month_logger.info("loading items into icestac catalog") + try: + catalog.load_items( + collection_id=collection_id, + items=items, + method="upsert", + evolve_schema=False, + ) + except ValueError as e: + if "Update the schema first (hint, use union_by_name)" not in str(e): + raise + + month_logger.warning(str(e)) + month_logger.info("retrying load with evolve_schema=True") + + catalog.load_items( + collection_id=collection_id, + items=items, + method="upsert", + evolve_schema=True, + ) if __name__ == "__main__": diff --git a/src/icestac/catalog.py b/src/icestac/catalog.py index ac15e47..8db6eb9 100644 --- a/src/icestac/catalog.py +++ b/src/icestac/catalog.py @@ -63,13 +63,18 @@ def create_item_table( ) def load_items( - self, collection_id: str, items: ItemsInput, method: Method = "upsert" + self, + collection_id: str, + items: ItemsInput, + method: Method = "upsert", + evolve_schema: bool = False, ) -> None: - """Load items into the table matching their collection ID.""" + """Load items, optionally evolving their collection table schema.""" load_items( items, table=self.catalog.load_table( identifier=f"{self.namespace}.{collection_id}" ), method=method, + evolve_schema=evolve_schema, ) diff --git a/src/icestac/load.py b/src/icestac/load.py index a389be1..a516e92 100644 --- a/src/icestac/load.py +++ b/src/icestac/load.py @@ -14,8 +14,9 @@ def load_items( items: ItemsInput, table: Table, method: Method = "upsert", + evolve_schema: bool = False, ) -> None: - """Load STAC items into their collection's Iceberg table.""" + """Load STAC items, optionally evolving the Iceberg schema by name.""" if method not in ("append", "upsert"): raise ValueError(f"Unsupported load method: {method}") @@ -36,10 +37,15 @@ def load_items( f"{sorted(map(str, collection_ids))}" ) - if method == "upsert": - table.upsert( - df=arrow_table, - join_cols=["id"], - ) - else: - table.append(df=arrow_table) + with table.transaction() as transaction: + if evolve_schema: + with transaction.update_schema() as update: + update.union_by_name(arrow_table.schema) + + if method == "upsert": + transaction.upsert( + df=arrow_table, + join_cols=["id"], + ) + else: + transaction.append(df=arrow_table) diff --git a/tests/test_catalog.py b/tests/test_catalog.py index 61922bc..9632caa 100644 --- a/tests/test_catalog.py +++ b/tests/test_catalog.py @@ -213,3 +213,27 @@ def test_load_items( result = table.scan().to_arrow() assert len(result) == len(expected_items) assert result.column("id").to_pylist() == [item["id"] for item in expected_items] + + +def test_catalog_load_items_evolves_schema( + test_catalog: IcestacCatalog, + test_collection_id: str, + sample_stac_item: dict[str, Any], +) -> None: + table = test_catalog.create_item_table( + iceberg_schema=convert_schema(get_schema_from_items(sample_stac_item)), + collection_id=test_collection_id, + ) + evolved_item = { + **sample_stac_item, + "properties": {**sample_stac_item["properties"], "new_field": True}, + } + + test_catalog.load_items( + collection_id=test_collection_id, + items=evolved_item, + evolve_schema=True, + ) + + table.refresh() + assert table.schema().find_field("new_field") diff --git a/tests/test_load.py b/tests/test_load.py index d12570e..c55dbaf 100644 --- a/tests/test_load.py +++ b/tests/test_load.py @@ -4,7 +4,7 @@ import pytest from icestac.catalog import IcestacCatalog -from icestac.load import load_items +from icestac.load import Method, load_items from icestac.schema import ItemsInput, convert_schema, get_schema_from_items from tests.helpers import items_to_list @@ -206,18 +206,57 @@ def test_load_items_different_schema( test_catalog: IcestacCatalog, sample_stac_item: dict[str, Any], ) -> None: - # Create the table arrow_schema = get_schema_from_items(sample_stac_item) table = test_catalog.create_item_table( iceberg_schema=convert_schema(arrow_schema), collection_id=sample_stac_item["collection"], ) - - # load an item load_items([sample_stac_item], table) - # change the schema - item_new_schema = sample_stac_item.copy() + item_new_schema = deepcopy(sample_stac_item) item_new_schema["properties"]["new_field"] = True with pytest.raises(ValueError, match="Update the schema first"): load_items([item_new_schema], table) + + +@pytest.mark.parametrize("method", ["append", "upsert"]) +def test_load_items_evolves_schema( + test_catalog: IcestacCatalog, + sample_stac_item: dict[str, Any], + method: Method, +) -> None: + table = test_catalog.create_item_table( + iceberg_schema=convert_schema(get_schema_from_items(sample_stac_item)), + collection_id=sample_stac_item["collection"], + ) + load_items(sample_stac_item, table) + + evolved_item = deepcopy(sample_stac_item) + evolved_item["id"] = "evolved-item" + evolved_item["properties"]["processing:software"] = { + "Atmospheric Correction": "6.0" + } + load_items(evolved_item, table, method=method, evolve_schema=True) + + table.refresh() + assert table.schema().find_field("processing:software.Atmospheric Correction") + assert len(table.scan().to_arrow()) == 2 + + +def test_load_items_does_not_evolve_schema_when_write_fails( + test_catalog: IcestacCatalog, + sample_stac_item: dict[str, Any], +) -> None: + table = test_catalog.create_item_table( + iceberg_schema=convert_schema(get_schema_from_items(sample_stac_item)), + collection_id=sample_stac_item["collection"], + ) + evolved_item = deepcopy(sample_stac_item) + evolved_item["properties"]["new_field"] = True + + with pytest.raises(ValueError, match="Duplicate rows"): + load_items([evolved_item, evolved_item], table, evolve_schema=True) + + table.refresh() + with pytest.raises(ValueError, match="Could not find field"): + table.schema().find_field("new_field") From bf1980adf31397d4b9e9f65a37605880b2e78b02 Mon Sep 17 00:00:00 2001 From: hrodmn Date: Fri, 21 Aug 2026 05:52:48 -0500 Subject: [PATCH 20/23] docs: update README --- README.md | 183 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 108 insertions(+), 75 deletions(-) diff --git a/README.md b/README.md index 9acecc4..f552a0c 100644 --- a/README.md +++ b/README.md @@ -2,116 +2,91 @@ ## Overview -`icestac` uses rustac's flattened Arrow representation of STAC Items and writes it to Apache Iceberg. +`icestac` helps you store STAC Items in Apache Iceberg using rustac's flattened Arrow representation. -The library is split into three pieces: +The library has three main pieces: - `src/icestac/schema.py` validates STAC inputs, preserves Arrow metadata, and converts Arrow schemas to Iceberg schemas with field IDs. - `src/icestac/catalog.py` wraps a PyIceberg catalog and creates one item table per collection in the `icestac` namespace. - `src/icestac/load.py` accepts STAC dictionaries or `arro3.core.Table` data and writes it with `append` or `upsert` semantics. -This is an early foundation, not a released storage specification. Current boundaries are intentional: +The project is still an early foundation rather than a released storage specification. A few boundaries are worth knowing before you begin: -- A table name matches its items' collection ID. Periods are unsupported because Iceberg uses them as namespace delimiters; `icestac` does not silently rewrite IDs. -- Tables default to partitioning by `datetime` month, but callers can replace that layout with native PyIceberg partition and sort objects. Built-in spatial layouts remain deferred until there are concrete query patterns. -- Table schemas do not evolve automatically. Loading a new shape fails until the Iceberg schema is updated separately. -- Geometry is stored as WKB, but PyIceberg does not currently emit the GeoParquet and STAC GeoParquet file metadata required to claim compliance with those specifications. +- `icestac` uses each collection ID as its table name. Iceberg treats periods as namespace delimiters, so collection IDs that contain periods are unsupported. Rename those IDs before loading them; `icestac` will not rewrite them for you. +- Tables use monthly `datetime` partitions by default. You can replace that layout with native PyIceberg partition and sort objects. We plan to add built-in spatial layouts as support for geospatial types across the Iceberg ecosystem improves. +- Table schemas remain strict by default. Loading a new shape fails unless you opt into compatible schema evolution. +- Geometry is stored as WKB. PyIceberg does not yet emit the GeoParquet and STAC GeoParquet file metadata needed for compliance with those specifications. -## Development - -### Tests - -```bash -# Run all tests -uv run pytest -``` - -### Local Instance - -**1. Start the local environment:** +## Core API -```bash -docker compose up -``` +The API adds a few STAC-focused conveniences while keeping native PyIceberg objects available for table layout and schema management. -This starts three services: -- **Iceberg REST Catalog** at `http://localhost:8181` -- **MinIO** (S3-compatible storage) at `http://localhost:9000` (API) and `http://localhost:9001` (Console) -- **MinIO Client** — initializes the `warehouse` bucket on startup +To get started, create a table and load some items. `IcestacCatalog.create_item_table(...)` uses a monthly partition based on the `datetime` property unless you provide another layout. -**2. Configure catalog access:** +```python +from pyiceberg.catalog import load_catalog -A `.pyiceberg.yaml` is included in the repo with default credentials for the local Docker environment: +from icestac.catalog import IcestacCatalog +from icestac.schema import convert_schema, get_schema_from_items -```yaml -catalog: - default: - type: rest - uri: http://localhost:8181 - warehouse: s3://warehouse/ - s3.endpoint: http://localhost:9000 - s3.access-key-id: admin - s3.secret-access-key: password - s3.path-style-access: "true" +catalog = IcestacCatalog(catalog=load_catalog()) +arrow_schema = get_schema_from_items(items) +iceberg_schema = convert_schema(arrow_schema) +catalog.create_item_table(collection_id=collection_id, iceberg_schema=iceberg_schema) +catalog.load_items(collection_id=collection_id, items=items) ``` -PyIceberg will pick this up automatically when running from the project directory. See the [PyIceberg configuration docs](https://py.iceberg.apache.org/configuration/) for other configuration options. +`get_schema_from_items(...)` accepts one STAC item dictionary, a list of dictionaries, or an `arro3.core.Table`. `convert_schema(...)` validates the item schema and returns an Iceberg schema with the field IDs needed for table layout. `load_items(...)` accepts the same inputs, checks that every row belongs to the target collection, and upserts on STAC `id` by default. -**3. Load sample items:** +Incoming fields that are absent from the table schema raise an error. If you want PyIceberg to add compatible fields by name, pass `evolve_schema=True`. The schema update and write then commit in one transaction: -`main.py` is the best example of the current ingestion workflow: - -```bash -uv run python main.py +```python +catalog.load_items( + collection_id=collection_id, + items=items, + evolve_schema=True, +) ``` -The script downloads four months of HLS STAC GeoParquet from public S3, reads each file directly as Arrow, and loads it through `IcestacCatalog`. - -The source collection ID, `HLSS30_2.0`, contains a period and cannot be used unchanged as an Iceberg identifier. The demo explicitly migrates the dataset to `HLSS30_2_0` by replacing every item's collection value before creating the matching table. This is a dataset decision, not automatic library slugification. - -The core API remains explicit: +If you manage schema changes separately, update the table with PyIceberg before loading: ```python -from pyiceberg.catalog import load_catalog +import pyarrow -from icestac.catalog import IcestacCatalog -from icestac.schema import convert_schema, get_schema_from_items - -catalog = IcestacCatalog(catalog=load_catalog()) arrow_schema = get_schema_from_items(items) -iceberg_schema = convert_schema(arrow_schema) -catalog.create_item_table(collection_id=collection_id, iceberg_schema=iceberg_schema) +table = catalog.catalog.load_table((catalog.namespace, collection_id)) +with table.update_schema() as update: + update.union_by_name(pyarrow.schema(arrow_schema)) + catalog.load_items(collection_id=collection_id, items=items) ``` -`get_schema_from_items(...)` accepts one STAC dictionary, a list of dictionaries, or an `arro3.core.Table`. `convert_schema(...)` validates the item schema and returns the Iceberg schema with the field IDs used to configure table layout. `load_items(...)` accepts the same item inputs, checks that every row belongs to the target collection, and upserts on STAC `id` by default. - -Omitting layout configuration creates the monthly `datetime` partition and no sort order. To use a custom layout, build native PyIceberg objects from the prepared schema: +Without layout configuration, a table gets the monthly `datetime` partition and no sort order. For a custom layout, build native PyIceberg objects from the prepared schema. This example assumes your STAC items have a `sortme` property: ```python from pyiceberg.partitioning import PartitionField, PartitionSpec from pyiceberg.table.sorting import SortField, SortOrder from pyiceberg.transforms import IdentityTransform -source_id = iceberg_schema.find_field("title").field_id +sort_id = iceberg_schema.find_field("sortme").field_id catalog.create_item_table( collection_id=collection_id, iceberg_schema=iceberg_schema, partition_spec=PartitionSpec( PartitionField( - source_id=source_id, + source_id=sort_id, field_id=1000, transform=IdentityTransform(), - name="title", + name="sortme", ) ), - sort_order=SortOrder(SortField(source_id=source_id)), + sort_order=SortOrder(SortField(source_id=sort_id)), ) ``` -A supplied partition specification replaces the monthly default; pass `PartitionSpec()` for an unpartitioned table. Sorting is optional. PyIceberg validates partition and sort references, so build both against the same prepared schema passed to `create_item_table(...)`. +A partition specification replaces the monthly default. Pass `PartitionSpec()` for an unpartitioned table, and omit the sort order if you do not need one. PyIceberg validates partition and sort references, so build both from the same prepared schema that you pass to `create_item_table(...)`. -STAC interval items can have a null `datetime` when they include `start_datetime` and `end_datetime`. PyIceberg writes these items to the valid `datetime_month=null` partition. For a collection of interval items, override the default to partition by the start month: +STAC interval items may have a null `datetime` when they include `start_datetime` and `end_datetime`. PyIceberg writes these items to a valid `datetime_month=null` partition. For a collection of interval items, you can partition by the start month instead: ```python from pyiceberg.partitioning import PartitionField, PartitionSpec @@ -132,13 +107,71 @@ catalog.create_item_table( ) ``` -Iceberg partition transforms use one source field, so mixed point and interval collections need a derived timestamp column to support a per-row fallback. +Iceberg partition transforms use one source field. If your collection mixes point and interval items, add a derived timestamp column to support a per-row fallback. + +## Development + +### Tests + +Run the test suite with: + +```bash +uv run pytest +``` + +### Local instance + +You can run a complete catalog and object storage environment on your machine. + +**1. Start the local environment** + +```bash +docker compose up +``` + +This starts three services: + +- **Iceberg REST Catalog** at `http://localhost:8181` +- **MinIO** (S3-compatible storage) at `http://localhost:9000` for the API and `http://localhost:9001` for the console +- **MinIO Client**, which creates the `warehouse` bucket on startup + +**2. Configure catalog access** + +The repository includes `.pyiceberg.yaml` with default credentials for the local Docker environment: + +```yaml +catalog: + default: + type: rest + uri: http://localhost:8181 + warehouse: s3://warehouse/ + s3.endpoint: http://localhost:9000 + s3.access-key-id: admin + s3.secret-access-key: password + s3.path-style-access: "true" +``` + +PyIceberg reads this file when you run commands from the project directory. If you need a different setup, see the [PyIceberg configuration docs](https://py.iceberg.apache.org/configuration/). + +**3. Load sample items** + +Once the services are running, use `main.py` to try the current ingestion workflow: + +```bash +uv run python main.py +``` + +The script downloads eight months of HLS STAC GeoParquet from public S3, reads each file directly as Arrow, and loads it through `IcestacCatalog`. + +The source collection ID, `HLSS30_2.0`, contains a period, which Iceberg interprets as a namespace delimiter. For this demo, the script renames the collection to `HLSS30_2_0` in every item before creating the table. The script makes this dataset decision explicitly because `icestac` does not rewrite collection IDs. + +The June 2026 items introduce a compatible schema change. `main.py` passes `evolve_schema=True` when it loads that batch so you can see schema evolution in use. -**4. Query with DuckDB:** +**4. Query with DuckDB** -After ingesting items, query the Iceberg tables using DuckDB's `iceberg` extension. Tables live under the `icestac` namespace. +After the load finishes, you can query the Iceberg tables with DuckDB's `iceberg` extension. The tables live under the `icestac` namespace. -First, configure the extensions and MinIO credentials: +Start by configuring the extensions and MinIO credentials: ```sql INSTALL iceberg; LOAD iceberg; @@ -155,7 +188,7 @@ CREATE OR REPLACE SECRET minio ( ); ``` -Query via the REST catalog: +Then query through the REST catalog: ```sql ATTACH 'icestac' AS catalog ( @@ -174,7 +207,7 @@ FROM catalog.icestac.HLSS30_2_0; DESCRIBE SELECT bbox, geometry FROM catalog.icestac.HLSS30_2_0; ``` -Or scan the table directly from its S3 path (no catalog required): +You can also scan the table directly from its S3 path without a catalog: ```sql SET unsafe_enable_version_guessing = true; @@ -183,9 +216,9 @@ FROM iceberg_scan('s3://warehouse/icestac/HLSS30_2_0') LIMIT 10; ``` -## Delete a Table +## Delete a table -To remove a table from the local Iceberg REST catalog, call `drop_table(...)` on the underlying PyIceberg catalog: +To start over with a table, call `drop_table(...)` on the underlying PyIceberg catalog: ```python from pyiceberg.catalog import load_catalog @@ -198,9 +231,9 @@ catalog.catalog.drop_table( ) ``` -`purge_requested=True` asks the REST catalog to delete the underlying table data as well as the catalog entry. +With `purge_requested=True`, the REST catalog deletes the table data along with the catalog entry. -If the metadata entry is removed but table files remain in MinIO, delete the warehouse path manually: +If the catalog entry is gone but files remain in MinIO, remove the warehouse path: ```bash docker compose exec mc mc rm --recursive --force minio/warehouse/icestac/HLSS30_2_0 From 54428c77d427de2ef490b2612ba6d696b5d1d988 Mon Sep 17 00:00:00 2001 From: hrodmn Date: Fri, 21 Aug 2026 06:19:15 -0500 Subject: [PATCH 21/23] fix: just have get_schema_from_items return an Iceberg schema --- README.md | 24 +++++++++++------------- main.py | 6 +++--- src/icestac/schema.py | 18 ++++++------------ tests/test_catalog.py | 31 +++++++++++++++---------------- tests/test_load.py | 42 +++++++++++++++++++++--------------------- tests/test_schema.py | 32 +++++++------------------------- 6 files changed, 63 insertions(+), 90 deletions(-) diff --git a/README.md b/README.md index f552a0c..9b03cc8 100644 --- a/README.md +++ b/README.md @@ -27,16 +27,15 @@ To get started, create a table and load some items. `IcestacCatalog.create_item_ from pyiceberg.catalog import load_catalog from icestac.catalog import IcestacCatalog -from icestac.schema import convert_schema, get_schema_from_items +from icestac.schema import get_schema_from_items catalog = IcestacCatalog(catalog=load_catalog()) -arrow_schema = get_schema_from_items(items) -iceberg_schema = convert_schema(arrow_schema) +iceberg_schema = get_schema_from_items(items) catalog.create_item_table(collection_id=collection_id, iceberg_schema=iceberg_schema) catalog.load_items(collection_id=collection_id, items=items) ``` -`get_schema_from_items(...)` accepts one STAC item dictionary, a list of dictionaries, or an `arro3.core.Table`. `convert_schema(...)` validates the item schema and returns an Iceberg schema with the field IDs needed for table layout. `load_items(...)` accepts the same inputs, checks that every row belongs to the target collection, and upserts on STAC `id` by default. +`get_schema_from_items(...)` accepts one STAC item dictionary, a list of dictionaries, or an `arro3.core.Table`, validates it, and returns an Iceberg schema with the field IDs needed for table layout. `load_items(...)` accepts the same inputs, checks that every row belongs to the target collection, and upserts on STAC `id` by default. Incoming fields that are absent from the table schema raise an error. If you want PyIceberg to add compatible fields by name, pass `evolve_schema=True`. The schema update and write then commit in one transaction: @@ -51,33 +50,32 @@ catalog.load_items( If you manage schema changes separately, update the table with PyIceberg before loading: ```python -import pyarrow - -arrow_schema = get_schema_from_items(items) +iceberg_schema = get_schema_from_items(items) table = catalog.catalog.load_table((catalog.namespace, collection_id)) with table.update_schema() as update: - update.union_by_name(pyarrow.schema(arrow_schema)) + update.union_by_name(iceberg_schema) catalog.load_items(collection_id=collection_id, items=items) ``` -Without layout configuration, a table gets the monthly `datetime` partition and no sort order. For a custom layout, build native PyIceberg objects from the prepared schema. This example assumes your STAC items have a `sortme` property: +Without layout configuration, a table gets the monthly `datetime` partition and no sort order. For a custom layout, build native PyIceberg objects from the prepared schema. This example partitions items by year and sorts them by a `sortme` property: ```python from pyiceberg.partitioning import PartitionField, PartitionSpec from pyiceberg.table.sorting import SortField, SortOrder -from pyiceberg.transforms import IdentityTransform +from pyiceberg.transforms import YearTransform +datetime_id = iceberg_schema.find_field("datetime").field_id sort_id = iceberg_schema.find_field("sortme").field_id catalog.create_item_table( collection_id=collection_id, iceberg_schema=iceberg_schema, partition_spec=PartitionSpec( PartitionField( - source_id=sort_id, + source_id=datetime_id, field_id=1000, - transform=IdentityTransform(), - name="sortme", + transform=YearTransform(), + name="datetime_year", ) ), sort_order=SortOrder(SortField(source_id=sort_id)), diff --git a/main.py b/main.py index e6b1645..4cc5d48 100644 --- a/main.py +++ b/main.py @@ -9,7 +9,7 @@ from rustac import DuckdbClient from icestac.catalog import IcestacCatalog -from icestac.schema import convert_schema, get_schema_from_items +from icestac.schema import get_schema_from_items logger = logging.getLogger("icestac-demo") @@ -85,12 +85,12 @@ async def run() -> None: ), ) ) - arrow_schema = get_schema_from_items(items) + iceberg_schema = get_schema_from_items(items) if not table_exists: try: catalog.create_item_table( - iceberg_schema=convert_schema(arrow_schema), + iceberg_schema=iceberg_schema, collection_id=collection_id, ) except TableAlreadyExistsError: diff --git a/src/icestac/schema.py b/src/icestac/schema.py index 38be8a3..cf068f6 100644 --- a/src/icestac/schema.py +++ b/src/icestac/schema.py @@ -83,8 +83,8 @@ def _first_item_from_arrow(items: ArrowTable) -> dict[str, Any]: return feature_collection["features"][0] -def get_schema_from_items(items: ItemsInput) -> ArrowSchema: - """Derive an enforced Arrow schema from STAC dictionaries or Arrow data.""" +def get_schema_from_items(items: ItemsInput) -> IcebergSchema: + """Derive an Iceberg schema from STAC dictionaries or Arrow data.""" if isinstance(items, dict): item = cast(dict[str, Any], items) items = [item] @@ -98,19 +98,13 @@ def get_schema_from_items(items: ItemsInput) -> ArrowSchema: if not isinstance(items, ArrowTable): items = rustac.to_arrow(items) - return IcestacItem.enforce_required_fields(items.schema) - - -def convert_schema(schema: ArrowSchema) -> IcebergSchema: - """Validate and convert an Arrow item schema to Iceberg with field IDs.""" + schema = IcestacItem.enforce_required_fields(items.schema) IcestacItem.validate_schema(schema) - _schema = _pyarrow_to_schema_without_ids( - pa.schema(IcestacItem.enforce_required_fields(schema)) - ) + schema_without_ids = _pyarrow_to_schema_without_ids(pa.schema(schema)) fields = [] - for i, _field in enumerate(_schema.fields, start=1): - field_dict = _field.model_dump() + for i, field in enumerate(schema_without_ids.fields, start=1): + field_dict = field.model_dump() field_dict["id"] = i fields.append(NestedField(**field_dict)) diff --git a/tests/test_catalog.py b/tests/test_catalog.py index 9632caa..44b100b 100644 --- a/tests/test_catalog.py +++ b/tests/test_catalog.py @@ -15,7 +15,6 @@ from icestac.schema import ( IcestacItem, ItemsInput, - convert_schema, get_schema_from_items, ) from tests.helpers import items_to_list @@ -27,9 +26,9 @@ def test_create_item_table( items: ItemsInput, ) -> None: expected_items = items_to_list(items) - arrow_schema = get_schema_from_items(items) + iceberg_schema = get_schema_from_items(items) table = test_catalog.create_item_table( - iceberg_schema=convert_schema(arrow_schema), + iceberg_schema=iceberg_schema, collection_id=test_collection_id, ) @@ -57,7 +56,7 @@ def test_create_item_table( with pytest.raises(TableAlreadyExistsError): test_catalog.create_item_table( - iceberg_schema=convert_schema(arrow_schema), + iceberg_schema=iceberg_schema, collection_id=test_collection_id, ) @@ -66,10 +65,10 @@ def test_create_item_table_bad_collection_id( test_catalog: IcestacCatalog, items: ItemsInput, ) -> None: - arrow_schema = get_schema_from_items(items) + iceberg_schema = get_schema_from_items(items) with pytest.raises(InvalidCollectionIdError): test_catalog.create_item_table( - iceberg_schema=convert_schema(arrow_schema), + iceberg_schema=iceberg_schema, collection_id="bad.collection", ) @@ -79,7 +78,7 @@ def test_create_item_table_custom_layout( test_collection_id: str, sample_stac_item: dict[str, Any], ) -> None: - iceberg_schema = convert_schema(get_schema_from_items(sample_stac_item)) + iceberg_schema = get_schema_from_items(sample_stac_item) title_id = iceberg_schema.find_field("title").field_id table = test_catalog.create_item_table( @@ -114,7 +113,7 @@ def test_create_item_table_unpartitioned( ) -> None: table = test_catalog.create_item_table( collection_id=test_collection_id, - iceberg_schema=convert_schema(get_schema_from_items(sample_stac_item)), + iceberg_schema=get_schema_from_items(sample_stac_item), partition_spec=PartitionSpec(), ) @@ -126,7 +125,7 @@ def test_create_item_table_rejects_invalid_schema( test_collection_id: str, sample_stac_item: dict[str, Any], ) -> None: - iceberg_schema = convert_schema(get_schema_from_items(sample_stac_item)) + iceberg_schema = get_schema_from_items(sample_stac_item) missing_id = IcebergSchema( *(field for field in iceberg_schema.fields if field.name != "id") ) @@ -146,7 +145,7 @@ def test_create_item_table_rejects_unknown_partition_source( with pytest.raises(ValueError): test_catalog.create_item_table( collection_id=test_collection_id, - iceberg_schema=convert_schema(get_schema_from_items(sample_stac_item)), + iceberg_schema=get_schema_from_items(sample_stac_item), partition_spec=PartitionSpec( PartitionField( source_id=9999, @@ -166,7 +165,7 @@ def test_create_item_table_rejects_unknown_sort_source( with pytest.raises(ValueError): test_catalog.create_item_table( collection_id=test_collection_id, - iceberg_schema=convert_schema(get_schema_from_items(sample_stac_item)), + iceberg_schema=get_schema_from_items(sample_stac_item), sort_order=SortOrder(SortField(source_id=9999)), ) @@ -176,9 +175,9 @@ def test_load_items_rejects_a_different_collection( test_collection_id: str, sample_stac_item: dict[str, Any], ) -> None: - arrow_schema = get_schema_from_items(sample_stac_item) + iceberg_schema = get_schema_from_items(sample_stac_item) table = test_catalog.create_item_table( - iceberg_schema=convert_schema(arrow_schema), + iceberg_schema=iceberg_schema, collection_id=test_collection_id, ) sample_stac_item["collection"] = "different-collection" @@ -195,9 +194,9 @@ def test_load_items( items: ItemsInput, ) -> None: expected_items = items_to_list(items) - arrow_schema = get_schema_from_items(items) + iceberg_schema = get_schema_from_items(items) table = test_catalog.create_item_table( - iceberg_schema=convert_schema(arrow_schema), + iceberg_schema=iceberg_schema, collection_id=test_collection_id, ) @@ -221,7 +220,7 @@ def test_catalog_load_items_evolves_schema( sample_stac_item: dict[str, Any], ) -> None: table = test_catalog.create_item_table( - iceberg_schema=convert_schema(get_schema_from_items(sample_stac_item)), + iceberg_schema=get_schema_from_items(sample_stac_item), collection_id=test_collection_id, ) evolved_item = { diff --git a/tests/test_load.py b/tests/test_load.py index c55dbaf..a7891d6 100644 --- a/tests/test_load.py +++ b/tests/test_load.py @@ -5,7 +5,7 @@ from icestac.catalog import IcestacCatalog from icestac.load import Method, load_items -from icestac.schema import ItemsInput, convert_schema, get_schema_from_items +from icestac.schema import ItemsInput, get_schema_from_items from tests.helpers import items_to_list @@ -18,9 +18,9 @@ def test_load_items_upsert_default( expected_items = items_to_list(items) # Create the table - arrow_schema = get_schema_from_items(items) + iceberg_schema = get_schema_from_items(items) table = test_catalog.create_item_table( - iceberg_schema=convert_schema(arrow_schema), + iceberg_schema=iceberg_schema, collection_id=test_collection_id, ) @@ -43,9 +43,9 @@ def test_load_items_upsert_explicit( expected_items = items_to_list(items) # Create the table - arrow_schema = get_schema_from_items(items) + iceberg_schema = get_schema_from_items(items) table = test_catalog.create_item_table( - iceberg_schema=convert_schema(arrow_schema), + iceberg_schema=iceberg_schema, collection_id=expected_items[0]["collection"], ) @@ -65,9 +65,9 @@ def test_load_items_upsert_updates_existing( expected_items = items_to_list(items) # Create the table - arrow_schema = get_schema_from_items(items) + iceberg_schema = get_schema_from_items(items) table = test_catalog.create_item_table( - iceberg_schema=convert_schema(arrow_schema), + iceberg_schema=iceberg_schema, collection_id=expected_items[0]["collection"], ) @@ -102,9 +102,9 @@ def test_load_items_append( expected_items = items_to_list(items) # Create the table - arrow_schema = get_schema_from_items(items) + iceberg_schema = get_schema_from_items(items) table = test_catalog.create_item_table( - iceberg_schema=convert_schema(arrow_schema), + iceberg_schema=iceberg_schema, collection_id=expected_items[0]["collection"], ) @@ -124,9 +124,9 @@ def test_load_items_append_creates_duplicates( expected_items = items_to_list(items) # Create the table - arrow_schema = get_schema_from_items(items) + iceberg_schema = get_schema_from_items(items) table = test_catalog.create_item_table( - iceberg_schema=convert_schema(arrow_schema), + iceberg_schema=iceberg_schema, collection_id=expected_items[0]["collection"], ) @@ -147,9 +147,9 @@ def test_load_items_multiple_batches( expected_items = items_to_list(items) # Create the table - arrow_schema = get_schema_from_items(items) + iceberg_schema = get_schema_from_items(items) table = test_catalog.create_item_table( - iceberg_schema=convert_schema(arrow_schema), + iceberg_schema=iceberg_schema, collection_id=expected_items[0]["collection"], ) @@ -171,9 +171,9 @@ def test_load_items_rejects_invalid_method( test_catalog: IcestacCatalog, sample_stac_item: dict[str, Any], ) -> None: - arrow_schema = get_schema_from_items(sample_stac_item) + iceberg_schema = get_schema_from_items(sample_stac_item) table = test_catalog.create_item_table( - iceberg_schema=convert_schema(arrow_schema), + iceberg_schema=iceberg_schema, collection_id=sample_stac_item["collection"], ) @@ -191,9 +191,9 @@ def test_load_items_supports_interval_datetime( "start_datetime": "2024-01-01T00:00:00Z", "end_datetime": "2024-01-02T00:00:00Z", } - arrow_schema = get_schema_from_items(interval_item) + iceberg_schema = get_schema_from_items(interval_item) table = test_catalog.create_item_table( - iceberg_schema=convert_schema(arrow_schema), + iceberg_schema=iceberg_schema, collection_id=interval_item["collection"], ) @@ -206,9 +206,9 @@ def test_load_items_different_schema( test_catalog: IcestacCatalog, sample_stac_item: dict[str, Any], ) -> None: - arrow_schema = get_schema_from_items(sample_stac_item) + iceberg_schema = get_schema_from_items(sample_stac_item) table = test_catalog.create_item_table( - iceberg_schema=convert_schema(arrow_schema), + iceberg_schema=iceberg_schema, collection_id=sample_stac_item["collection"], ) load_items([sample_stac_item], table) @@ -226,7 +226,7 @@ def test_load_items_evolves_schema( method: Method, ) -> None: table = test_catalog.create_item_table( - iceberg_schema=convert_schema(get_schema_from_items(sample_stac_item)), + iceberg_schema=get_schema_from_items(sample_stac_item), collection_id=sample_stac_item["collection"], ) load_items(sample_stac_item, table) @@ -248,7 +248,7 @@ def test_load_items_does_not_evolve_schema_when_write_fails( sample_stac_item: dict[str, Any], ) -> None: table = test_catalog.create_item_table( - iceberg_schema=convert_schema(get_schema_from_items(sample_stac_item)), + iceberg_schema=get_schema_from_items(sample_stac_item), collection_id=sample_stac_item["collection"], ) evolved_item = deepcopy(sample_stac_item) diff --git a/tests/test_schema.py b/tests/test_schema.py index fa4ff3b..50a12fa 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -2,21 +2,20 @@ import pyarrow as pa import pytest -from arro3.core import Schema from pydantic import ValidationError +from pyiceberg.schema import Schema -from icestac.schema import IcestacItem, convert_schema, get_schema_from_items +from icestac.schema import IcestacItem, get_schema_from_items def test_get_schema_from_items(sample_stac_item: dict[str, Any]) -> None: - """Test that we can extract an Arrow schema from a STAC item.""" + """Test that we can derive an Iceberg schema from a STAC item.""" schema = get_schema_from_items(sample_stac_item) assert isinstance(schema, Schema) - assert "id" in schema.names - assert "datetime" in schema.names - assert "collection" in schema.names - assert schema.field("geometry").metadata[b"ARROW:extension:name"] == b"geoarrow.wkb" + assert schema.find_field("title").field_id > 0 + assert schema.find_field("id").required + assert str(schema.find_field("geometry").field_type) == "binary" def test_get_schema_from_items_validates(sample_stac_item: dict[str, Any]) -> None: @@ -43,28 +42,11 @@ def test_validate_schema_valid(sample_stac_item: dict[str, Any]) -> None: IcestacItem.validate_schema(schema) -def test_convert_schema_prepares_item_schema( - sample_stac_item: dict[str, Any], -) -> None: - """Test that conversion preserves item fields and assigns source IDs.""" - iceberg_schema = convert_schema(get_schema_from_items(sample_stac_item)) - - assert iceberg_schema.find_field("title").field_id > 0 - assert iceberg_schema.find_field("id").required - assert str(iceberg_schema.find_field("geometry").field_type) == "binary" - - -def test_convert_schema_validates_required_fields() -> None: - """Test that conversion rejects an Arrow schema missing STAC fields.""" - with pytest.raises(ValueError, match="missing required STAC fields.*'id'"): - convert_schema(pa.schema([("type", pa.string())])) - - def test_validate_prepared_schema_missing_required_field( sample_stac_item: dict[str, Any], ) -> None: """Test that required-field validation accepts prepared Iceberg schemas.""" - iceberg_schema = convert_schema(get_schema_from_items(sample_stac_item)) + iceberg_schema = get_schema_from_items(sample_stac_item) missing_id = type(iceberg_schema)( *(field for field in iceberg_schema.fields if field.name != "id") ) From 4945cada8a3d250854e8341051a0f258401cafb3 Mon Sep 17 00:00:00 2001 From: hrodmn Date: Wed, 9 Sep 2026 12:09:37 -0500 Subject: [PATCH 22/23] feat: consolidate to arrow table as input format --- .github/workflows/ci.yml | 1 + .gitignore | 4 +- README.md | 185 ++++++--------------- main.py | 151 ++++++++---------- pyproject.toml | 3 +- src/icestac/catalog.py | 7 +- src/icestac/load.py | 59 ++++--- src/icestac/schema.py | 178 ++++++++++++--------- tests/conftest.py | 22 +-- tests/helpers.py | 18 +-- tests/test_catalog.py | 91 ++++++++--- tests/test_load.py | 336 ++++++++++++++++++++++++++++++++++++--- tests/test_main.py | 94 +++++++++++ tests/test_schema.py | 129 ++++++++++----- uv.lock | 175 +++++++++++++------- 15 files changed, 945 insertions(+), 508 deletions(-) create mode 100644 tests/test_main.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3e6c289..ad15714 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,7 @@ jobs: uses: astral-sh/setup-uv@v7 with: version: "0.10.*" + python-version: ${{ matrix.python-version }} enable-cache: true - name: Run pre-commit diff --git a/.gitignore b/.gitignore index 0e08b1d..8ed4cdf 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,7 @@ coverage.xml minio-data/ # dev docs -dev-docs/plans/ -dev-docs/brainstorms/ +dev-docs/ data/ +.worktrees/ diff --git a/README.md b/README.md index 9b03cc8..12adab0 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,10 @@ # icestac -## Overview +Store STAC Items in Apache Iceberg, with one table per collection. -`icestac` helps you store STAC Items in Apache Iceberg using rustac's flattened Arrow representation. +## Load items -The library has three main pieces: - -- `src/icestac/schema.py` validates STAC inputs, preserves Arrow metadata, and converts Arrow schemas to Iceberg schemas with field IDs. -- `src/icestac/catalog.py` wraps a PyIceberg catalog and creates one item table per collection in the `icestac` namespace. -- `src/icestac/load.py` accepts STAC dictionaries or `arro3.core.Table` data and writes it with `append` or `upsert` semantics. - -The project is still an early foundation rather than a released storage specification. A few boundaries are worth knowing before you begin: - -- `icestac` uses each collection ID as its table name. Iceberg treats periods as namespace delimiters, so collection IDs that contain periods are unsupported. Rename those IDs before loading them; `icestac` will not rewrite them for you. -- Tables use monthly `datetime` partitions by default. You can replace that layout with native PyIceberg partition and sort objects. We plan to add built-in spatial layouts as support for geospatial types across the Iceberg ecosystem improves. -- Table schemas remain strict by default. Loading a new shape fails unless you opt into compatible schema evolution. -- Geometry is stored as WKB. PyIceberg does not yet emit the GeoParquet and STAC GeoParquet file metadata needed for compliance with those specifications. - -## Core API - -The API adds a few STAC-focused conveniences while keeping native PyIceberg objects available for table layout and schema management. - -To get started, create a table and load some items. `IcestacCatalog.create_item_table(...)` uses a monthly partition based on the `datetime` property unless you provide another layout. +The input is an Arrow table (`pyarrow.Table` or `arro3.core.Table`) in rustac's flattened STAC format. ```python from pyiceberg.catalog import load_catalog @@ -29,47 +12,47 @@ from pyiceberg.catalog import load_catalog from icestac.catalog import IcestacCatalog from icestac.schema import get_schema_from_items +# items is an Arrow table containing items from one collection. +collection_id = "my-collection" catalog = IcestacCatalog(catalog=load_catalog()) -iceberg_schema = get_schema_from_items(items) -catalog.create_item_table(collection_id=collection_id, iceberg_schema=iceberg_schema) +schema = get_schema_from_items(items) +catalog.create_item_table(collection_id=collection_id, iceberg_schema=schema) catalog.load_items(collection_id=collection_id, items=items) ``` -`get_schema_from_items(...)` accepts one STAC item dictionary, a list of dictionaries, or an `arro3.core.Table`, validates it, and returns an Iceberg schema with the field IDs needed for table layout. `load_items(...)` accepts the same inputs, checks that every row belongs to the target collection, and upserts on STAC `id` by default. +Use the collection ID from your items. IDs containing periods are unsupported because icestac uses them in dotted table identifiers. Tables live in the `icestac` namespace by default. + +Loading checks the collection ID and upserts on STAC `id`. To append without replacing existing items, pass `method="append"`. -Incoming fields that are absent from the table schema raise an error. If you want PyIceberg to add compatible fields by name, pass `evolve_schema=True`. The schema update and write then commit in one transaction: +### Schema evolution + +New fields raise an error unless you pass `evolve_schema=True`: ```python -catalog.load_items( - collection_id=collection_id, - items=items, - evolve_schema=True, -) +catalog.load_items(collection_id=collection_id, items=items, evolve_schema=True) ``` -If you manage schema changes separately, update the table with PyIceberg before loading: +Compatible schema changes and the write commit in one transaction. To update a schema without loading items, use PyIceberg: ```python -iceberg_schema = get_schema_from_items(items) table = catalog.catalog.load_table((catalog.namespace, collection_id)) with table.update_schema() as update: - update.union_by_name(iceberg_schema) - -catalog.load_items(collection_id=collection_id, items=items) + update.union_by_name(get_schema_from_items(items)) ``` -Without layout configuration, a table gets the monthly `datetime` partition and no sort order. For a custom layout, build native PyIceberg objects from the prepared schema. This example partitions items by year and sorts them by a `sortme` property: +### Table layout + +Tables use monthly `datetime` partitions and no sort order by default. Pass native PyIceberg objects to choose another layout. For example, partition by year and sort by `datetime`: ```python from pyiceberg.partitioning import PartitionField, PartitionSpec from pyiceberg.table.sorting import SortField, SortOrder from pyiceberg.transforms import YearTransform -datetime_id = iceberg_schema.find_field("datetime").field_id -sort_id = iceberg_schema.find_field("sortme").field_id +datetime_id = schema.find_field("datetime").field_id catalog.create_item_table( collection_id=collection_id, - iceberg_schema=iceberg_schema, + iceberg_schema=schema, partition_spec=PartitionSpec( PartitionField( source_id=datetime_id, @@ -78,103 +61,47 @@ catalog.create_item_table( name="datetime_year", ) ), - sort_order=SortOrder(SortField(source_id=sort_id)), -) -``` - -A partition specification replaces the monthly default. Pass `PartitionSpec()` for an unpartitioned table, and omit the sort order if you do not need one. PyIceberg validates partition and sort references, so build both from the same prepared schema that you pass to `create_item_table(...)`. - -STAC interval items may have a null `datetime` when they include `start_datetime` and `end_datetime`. PyIceberg writes these items to a valid `datetime_month=null` partition. For a collection of interval items, you can partition by the start month instead: - -```python -from pyiceberg.partitioning import PartitionField, PartitionSpec -from pyiceberg.transforms import MonthTransform - -start_datetime_id = iceberg_schema.find_field("start_datetime").field_id -catalog.create_item_table( - collection_id=collection_id, - iceberg_schema=iceberg_schema, - partition_spec=PartitionSpec( - PartitionField( - source_id=start_datetime_id, - field_id=1000, - transform=MonthTransform(), - name="start_datetime_month", - ) - ), + sort_order=SortOrder(SortField(source_id=datetime_id)), ) ``` -Iceberg partition transforms use one source field. If your collection mixes point and interval items, add a derived timestamp column to support a per-row fallback. +Use this instead of the earlier `create_item_table` call. Build field references from the schema you pass to it. Pass `PartitionSpec()` for an unpartitioned table. -## Development +Interval items with a null `datetime` go into a null partition. To partition them by their start time, use the `start_datetime` field with `MonthTransform()`. -### Tests +Geometry uses WKB in an Iceberg binary column. The output files lack the metadata required for GeoParquet and STAC GeoParquet compliance. -Run the test suite with: +## Local demo -```bash -uv run pytest -``` - -### Local instance +Run these commands from the repository root. You need [uv](https://docs.astral.sh/uv/), Docker Compose, and DuckDB for the query examples. -You can run a complete catalog and object storage environment on your machine. - -**1. Start the local environment** +### Start the services ```bash -docker compose up -``` - -This starts three services: - -- **Iceberg REST Catalog** at `http://localhost:8181` -- **MinIO** (S3-compatible storage) at `http://localhost:9000` for the API and `http://localhost:9001` for the console -- **MinIO Client**, which creates the `warehouse` bucket on startup - -**2. Configure catalog access** - -The repository includes `.pyiceberg.yaml` with default credentials for the local Docker environment: - -```yaml -catalog: - default: - type: rest - uri: http://localhost:8181 - warehouse: s3://warehouse/ - s3.endpoint: http://localhost:9000 - s3.access-key-id: admin - s3.secret-access-key: password - s3.path-style-access: "true" +docker compose up -d ``` -PyIceberg reads this file when you run commands from the project directory. If you need a different setup, see the [PyIceberg configuration docs](https://py.iceberg.apache.org/configuration/). +- Iceberg REST catalog: `http://localhost:8181` +- MinIO API: `http://localhost:9000`; console: `http://localhost:9001` +- The `mc` service creates the `warehouse` bucket. -**3. Load sample items** +The checked-in `.pyiceberg.yaml` points to these services. The credentials (`admin` / `password`) are for local development only. See [PyIceberg configuration](https://py.iceberg.apache.org/configuration/) for other environments. -Once the services are running, use `main.py` to try the current ingestion workflow: +### Load HLS items ```bash -uv run python main.py +uv run main.py ``` -The script downloads eight months of HLS STAC GeoParquet from public S3, reads each file directly as Arrow, and loads it through `IcestacCatalog`. - -The source collection ID, `HLSS30_2.0`, contains a period, which Iceberg interprets as a namespace delimiter. For this demo, the script renames the collection to `HLSS30_2_0` in every item before creating the table. The script makes this dataset decision explicitly because `icestac` does not rewrite collection IDs. - -The June 2026 items introduce a compatible schema change. `main.py` passes `evolve_schema=True` when it loads that batch so you can see schema evolution in use. - -**4. Query with DuckDB** +The demo downloads January–August 2026 HLS STAC GeoParquet from public S3 into `data/`, reuses cached files, and upserts each month. Each batch must fit in memory. -After the load finishes, you can query the Iceberg tables with DuckDB's `iceberg` extension. The tables live under the `icestac` namespace. +It renames collection `HLSS30_2.0` to `HLSS30_2_0`, sorts items by DuckDB's Hilbert index of their bbox lower-left coordinates, and caps Parquet row groups at 50,000 rows. New tables record the Hilbert sort order. Compatible schema changes use `evolve_schema=True`. -Start by configuring the extensions and MinIO credentials: +### Query with DuckDB ```sql INSTALL iceberg; LOAD iceberg; INSTALL httpfs; LOAD httpfs; -INSTALL spatial; LOAD spatial; CREATE OR REPLACE SECRET minio ( TYPE S3, @@ -184,11 +111,7 @@ CREATE OR REPLACE SECRET minio ( USE_SSL false, URL_STYLE 'path' ); -``` - -Then query through the REST catalog: -```sql ATTACH 'icestac' AS catalog ( TYPE ICEBERG, ENDPOINT 'http://localhost:8181', @@ -199,41 +122,23 @@ SELECT id, datetime, collection, geometry FROM catalog.icestac.HLSS30_2_0 LIMIT 10; -SELECT count(*) -FROM catalog.icestac.HLSS30_2_0; - -DESCRIBE SELECT bbox, geometry FROM catalog.icestac.HLSS30_2_0; -``` - -You can also scan the table directly from its S3 path without a catalog: - -```sql -SET unsafe_enable_version_guessing = true; -SELECT * -FROM iceberg_scan('s3://warehouse/icestac/HLSS30_2_0') -LIMIT 10; +SELECT count(*) FROM catalog.icestac.HLSS30_2_0; ``` -## Delete a table +### Delete the demo table -To start over with a table, call `drop_table(...)` on the underlying PyIceberg catalog: +**This deletes the table and its data.** ```python from pyiceberg.catalog import load_catalog -from icestac.catalog import IcestacCatalog -catalog = IcestacCatalog(catalog=load_catalog()) -catalog.catalog.drop_table( - ("icestac", "HLSS30_2_0"), - purge_requested=True, -) +load_catalog().purge_table(("icestac", "HLSS30_2_0")) ``` -With `purge_requested=True`, the REST catalog deletes the table data along with the catalog entry. +To remove only the catalog entry and keep the files, use `drop_table(...)` instead. -If the catalog entry is gone but files remain in MinIO, remove the warehouse path: +## Tests ```bash -docker compose exec mc mc rm --recursive --force minio/warehouse/icestac/HLSS30_2_0 +uv run pytest ``` - diff --git a/main.py b/main.py index 4cc5d48..34c18e3 100644 --- a/main.py +++ b/main.py @@ -1,11 +1,13 @@ import asyncio import logging +from pathlib import Path -import pyarrow -from arro3.core import Table as ArrowTable +import pyarrow as pa from obstore.store import LocalStore, S3Store from pyiceberg.catalog import load_catalog from pyiceberg.exceptions import TableAlreadyExistsError +from pyiceberg.table import TableProperties +from pyiceberg.table.sorting import SortField, SortOrder from rustac import DuckdbClient from icestac.catalog import IcestacCatalog @@ -15,109 +17,94 @@ HLS_STAC_GEOPARQUET_BUCKET = "nasa-maap-data-store" HLS_STAC_GEOPARQUET_PREFIX = "file-staging/nasa-map/hls-stac-geoparquet-archive/v2" -HLS_STAC_GEOPARQUET_PATH_FMT = ( - "{collection}/year={year}/month={month}/{collection}-{year}-{month}.parquet" -) - - -async def copy_hls_stac_geoparquet(path: str, store: LocalStore) -> None: - """Copy one public HLS STAC GeoParquet file into a local store.""" - hls_stac_store = S3Store( - bucket=HLS_STAC_GEOPARQUET_BUCKET, - prefix=HLS_STAC_GEOPARQUET_PREFIX, - region="us-west-2", - skip_signature=True, +MAX_ROW_GROUP_SIZE = 50_000 + + +def read_hls_items(client: DuckdbClient, path: Path, collection_id: str) -> pa.Table: + """Read an HLS batch, rename its collection, and sort by bbox Hilbert index.""" + # Keep GeoParquet geometry as WKB for Iceberg's binary column. + client.execute("SET enable_geoparquet_conversion = false") + client.execute("SET TimeZone = 'UTC'") + items = pa.table( + client.query_to_table( + """ + SELECT * REPLACE (? AS collection), + CASE WHEN isfinite(bbox.xmin) AND isfinite(bbox.ymin) + THEN ST_Hilbert( + bbox.xmin, bbox.ymin, + {min_x: -180.0, min_y: -90.0, + max_x: 180.0, max_y: 90.0}::BOX_2D + )::BIGINT + END AS hilbert_idx + FROM read_parquet(?, hive_partitioning = false) + ORDER BY hilbert_idx + """, + [collection_id, str(path)], + ) ) - - resp = await hls_stac_store.get_async(path) - await store.put_async(path, resp) + if not len(items): + raise ValueError(f"No items found in {path}") + if items["hilbert_idx"].null_count: + raise ValueError(f"Missing or non-finite bbox coordinates in {path}") + return items async def run() -> None: - """Load several months of HLS STAC GeoParquet into local Iceberg.""" + """Load January–August 2026 HLS items into the local Iceberg catalog.""" logging.basicConfig( level=logging.INFO, format="%(asctime)s %(levelname)s:%(name)s:%(message)s", datefmt="%Y-%m-%dT%H:%M:%S%z", ) catalog = IcestacCatalog(catalog=load_catalog()) - - duckdb_client = DuckdbClient() - duckdb_client.execute("SET TimeZone = 'UTC';") - local_store = LocalStore("data") - + client = DuckdbClient() + data_dir = Path("data") + data_dir.mkdir(exist_ok=True) + local_store = LocalStore(data_dir) + source_store = S3Store( + bucket=HLS_STAC_GEOPARQUET_BUCKET, + prefix=HLS_STAC_GEOPARQUET_PREFIX, + region="us-west-2", + skip_signature=True, + ) source_collection_id = "HLSS30_2.0" collection_id = "HLSS30_2_0" - table_exists = False - for month in range(1, 9, 1): - month_logger = logger.getChild(f"2026-{month}") - stac_geoparquet_path = HLS_STAC_GEOPARQUET_PATH_FMT.format( - collection=source_collection_id, - year="2026", - month=str(month), + for month in range(1, 9): + path = ( + f"{source_collection_id}/year=2026/month={month}/" + f"{source_collection_id}-2026-{month}.parquet" ) - try: - _ = local_store.head(stac_geoparquet_path) + local_store.head(path) except FileNotFoundError: - month_logger.info("downloading %s", stac_geoparquet_path) - await copy_hls_stac_geoparquet( - path=stac_geoparquet_path, - store=local_store, - ) - - month_logger.info("loading items as arrow table") - items = duckdb_client.search_to_arrow(href=f"data/{stac_geoparquet_path}") + logger.info("Downloading %s", path) + response = await source_store.get_async(path) + await local_store.put_async(path, response) - if not items: - raise ValueError("No items found") + logger.info("Reading and sorting %s", path) + items = read_hls_items(client, data_dir / path, collection_id) - items_table = pyarrow.table(items) - collection_index = items_table.schema.get_field_index("collection") - collection_field = items_table.schema.field(collection_index) - items = ArrowTable.from_arrow( - items_table.set_column( - collection_index, - collection_field, - pyarrow.array( - [collection_id] * len(items_table), type=collection_field.type - ), - ) - ) - iceberg_schema = get_schema_from_items(items) - - if not table_exists: + if month == 1: + schema = get_schema_from_items(items) try: - catalog.create_item_table( - iceberg_schema=iceberg_schema, + table = catalog.create_item_table( + iceberg_schema=schema, collection_id=collection_id, + sort_order=SortOrder( + SortField(source_id=schema.find_field("hilbert_idx").field_id) + ), ) except TableAlreadyExistsError: - month_logger.warning("%s table already exists; using it", collection_id) - table_exists = True - - month_logger.info("loading items into icestac catalog") - try: - catalog.load_items( - collection_id=collection_id, - items=items, - method="upsert", - evolve_schema=False, - ) - except ValueError as e: - if "Update the schema first (hint, use union_by_name)" not in str(e): - raise - - month_logger.warning(str(e)) - month_logger.info("retrying load with evolve_schema=True") + table = catalog.catalog.load_table((catalog.namespace, collection_id)) + logger.info("Using existing table %s", collection_id) + with table.transaction() as transaction: + transaction.set_properties( + {TableProperties.PARQUET_ROW_GROUP_LIMIT: str(MAX_ROW_GROUP_SIZE)} + ) - catalog.load_items( - collection_id=collection_id, - items=items, - method="upsert", - evolve_schema=True, - ) + logger.info("Loading %s items for 2026-%02d", len(items), month) + catalog.load_items(collection_id, items, evolve_schema=True) if __name__ == "__main__": diff --git a/pyproject.toml b/pyproject.toml index 7cba726..c828005 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,9 +19,8 @@ classifiers = [ requires-python = ">=3.11" dependencies = [ "pyarrow>=23.0.0", - "pyiceberg[pyiceberg-core]>=0.10.0", + "pyiceberg[pyiceberg-core]>=0.12.0", "rustac[arrow]>=0.9.3", - "stac-pydantic>=3.4.0", ] [build-system] diff --git a/src/icestac/catalog.py b/src/icestac/catalog.py index 8db6eb9..d6f3b21 100644 --- a/src/icestac/catalog.py +++ b/src/icestac/catalog.py @@ -5,7 +5,7 @@ from pyiceberg.catalog import Catalog from pyiceberg.partitioning import PartitionField, PartitionSpec from pyiceberg.schema import Schema as IcebergSchema -from pyiceberg.table import Table +from pyiceberg.table import Table, TableProperties from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder from pyiceberg.transforms import MonthTransform @@ -29,9 +29,6 @@ class IcestacCatalog: catalog: Catalog namespace: str = DEFAULT_NAMESPACE - def __post_init__(self) -> None: - self.catalog.create_namespace_if_not_exists(self.namespace) - def create_item_table( self, collection_id: str, @@ -42,6 +39,7 @@ def create_item_table( """Create an Iceberg item table for a collection.""" validate_collection_id(collection_id) IcestacItem.validate_schema(iceberg_schema) + self.catalog.create_namespace_if_not_exists(self.namespace) # TODO: check if collection record is present in collections table @@ -60,6 +58,7 @@ def create_item_table( schema=iceberg_schema, partition_spec=partition_spec, sort_order=sort_order, + properties={TableProperties.COMMIT_NUM_RETRIES: "0"}, ) def load_items( diff --git a/src/icestac/load.py b/src/icestac/load.py index a516e92..7297246 100644 --- a/src/icestac/load.py +++ b/src/icestac/load.py @@ -1,11 +1,10 @@ -from typing import Any, Literal, cast +import warnings +from typing import Literal -import pyarrow -import rustac -from arro3.core import Table as ArrowTable -from pyiceberg.table import Table +from pyiceberg.table import Table, TableProperties +from pyiceberg.table.upsert_util import create_match_filter, has_duplicate_rows -from icestac.schema import IcestacItem, ItemsInput +from icestac.schema import ItemsInput, prepare_arrow_table Method = Literal["append", "upsert"] @@ -16,19 +15,19 @@ def load_items( method: Method = "upsert", evolve_schema: bool = False, ) -> None: - """Load STAC items, optionally evolving the Iceberg schema by name.""" + """Load STAC items, optionally evolving the Iceberg schema by name. + + Upserts are complete replacements: omitted optional fields are written as + null rather than retained from the previous item. Upsert commit retries + are disabled because PyIceberg cannot safely replay the match against + refreshed table state. A known ``CommitFailedException`` + is propagated for a fresh retry; a ``CommitStateUnknownException`` is also + propagated and requires refreshing/reconciling before retrying the load. + """ if method not in ("append", "upsert"): raise ValueError(f"Unsupported load method: {method}") - if isinstance(items, dict): - item = cast(dict[str, Any], items) - items = [item] - - if not isinstance(items, ArrowTable): - items = rustac.to_arrow(items) - - enforced_schema = IcestacItem.enforce_required_fields(items.schema) - arrow_table = pyarrow.table(items).cast(pyarrow.schema(enforced_schema)) + arrow_table = prepare_arrow_table(items) collection_ids = set(arrow_table.column("collection").unique().to_pylist()) expected_collection_id = table.name()[-1] if collection_ids != {expected_collection_id}: @@ -37,15 +36,35 @@ def load_items( f"{sorted(map(str, collection_ids))}" ) + if method == "upsert": + # PyIceberg retries a staged merge against refreshed metadata without + # rerunning the original upsert lookup, which can duplicate an ID. + table.metadata.properties[TableProperties.COMMIT_NUM_RETRIES] = "0" + with table.transaction() as transaction: if evolve_schema: with transaction.update_schema() as update: update.union_by_name(arrow_table.schema) if method == "upsert": - transaction.upsert( - df=arrow_table, - join_cols=["id"], - ) + # Overwrite matching IDs so omitted optional fields are cleared. + # PyIceberg handles schema alignment, including staged evolution. + if has_duplicate_rows(arrow_table, ["id"]): + raise ValueError( + "Duplicate rows found in source dataset based on the key " + "columns. No upsert executed" + ) + with warnings.catch_warnings(): + # New IDs legitimately have no existing records to delete. + warnings.filterwarnings( + "ignore", + message="^Delete operation did not match any records$", + category=UserWarning, + module=r"^pyiceberg\.table$", + ) + transaction.overwrite( + df=arrow_table, + overwrite_filter=create_match_filter(arrow_table, ["id"]), + ) else: transaction.append(df=arrow_table) diff --git a/src/icestac/schema.py b/src/icestac/schema.py index cf068f6..17de122 100644 --- a/src/icestac/schema.py +++ b/src/icestac/schema.py @@ -1,69 +1,70 @@ -from types import NoneType -from typing import Any, cast, get_args - import pyarrow as pa -import rustac from arro3.core import Schema as ArrowSchema from arro3.core import Table as ArrowTable from pyiceberg.io.pyarrow import _pyarrow_to_schema_without_ids from pyiceberg.schema import Schema as IcebergSchema -from pyiceberg.types import NestedField -from stac_pydantic.item import Item - -ItemsInput = ArrowTable | list[dict[str, Any]] | dict[str, Any] +from pyiceberg.schema import assign_fresh_schema_ids +from pyiceberg.types import ( + BinaryType, + ListType, + StringType, + StructType, + TimestampType, + TimestamptzType, +) + +ItemsInput = pa.Table | ArrowTable +LINK_TYPE = pa.list_( + pa.struct( + [ + pa.field("href", pa.string(), nullable=False), + pa.field("rel", pa.string(), nullable=False), + pa.field("type", pa.string()), + pa.field("title", pa.string()), + ] + ) +) +EMPTY_ITEMS_ERROR = "Cannot infer or load a schema from an empty Arrow table" -class IcestacItem(Item): - collection: str +class IcestacItem: + """Structural fields required by the flattened STAC representation.""" @classmethod def get_required_fields(cls) -> set[str]: - """ - Get the set of required field names from IcestacItem. - - Returns: - Set of required field names, with special handling for flattened properties - """ - required_fields = set() - - for field_name, field_info in cls.model_fields.items(): - if field_info.is_required(): - required_fields.add(field_name) - - # Special handling: rustac flattens properties.datetime to just "datetime" - if "properties" in required_fields: - required_fields.remove("properties") - required_fields.add("datetime") - - return required_fields + """Return required top-level fields, including flattened datetime.""" + return {"geometry", "type", "id", "datetime", "links", "collection", "assets"} @classmethod def get_non_nullable_fields(cls) -> set[str]: - """Get STAC fields whose values cannot be null.""" - fields = { - name - for name, info in cls.model_fields.items() - if info.is_required() and NoneType not in get_args(info.annotation) - } - fields.discard("properties") - return fields + """Return required fields that cannot contain null values.""" + return {"type", "id", "links", "collection", "assets"} @classmethod def enforce_required_fields(cls, schema: ArrowSchema) -> ArrowSchema: - """Mark non-null STAC fields as non-nullable without losing metadata.""" + """Make required fields non-null and give empty links a concrete type.""" non_nullable_fields = cls.get_non_nullable_fields() - fields = [ - field.with_nullable(False) if field.name in non_nullable_fields else field - for field in schema - ] - return ArrowSchema(fields=fields, metadata=schema.metadata) + arrow_schema = pa.schema(schema) + fields = [] + for field in arrow_schema: + if field.name in non_nullable_fields: + field = field.with_nullable(False) + if ( + field.name == "links" + and pa.types.is_list(field.type) + and pa.types.is_null(field.type.value_type) + ): + field = field.with_type(LINK_TYPE) + fields.append(field) + return ArrowSchema.from_arrow(pa.schema(fields, metadata=arrow_schema.metadata)) @classmethod - def validate_schema(cls, schema: ArrowSchema | IcebergSchema) -> None: - """Validate that a schema contains the required STAC item fields.""" - schema_fields = set( - schema.column_names if isinstance(schema, IcebergSchema) else schema.names - ) + def validate_schema(cls, schema: pa.Schema | ArrowSchema | IcebergSchema) -> None: + """Validate required fields and the types used by the STAC representation.""" + if isinstance(schema, IcebergSchema): + schema_fields = set(schema.column_names) + else: + schema_fields = set(schema.names) missing_fields = cls.get_required_fields() - schema_fields if missing_fields: @@ -71,41 +72,62 @@ def validate_schema(cls, schema: ArrowSchema | IcebergSchema) -> None: f"Schema is missing required STAC fields: {sorted(missing_fields)}" ) + if isinstance(schema, IcebergSchema): + fields = {field.name: field.field_type for field in schema.fields} + expected = { + "id": StringType, + "collection": StringType, + "geometry": BinaryType, + "datetime": (TimestampType, TimestamptzType), + "links": ListType, + "assets": StructType, + } + invalid = [ + name + for name, field_type in expected.items() + if not isinstance(fields[name], field_type) + ] + else: + arrow_schema = pa.schema(schema) + invalid = [] + for name, predicate in ( + ("id", pa.types.is_string), + ("collection", pa.types.is_string), + ("geometry", pa.types.is_binary), + ("datetime", pa.types.is_timestamp), + ("links", pa.types.is_list), + ("assets", pa.types.is_struct), + ): + field_type = arrow_schema.field(name).type + if name in ("id", "collection") and pa.types.is_dictionary(field_type): + field_type = field_type.value_type + if not predicate(field_type): + invalid.append(name) + + if invalid: + raise ValueError( + "Unsupported types for STAC fields: " + ", ".join(sorted(invalid)) + ) -def _first_item_from_arrow(items: ArrowTable) -> dict[str, Any]: - table = pa.table(items) +def prepare_arrow_table(items: ItemsInput) -> pa.Table: + """Check an Arrow table's structural schema and normalize required fields.""" + if isinstance(items, ArrowTable): + table = pa.table(items) + elif isinstance(items, pa.Table): + table = items + else: + raise TypeError("items must be a pyarrow.Table or arro3.core.Table") if len(table) == 0: - raise ValueError("Cannot validate an empty Arrow table") + raise ValueError(EMPTY_ITEMS_ERROR) - feature_collection = rustac.from_arrow(table.slice(0, 1)) - - return feature_collection["features"][0] + IcestacItem.validate_schema(table.schema) + schema = IcestacItem.enforce_required_fields(ArrowSchema.from_arrow(table.schema)) + return table.cast(pa.schema(schema)) def get_schema_from_items(items: ItemsInput) -> IcebergSchema: - """Derive an Iceberg schema from STAC dictionaries or Arrow data.""" - if isinstance(items, dict): - item = cast(dict[str, Any], items) - items = [item] - elif isinstance(items, list): - item = items[0] - elif isinstance(items, ArrowTable): - item = _first_item_from_arrow(items) - - IcestacItem.model_validate(item) - - if not isinstance(items, ArrowTable): - items = rustac.to_arrow(items) - - schema = IcestacItem.enforce_required_fields(items.schema) - IcestacItem.validate_schema(schema) - schema_without_ids = _pyarrow_to_schema_without_ids(pa.schema(schema)) - - fields = [] - for i, field in enumerate(schema_without_ids.fields, start=1): - field_dict = field.model_dump() - field_dict["id"] = i - fields.append(NestedField(**field_dict)) - - return IcebergSchema(*fields) + """Derive an Iceberg schema from a structurally valid Arrow table.""" + arrow_table = prepare_arrow_table(items) + schema_without_ids = _pyarrow_to_schema_without_ids(arrow_table.schema) + return assign_fresh_schema_ids(schema_without_ids) diff --git a/tests/conftest.py b/tests/conftest.py index d2be113..435c154 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,13 +3,12 @@ from pathlib import Path from typing import Any, Generator +import pyarrow as pa import pytest import rustac -from arro3.core import Table as ArrowTable from pyiceberg.catalog import load_catalog from icestac.catalog import IcestacCatalog -from icestac.schema import ItemsInput @pytest.fixture @@ -91,22 +90,11 @@ def sample_stac_items(sample_stac_item) -> list[dict[str, Any]]: @pytest.fixture -def sample_stac_item_arrow_table(sample_stac_items) -> ArrowTable: - return rustac.to_arrow(sample_stac_items) +def sample_stac_item_arrow_table(sample_stac_items) -> pa.Table: + return pa.table(rustac.to_arrow(sample_stac_items)) @pytest.fixture -def sample_item_arrow_table(sample_stac_item_arrow_table: ArrowTable) -> ArrowTable: - """Backward-compatible alias for the Arrow-backed STAC items fixture.""" +def items(sample_stac_item_arrow_table: pa.Table) -> pa.Table: + """Provide a batch in the public Arrow ingestion format.""" return sample_stac_item_arrow_table - - -@pytest.fixture( - params=[ - pytest.param("sample_stac_item", id="single-item"), - pytest.param("sample_stac_items", id="item-list"), - pytest.param("sample_stac_item_arrow_table", id="arrow-table"), - ] -) -def items(request) -> ItemsInput: - return request.getfixturevalue(request.param) diff --git a/tests/helpers.py b/tests/helpers.py index 72f48e4..35d7873 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,16 +1,14 @@ -from typing import Any, cast +from typing import Any import pyarrow as pa import rustac -from arro3.core import Table as ArrowTable -from icestac.schema import ItemsInput +def items_to_arrow(items: list[dict[str, Any]]) -> pa.Table: + """Convert test STAC fixtures to the public Arrow input format.""" + return pa.table(rustac.to_arrow(items)) -def items_to_list(items: ItemsInput) -> list[dict[str, Any]]: - if isinstance(items, dict): - item = cast(dict[str, Any], items) - return [item] - if isinstance(items, ArrowTable): - return rustac.from_arrow(pa.table(items))["features"] - return items + +def items_to_list(items: pa.Table) -> list[dict[str, Any]]: + """Reconstruct Arrow rows for test assertions.""" + return rustac.from_arrow(items)["features"] diff --git a/tests/test_catalog.py b/tests/test_catalog.py index 44b100b..9845eba 100644 --- a/tests/test_catalog.py +++ b/tests/test_catalog.py @@ -2,28 +2,29 @@ import pyarrow import pytest -from arro3.core import Table as ArrowTable +from arro3.core import Schema as ArrowSchema from pyiceberg.exceptions import TableAlreadyExistsError from pyiceberg.partitioning import PartitionField, PartitionSpec from pyiceberg.schema import Schema as IcebergSchema from pyiceberg.table.sorting import SortField, SortOrder from pyiceberg.transforms import IdentityTransform, MonthTransform -from rustac import to_arrow - from icestac.catalog import IcestacCatalog from icestac.errors import InvalidCollectionIdError -from icestac.schema import ( - IcestacItem, - ItemsInput, - get_schema_from_items, -) -from tests.helpers import items_to_list +from icestac.schema import IcestacItem, get_schema_from_items +from tests.helpers import items_to_arrow, items_to_list + + +def test_catalog_constructor_does_not_create_namespace( + test_catalog: IcestacCatalog, +) -> None: + """Constructing a catalog wrapper does not mutate a read-only backend.""" + assert not test_catalog.catalog.namespace_exists(test_catalog.namespace) def test_create_item_table( test_catalog: IcestacCatalog, test_collection_id: str, - items: ItemsInput, + items: pyarrow.Table, ) -> None: expected_items = items_to_list(items) iceberg_schema = get_schema_from_items(items) @@ -38,11 +39,10 @@ def test_create_item_table( assert not table.sort_order().fields # Ensure data has required fields marked as non-nullable to match table schema - arrow_data = ( - to_arrow(expected_items) if not isinstance(items, ArrowTable) else items + enforced_schema = IcestacItem.enforce_required_fields( + ArrowSchema.from_arrow(items.schema) ) - enforced_schema = IcestacItem.enforce_required_fields(arrow_data.schema) - arrow_table = pyarrow.table(arrow_data).cast(pyarrow.schema(enforced_schema)) + arrow_table = items.cast(pyarrow.schema(enforced_schema)) table.upsert( df=arrow_table, @@ -63,7 +63,7 @@ def test_create_item_table( def test_create_item_table_bad_collection_id( test_catalog: IcestacCatalog, - items: ItemsInput, + items: pyarrow.Table, ) -> None: iceberg_schema = get_schema_from_items(items) with pytest.raises(InvalidCollectionIdError): @@ -78,7 +78,8 @@ def test_create_item_table_custom_layout( test_collection_id: str, sample_stac_item: dict[str, Any], ) -> None: - iceberg_schema = get_schema_from_items(sample_stac_item) + items = items_to_arrow([sample_stac_item]) + iceberg_schema = get_schema_from_items(items) title_id = iceberg_schema.find_field("title").field_id table = test_catalog.create_item_table( @@ -106,14 +107,45 @@ def test_create_item_table_custom_layout( ) +def test_create_item_table_nested_layout( + test_catalog: IcestacCatalog, + test_collection_id: str, + sample_stac_item: dict[str, Any], +) -> None: + items = items_to_arrow([sample_stac_item]) + iceberg_schema = get_schema_from_items(items) + xmin_id = iceberg_schema.find_field("bbox.xmin").field_id + + table = test_catalog.create_item_table( + collection_id=test_collection_id, + iceberg_schema=iceberg_schema, + partition_spec=PartitionSpec( + PartitionField( + source_id=xmin_id, + field_id=1000, + transform=IdentityTransform(), + name="bbox_xmin", + ) + ), + sort_order=SortOrder(SortField(source_id=xmin_id)), + ) + + table_xmin_id = table.schema().find_field("bbox.xmin").field_id + table_ymax_id = table.schema().find_field("bbox.ymax").field_id + assert table_xmin_id != table_ymax_id + assert table.spec().fields[0].source_id == table_xmin_id + assert table.sort_order().fields[0].source_id == table_xmin_id + + def test_create_item_table_unpartitioned( test_catalog: IcestacCatalog, test_collection_id: str, sample_stac_item: dict[str, Any], ) -> None: + items = items_to_arrow([sample_stac_item]) table = test_catalog.create_item_table( collection_id=test_collection_id, - iceberg_schema=get_schema_from_items(sample_stac_item), + iceberg_schema=get_schema_from_items(items), partition_spec=PartitionSpec(), ) @@ -125,7 +157,8 @@ def test_create_item_table_rejects_invalid_schema( test_collection_id: str, sample_stac_item: dict[str, Any], ) -> None: - iceberg_schema = get_schema_from_items(sample_stac_item) + items = items_to_arrow([sample_stac_item]) + iceberg_schema = get_schema_from_items(items) missing_id = IcebergSchema( *(field for field in iceberg_schema.fields if field.name != "id") ) @@ -145,7 +178,7 @@ def test_create_item_table_rejects_unknown_partition_source( with pytest.raises(ValueError): test_catalog.create_item_table( collection_id=test_collection_id, - iceberg_schema=get_schema_from_items(sample_stac_item), + iceberg_schema=get_schema_from_items(items_to_arrow([sample_stac_item])), partition_spec=PartitionSpec( PartitionField( source_id=9999, @@ -165,7 +198,7 @@ def test_create_item_table_rejects_unknown_sort_source( with pytest.raises(ValueError): test_catalog.create_item_table( collection_id=test_collection_id, - iceberg_schema=get_schema_from_items(sample_stac_item), + iceberg_schema=get_schema_from_items(items_to_arrow([sample_stac_item])), sort_order=SortOrder(SortField(source_id=9999)), ) @@ -175,15 +208,20 @@ def test_load_items_rejects_a_different_collection( test_collection_id: str, sample_stac_item: dict[str, Any], ) -> None: - iceberg_schema = get_schema_from_items(sample_stac_item) + items = items_to_arrow([sample_stac_item]) + iceberg_schema = get_schema_from_items(items) table = test_catalog.create_item_table( iceberg_schema=iceberg_schema, collection_id=test_collection_id, ) - sample_stac_item["collection"] = "different-collection" + different_collection = items.set_column( + items.schema.get_field_index("collection"), + "collection", + pyarrow.array(["different-collection"] * len(items)), + ) with pytest.raises(ValueError, match="different-collection"): - test_catalog.load_items(test_collection_id, sample_stac_item) + test_catalog.load_items(test_collection_id, different_collection) assert len(table.scan().to_arrow()) == 0 @@ -191,7 +229,7 @@ def test_load_items_rejects_a_different_collection( def test_load_items( test_catalog: IcestacCatalog, test_collection_id: str, - items: ItemsInput, + items: pyarrow.Table, ) -> None: expected_items = items_to_list(items) iceberg_schema = get_schema_from_items(items) @@ -219,8 +257,9 @@ def test_catalog_load_items_evolves_schema( test_collection_id: str, sample_stac_item: dict[str, Any], ) -> None: + initial = items_to_arrow([sample_stac_item]) table = test_catalog.create_item_table( - iceberg_schema=get_schema_from_items(sample_stac_item), + iceberg_schema=get_schema_from_items(initial), collection_id=test_collection_id, ) evolved_item = { @@ -230,7 +269,7 @@ def test_catalog_load_items_evolves_schema( test_catalog.load_items( collection_id=test_collection_id, - items=evolved_item, + items=items_to_arrow([evolved_item]), evolve_schema=True, ) diff --git a/tests/test_load.py b/tests/test_load.py index a7891d6..3757dbd 100644 --- a/tests/test_load.py +++ b/tests/test_load.py @@ -1,18 +1,24 @@ from copy import deepcopy from typing import Any +import pyarrow as pa import pytest +import rustac +from pyiceberg.exceptions import CommitFailedException from icestac.catalog import IcestacCatalog from icestac.load import Method, load_items -from icestac.schema import ItemsInput, get_schema_from_items -from tests.helpers import items_to_list +from icestac.schema import get_schema_from_items, prepare_arrow_table +from tests.helpers import items_to_arrow, items_to_list +@pytest.mark.filterwarnings( + "error:Delete operation did not match any records:UserWarning" +) def test_load_items_upsert_default( test_catalog: IcestacCatalog, test_collection_id: str, - items: ItemsInput, + items: pa.Table, ) -> None: """Test loading items with default upsert method.""" expected_items = items_to_list(items) @@ -37,7 +43,7 @@ def test_load_items_upsert_default( def test_load_items_upsert_explicit( test_catalog: IcestacCatalog, - items: ItemsInput, + items: pa.Table, ) -> None: """Test loading items with explicit upsert method.""" expected_items = items_to_list(items) @@ -59,7 +65,7 @@ def test_load_items_upsert_explicit( def test_load_items_upsert_updates_existing( test_catalog: IcestacCatalog, - items: ItemsInput, + items: pa.Table, ) -> None: """Test that upsert updates existing records with same ID.""" expected_items = items_to_list(items) @@ -83,7 +89,7 @@ def test_load_items_upsert_updates_existing( modified_items.append(modified_item) # Load modified items with upsert - load_items(modified_items, table, method="upsert") + load_items(items_to_arrow(modified_items), table, method="upsert") # Verify only the original record count exists and they have updated titles result = table.scan().to_arrow() @@ -94,9 +100,109 @@ def test_load_items_upsert_updates_existing( assert all(title.startswith("Updated") for title in titles) +def test_upsert_reconstructs_evolved_existing_item( + test_catalog: IcestacCatalog, + sample_stac_item: dict[str, Any], +) -> None: + """An existing item can gain a top-level property during replacement.""" + initial = items_to_arrow([sample_stac_item]) + table = test_catalog.create_item_table( + iceberg_schema=get_schema_from_items(initial), + collection_id=sample_stac_item["collection"], + ) + load_items(initial, table) + + replacement = deepcopy(sample_stac_item) + replacement["properties"]["processing:software"] = {"version": "1.0"} + load_items(items_to_arrow([replacement]), table, evolve_schema=True) + + reconstructed = rustac.from_arrow(table.scan().to_arrow())["features"] + assert len(reconstructed) == 1 + assert reconstructed[0]["id"] == replacement["id"] + assert reconstructed[0]["properties"] == replacement["properties"] + + +def test_upsert_reconstructs_nested_asset_and_property_evolution( + test_catalog: IcestacCatalog, + sample_stac_item: dict[str, Any], +) -> None: + """Nested asset fields and flattened properties evolve in one replacement.""" + initial = items_to_arrow([sample_stac_item]) + table = test_catalog.create_item_table( + iceberg_schema=get_schema_from_items(initial), + collection_id=sample_stac_item["collection"], + ) + load_items(initial, table) + + replacement = deepcopy(sample_stac_item) + replacement["properties"]["processing:software"] = {"version": "1.0"} + replacement["assets"]["data"]["roles"] = ["data"] + replacement["assets"]["thumbnail"] = { + "href": "https://example.com/thumbnail.jpg", + "type": "image/jpeg", + } + load_items(items_to_arrow([replacement]), table, evolve_schema=True) + + reconstructed = rustac.from_arrow(table.scan().to_arrow())["features"][0] + assert reconstructed["properties"]["processing:software"] == {"version": "1.0"} + assert reconstructed["assets"] == replacement["assets"] + + +def test_upsert_clears_omitted_optional_fields( + test_catalog: IcestacCatalog, + sample_stac_item: dict[str, Any], +) -> None: + """Omitted optional values are cleared instead of being retained.""" + initial = items_to_arrow([sample_stac_item]) + table = test_catalog.create_item_table( + iceberg_schema=get_schema_from_items(initial), + collection_id=sample_stac_item["collection"], + ) + load_items(initial, table) + + replacement = deepcopy(sample_stac_item) + replacement["properties"] = {"datetime": sample_stac_item["properties"]["datetime"]} + replacement["assets"]["data"] = {"href": "https://example.com/replacement.tif"} + load_items(items_to_arrow([replacement]), table) + + reconstructed = rustac.from_arrow(table.scan().to_arrow())["features"][0] + assert "title" not in reconstructed["properties"] + assert reconstructed["assets"] == { + "data": {"href": replacement["assets"]["data"]["href"]} + } + + +def test_competing_upserts_fail_without_duplicate_ids( + test_catalog: IcestacCatalog, + sample_stac_item: dict[str, Any], +) -> None: + """A stale competing upsert fails instead of committing a duplicate.""" + initial = items_to_arrow([sample_stac_item]) + table = test_catalog.create_item_table( + iceberg_schema=get_schema_from_items(initial), + collection_id=sample_stac_item["collection"], + ) + first = test_catalog.catalog.load_table(table.name()) + second = test_catalog.catalog.load_table(table.name()) + arrow_table = prepare_arrow_table(items_to_arrow([sample_stac_item])) + + first_transaction = first.transaction() + second_transaction = second.transaction() + first_transaction.upsert(df=arrow_table, join_cols=["id"]) + second_transaction.upsert(df=arrow_table, join_cols=["id"]) + + first_transaction.commit_transaction() + with pytest.raises(CommitFailedException): + second_transaction.commit_transaction() + + second.refresh() + result = second.scan().to_arrow() + assert result.column("id").to_pylist() == [sample_stac_item["id"]] + + def test_load_items_append( test_catalog: IcestacCatalog, - items: ItemsInput, + items: pa.Table, ) -> None: """Test loading items with append method.""" expected_items = items_to_list(items) @@ -118,7 +224,7 @@ def test_load_items_append( def test_load_items_append_creates_duplicates( test_catalog: IcestacCatalog, - items: ItemsInput, + items: pa.Table, ) -> None: """Test that append creates duplicate records when IDs overlap.""" expected_items = items_to_list(items) @@ -141,7 +247,7 @@ def test_load_items_append_creates_duplicates( def test_load_items_multiple_batches( test_catalog: IcestacCatalog, - items: ItemsInput, + items: pa.Table, ) -> None: """Test loading items in multiple batches with different methods.""" expected_items = items_to_list(items) @@ -154,14 +260,14 @@ def test_load_items_multiple_batches( ) # Load first batch - load_items(expected_items[:2], table, method="upsert") + load_items(items_to_arrow(expected_items[:2]), table, method="upsert") result = table.scan().to_arrow() assert len(result) == min(2, len(expected_items)) # Load second batch if len(expected_items) > 2: - load_items(expected_items[2:], table, method="upsert") + load_items(items_to_arrow(expected_items[2:]), table, method="upsert") result = table.scan().to_arrow() assert len(result) == len(expected_items) @@ -171,14 +277,15 @@ def test_load_items_rejects_invalid_method( test_catalog: IcestacCatalog, sample_stac_item: dict[str, Any], ) -> None: - iceberg_schema = get_schema_from_items(sample_stac_item) + initial = items_to_arrow([sample_stac_item]) + iceberg_schema = get_schema_from_items(initial) table = test_catalog.create_item_table( iceberg_schema=iceberg_schema, collection_id=sample_stac_item["collection"], ) with pytest.raises(ValueError, match="Unsupported load method"): - load_items(sample_stac_item, table, method="insert") # type: ignore[arg-type] + load_items(initial, table, method="insert") # type: ignore[arg-type] def test_load_items_supports_interval_datetime( @@ -191,13 +298,14 @@ def test_load_items_supports_interval_datetime( "start_datetime": "2024-01-01T00:00:00Z", "end_datetime": "2024-01-02T00:00:00Z", } - iceberg_schema = get_schema_from_items(interval_item) + interval_items = items_to_arrow([interval_item]) + iceberg_schema = get_schema_from_items(interval_items) table = test_catalog.create_item_table( iceberg_schema=iceberg_schema, collection_id=interval_item["collection"], ) - load_items(interval_item, table) + load_items(interval_items, table) assert table.scan().to_arrow().column("id").to_pylist() == [interval_item["id"]] @@ -206,17 +314,18 @@ def test_load_items_different_schema( test_catalog: IcestacCatalog, sample_stac_item: dict[str, Any], ) -> None: - iceberg_schema = get_schema_from_items(sample_stac_item) + initial = items_to_arrow([sample_stac_item]) + iceberg_schema = get_schema_from_items(initial) table = test_catalog.create_item_table( iceberg_schema=iceberg_schema, collection_id=sample_stac_item["collection"], ) - load_items([sample_stac_item], table) + load_items(initial, table) item_new_schema = deepcopy(sample_stac_item) item_new_schema["properties"]["new_field"] = True with pytest.raises(ValueError, match="Update the schema first"): - load_items([item_new_schema], table) + load_items(items_to_arrow([item_new_schema]), table) @pytest.mark.parametrize("method", ["append", "upsert"]) @@ -225,38 +334,219 @@ def test_load_items_evolves_schema( sample_stac_item: dict[str, Any], method: Method, ) -> None: + initial = items_to_arrow([sample_stac_item]) table = test_catalog.create_item_table( - iceberg_schema=get_schema_from_items(sample_stac_item), + iceberg_schema=get_schema_from_items(initial), collection_id=sample_stac_item["collection"], ) - load_items(sample_stac_item, table) + load_items(initial, table) evolved_item = deepcopy(sample_stac_item) evolved_item["id"] = "evolved-item" evolved_item["properties"]["processing:software"] = { "Atmospheric Correction": "6.0" } - load_items(evolved_item, table, method=method, evolve_schema=True) + load_items(items_to_arrow([evolved_item]), table, method=method, evolve_schema=True) table.refresh() assert table.schema().find_field("processing:software.Atmospheric Correction") assert len(table.scan().to_arrow()) == 2 +def test_populated_link_fields_survive_reconstruction( + test_catalog: IcestacCatalog, + sample_stac_item: dict[str, Any], +) -> None: + item = deepcopy(sample_stac_item) + item["links"] = [ + { + "href": "https://example.com/query", + "rel": "data", + "type": "application/json", + "title": "Query", + "method": "POST", + "headers": {"content-type": "application/json"}, + "body": {"limit": 1}, + "merge": True, + } + ] + input_items = items_to_arrow([item]) + table = test_catalog.create_item_table( + iceberg_schema=get_schema_from_items(input_items), + collection_id=item["collection"], + ) + + load_items(input_items, table) + + reconstructed = rustac.from_arrow(table.scan().to_arrow())["features"][0] + assert reconstructed["links"] == item["links"] + + +def test_populated_links_require_evolution_after_empty_schema( + test_catalog: IcestacCatalog, + sample_stac_item: dict[str, Any], +) -> None: + empty_links = deepcopy(sample_stac_item) + empty_links["links"] = [] + empty_links_table = items_to_arrow([empty_links]) + table = test_catalog.create_item_table( + iceberg_schema=get_schema_from_items(empty_links_table), + collection_id=empty_links["collection"], + ) + load_items(empty_links_table, table) + + populated = deepcopy(sample_stac_item) + populated["id"] = "populated-links" + populated["links"] = [ + { + "href": "https://example.com/query", + "rel": "data", + "method": "POST", + "headers": {"content-type": "application/json"}, + "body": {"limit": 1}, + "merge": True, + } + ] + populated_table = items_to_arrow([populated]) + with pytest.raises(ValueError, match="Update the schema first"): + load_items(populated_table, table) + + load_items(populated_table, table, evolve_schema=True) + reconstructed = rustac.from_arrow(table.scan().to_arrow())["features"] + assert ( + next(item for item in reconstructed if item["id"] == populated["id"])["links"] + == populated["links"] + ) + + +def test_arrow_temporal_semantics_are_callers_responsibility( + test_catalog: IcestacCatalog, + sample_stac_item: dict[str, Any], +) -> None: + invalid = deepcopy(sample_stac_item) + invalid["properties"] = { + "datetime": None, + "start_datetime": "2024-01-02T00:00:00Z", + "end_datetime": "2024-01-01T00:00:00Z", + } + arrow_items = items_to_arrow([invalid]) + table = test_catalog.create_item_table( + iceberg_schema=get_schema_from_items(arrow_items), + collection_id=sample_stac_item["collection"], + ) + + load_items(arrow_items, table) + + assert table.scan().to_arrow().column("id").to_pylist() == [sample_stac_item["id"]] + + +def test_arrow_structural_edge_cases_follow_arrow_schema( + sample_stac_item: dict[str, Any], +) -> None: + arrow = items_to_arrow([sample_stac_item]) + geometry_index = arrow.schema.get_field_index("geometry") + null_geometry = arrow.set_column( + geometry_index, + "geometry", + pa.array([None], type=pa.binary()), + ) + + assert get_schema_from_items(null_geometry).find_field("geometry") + + +def test_load_items_accepts_arro3_table( + test_catalog: IcestacCatalog, + sample_stac_item: dict[str, Any], +) -> None: + items = rustac.to_arrow([sample_stac_item]) + table = test_catalog.create_item_table( + iceberg_schema=get_schema_from_items(items), + collection_id=sample_stac_item["collection"], + ) + + load_items(items, table) + + assert table.scan().to_arrow().column("id").to_pylist() == [sample_stac_item["id"]] + + +@pytest.mark.parametrize("method", ["append", "upsert"]) +def test_load_items_accepts_dictionary_columns( + test_catalog: IcestacCatalog, + items: pa.Table, + method: Method, +) -> None: + """Dictionary-encoded columns can be written without manual re-encoding.""" + for name in ("id", "collection", "title"): + items = items.set_column( + items.schema.get_field_index(name), name, items[name].dictionary_encode() + ) + table = test_catalog.create_item_table( + iceberg_schema=get_schema_from_items(items), + collection_id=items["collection"][0].as_py(), + ) + + load_items(items, table, method=method) + + result = table.scan().to_arrow() + assert sorted(zip(result["id"].to_pylist(), result["title"].to_pylist())) == sorted( + zip(items["id"].to_pylist(), items["title"].to_pylist()) + ) + + +def test_load_items_rejects_non_arrow_inputs( + test_catalog: IcestacCatalog, + sample_stac_item: dict[str, Any], +) -> None: + initial = items_to_arrow([sample_stac_item]) + table = test_catalog.create_item_table( + iceberg_schema=get_schema_from_items(initial), + collection_id=sample_stac_item["collection"], + ) + + with pytest.raises(TypeError, match="pyarrow.Table"): + load_items(sample_stac_item, table) + + +def test_load_items_rejects_invalid_arrow_schema( + test_catalog: IcestacCatalog, + items: pa.Table, +) -> None: + table = test_catalog.create_item_table( + iceberg_schema=get_schema_from_items(items), + collection_id="test-collection", + ) + id_index = items.schema.get_field_index("id") + invalid = items.set_column( + id_index, + "id", + pa.array([1, 2, 3], type=pa.int64()), + ) + + with pytest.raises(ValueError, match="Unsupported types for STAC fields: id"): + load_items(invalid, table) + assert len(table.scan().to_arrow()) == 0 + + def test_load_items_does_not_evolve_schema_when_write_fails( test_catalog: IcestacCatalog, sample_stac_item: dict[str, Any], ) -> None: + initial = items_to_arrow([sample_stac_item]) table = test_catalog.create_item_table( - iceberg_schema=get_schema_from_items(sample_stac_item), + iceberg_schema=get_schema_from_items(initial), collection_id=sample_stac_item["collection"], ) + load_items(initial, table) + before = rustac.from_arrow(table.scan().to_arrow())["features"] + evolved_item = deepcopy(sample_stac_item) evolved_item["properties"]["new_field"] = True + evolved = items_to_arrow([evolved_item, evolved_item]) with pytest.raises(ValueError, match="Duplicate rows"): - load_items([evolved_item, evolved_item], table, evolve_schema=True) + load_items(evolved, table, evolve_schema=True) table.refresh() + assert rustac.from_arrow(table.scan().to_arrow())["features"] == before with pytest.raises(ValueError, match="Could not find field"): table.schema().find_field("new_field") diff --git a/tests/test_main.py b/tests/test_main.py new file mode 100644 index 0000000..2d560ef --- /dev/null +++ b/tests/test_main.py @@ -0,0 +1,94 @@ +import asyncio +from copy import deepcopy +from pathlib import Path + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest +from pyiceberg.table import TableProperties +from rustac import DuckdbClient, RustacError + +import main +from tests.helpers import items_to_arrow + + +@pytest.fixture +def duckdb_client(): + """Use the installed spatial extension without downloading during tests.""" + try: + return DuckdbClient(install_extensions=False) + except RustacError as exc: + pytest.skip(f"DuckDB extensions are not installed: {exc}") + + +def test_demo_loads_cached_batches_and_can_rerun( + tmp_path, monkeypatch, test_catalog, sample_stac_item, duckdb_client +) -> None: + """Load real Parquet through DuckDB and Iceberg, including schema evolution.""" + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(main, "load_catalog", lambda: test_catalog.catalog) + monkeypatch.setattr(main, "DuckdbClient", lambda: duckdb_client) + for month in range(1, 9): + batch = [] + for index, xmin in enumerate((120.0, -120.0, 0.0)): + item = deepcopy(sample_stac_item) + item["id"] = f"{month}-{index}" + item["collection"] = "HLSS30_2.0" + item["bbox"] = [xmin, 0.0, xmin + 1, 1.0] + item["properties"]["datetime"] = f"2026-{month:02d}-01T00:00:00Z" + if month >= 6: + item["properties"]["new_field"] = True + batch.append(item) + path = Path( + f"data/HLSS30_2.0/year=2026/month={month}/HLSS30_2.0-2026-{month}.parquet" + ) + path.parent.mkdir(parents=True) + pq.write_table(items_to_arrow(batch), path) + prepared = main.read_hls_items(duckdb_client, path, "HLSS30_2_0") + keys = prepared["hilbert_idx"].to_pylist() + assert keys == sorted(keys) + assert len(set(keys)) == 3 + assert pa.types.is_binary(prepared.schema.field("geometry").type) + assert "year" not in prepared.column_names + assert "month" not in prepared.column_names + + asyncio.run(main.run()) + asyncio.run(main.run()) + + table = test_catalog.catalog.load_table(("icestac", "HLSS30_2_0")) + result = table.scan().to_arrow() + assert len(result) == 24 + assert len(result["id"].unique()) == 24 + assert result["collection"].unique().to_pylist() == ["HLSS30_2_0"] + assert result["new_field"].null_count == 15 + assert table.properties[TableProperties.PARQUET_ROW_GROUP_LIMIT] == "50000" + assert ( + table.sort_order().fields[0].source_id + == table.schema().find_field("hilbert_idx").field_id + ) + + +@pytest.mark.parametrize("case", ["empty", "null_bbox", "nonfinite_bbox"]) +def test_read_hls_items_rejects_invalid_batches( + tmp_path, items, duckdb_client, case +) -> None: + """Reject unusable batches before creating or writing an Iceberg table.""" + if case == "empty": + items = items.slice(0, 0) + else: + bbox_type = items.schema.field("bbox").type + value = ( + None + if case == "null_bbox" + else {"xmin": float("nan"), "ymin": 0.0, "xmax": 1.0, "ymax": 1.0} + ) + items = items.set_column( + items.schema.get_field_index("bbox"), + "bbox", + pa.array([value] * len(items), type=bbox_type), + ) + path = tmp_path / "items.parquet" + pq.write_table(items, path) + + with pytest.raises(ValueError, match="No items found|bbox coordinates"): + main.read_hls_items(duckdb_client, path, "test-collection") diff --git a/tests/test_schema.py b/tests/test_schema.py index 50a12fa..6f7c205 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -1,69 +1,113 @@ -from typing import Any - import pyarrow as pa import pytest -from pydantic import ValidationError from pyiceberg.schema import Schema +from pyiceberg.types import ListType +import rustac from icestac.schema import IcestacItem, get_schema_from_items -def test_get_schema_from_items(sample_stac_item: dict[str, Any]) -> None: - """Test that we can derive an Iceberg schema from a STAC item.""" - schema = get_schema_from_items(sample_stac_item) +def test_get_schema_from_items(items: pa.Table) -> None: + """Derive an Iceberg schema from a flattened Arrow table.""" + schema = get_schema_from_items(items) assert isinstance(schema, Schema) assert schema.find_field("title").field_id > 0 assert schema.find_field("id").required assert str(schema.find_field("geometry").field_type) == "binary" + nested_paths = ( + "bbox.xmin", + "bbox.ymax", + "links.element.href", + "assets.data.href", + ) + nested_ids = [schema.find_field(path).field_id for path in nested_paths] + assert all(field_id > 0 for field_id in nested_ids) + assert len(nested_ids) == len(set(nested_ids)) + links_type = schema.find_field("links").field_type + assert isinstance(links_type, ListType) + assert links_type.element_id > 0 -def test_get_schema_from_items_validates(sample_stac_item: dict[str, Any]) -> None: - """Test that get_schema_from_items validates the STAC item.""" - invalid_item = {"not": "a stac item"} - - with pytest.raises(ValidationError): - get_schema_from_items(invalid_item) +def test_get_schema_from_items_rejects_empty_table(items: pa.Table) -> None: + with pytest.raises( + ValueError, match="Cannot infer or load a schema from an empty Arrow table" + ): + get_schema_from_items(items.slice(0, 0)) -def test_get_schema_from_items_no_collection(sample_stac_item: dict[str, Any]) -> None: - """Test that missing collection field raises ValueError.""" - _ = sample_stac_item.pop("collection") - with pytest.raises(ValidationError): - _ = get_schema_from_items(sample_stac_item) +def test_public_ingestion_rejects_dictionary_inputs(sample_stac_item) -> None: + for items in (sample_stac_item, [sample_stac_item]): + with pytest.raises(TypeError, match="pyarrow.Table or arro3.core.Table"): + get_schema_from_items(items) -def test_validate_schema_valid(sample_stac_item: dict[str, Any]) -> None: - """Test that a valid STAC schema passes validation.""" - schema = get_schema_from_items(sample_stac_item) +def test_public_ingestion_accepts_arro3_table(sample_stac_item) -> None: + items = rustac.to_arrow([sample_stac_item]) - # Should not raise - IcestacItem.validate_schema(schema) + assert get_schema_from_items(items).find_field("id") -def test_validate_prepared_schema_missing_required_field( - sample_stac_item: dict[str, Any], +def test_get_schema_from_items_accepts_semantically_unvalidated_arrow( + sample_stac_item, ) -> None: - """Test that required-field validation accepts prepared Iceberg schemas.""" - iceberg_schema = get_schema_from_items(sample_stac_item) - missing_id = type(iceberg_schema)( - *(field for field in iceberg_schema.fields if field.name != "id") - ) + invalid_temporal_item = { + **sample_stac_item, + "properties": {"datetime": None}, + } + items = pa.table(rustac.to_arrow([invalid_temporal_item])) + + assert get_schema_from_items(items).find_field("datetime") + +def test_get_schema_from_items_rejects_missing_required_field(items: pa.Table) -> None: with pytest.raises(ValueError, match="missing required STAC fields.*'id'"): - IcestacItem.validate_schema(missing_id) + get_schema_from_items(items.drop(["id"])) + + +def test_get_schema_from_items_rejects_invalid_field_type(items: pa.Table) -> None: + index = items.schema.get_field_index("id") + invalid = items.set_column(index, "id", pa.array([1, 2, 3], type=pa.int64())) + + with pytest.raises(ValueError, match="Unsupported types for STAC fields: id"): + get_schema_from_items(invalid) + + +def test_get_schema_from_items_rejects_null_required_field(items: pa.Table) -> None: + index = items.schema.get_field_index("id") + invalid = items.set_column( + index, "id", pa.array([None, None, None], type=pa.string()) + ) + + with pytest.raises(ValueError, match="Casting field 'id' with null values"): + get_schema_from_items(invalid) + + +def test_get_schema_from_items_rejects_null_links(items: pa.Table) -> None: + """Empty-link normalization must not replace null links with empty lists.""" + invalid = items.set_column( + items.schema.get_field_index("links"), + "links", + pa.array([[], None, []], type=pa.list_(pa.null())), + ) + + with pytest.raises(ValueError, match="Casting field 'links' with null values"): + get_schema_from_items(invalid) + + +def test_validate_schema_valid(items: pa.Table) -> None: + """A valid inferred schema passes structural validation.""" + IcestacItem.validate_schema(get_schema_from_items(items)) def test_validate_schema_missing_required_field() -> None: - """Test that a schema missing required fields raises ValueError.""" - # Create a schema missing the required 'id' field schema = pa.schema( [ ("type", pa.string()), - ("geometry", pa.string()), + ("geometry", pa.binary()), ("collection", pa.string()), - ("datetime", pa.string()), + ("datetime", pa.timestamp("ms")), ] ) @@ -71,21 +115,18 @@ def test_validate_schema_missing_required_field() -> None: IcestacItem.validate_schema(schema) -def test_validate_schema_missing_datetime() -> None: - """Test that a schema missing datetime field raises ValueError.""" - # Create a schema with all required fields except datetime +def test_validate_schema_rejects_invalid_field_type() -> None: schema = pa.schema( [ ("type", pa.string()), - ("id", pa.string()), - ("geometry", pa.string()), + ("id", pa.int64()), + ("geometry", pa.binary()), ("collection", pa.string()), - ("stac_version", pa.string()), - ("links", pa.string()), - ("assets", pa.string()), - ("bbox", pa.list_(pa.float64())), + ("datetime", pa.timestamp("ms")), + ("links", pa.list_(pa.string())), + ("assets", pa.struct([])), ] ) - with pytest.raises(ValueError, match="missing required STAC fields.*'datetime'"): + with pytest.raises(ValueError, match="Unsupported types for STAC fields: id"): IcestacItem.validate_schema(schema) diff --git a/uv.lock b/uv.lock index 6db6ae2..b7af234 100644 --- a/uv.lock +++ b/uv.lock @@ -396,18 +396,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/01/c9/97cc5aae1648dcb851958a3ddf73ccd7dbe5650d95203ecb4d7720b4cdbf/fsspec-2026.1.0-py3-none-any.whl", hash = "sha256:cb76aa913c2285a3b49bdd5fc55b1d7c708d7208126b60f2eb8194fe1b4cbdcc", size = 201838, upload-time = "2026-01-09T15:21:34.041Z" }, ] -[[package]] -name = "geojson-pydantic" -version = "2.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5c/52/961c8f7c51067f5d853a732cd4abc09b4d15c742384406dda8348b98071e/geojson_pydantic-2.1.0.tar.gz", hash = "sha256:78a52b2a7cd9c113bac4898a81ce00c146c7927dd2804f1c7e9fd05c2515073f", size = 9398, upload-time = "2025-10-08T13:31:12.474Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/13/18/8a9dca353e605b344408114f6b045b11d14082d19f4668b073259d3ed1a9/geojson_pydantic-2.1.0-py3-none-any.whl", hash = "sha256:f9091bed334ab9fbb1bef113674edc1212a3737f374a0b13b1aa493f57964c1d", size = 8819, upload-time = "2025-10-08T13:31:11.646Z" }, -] - [[package]] name = "greenlet" version = "3.3.1" @@ -463,7 +451,6 @@ dependencies = [ { name = "pyarrow" }, { name = "pyiceberg", extra = ["pyiceberg-core"] }, { name = "rustac", extra = ["arrow"] }, - { name = "stac-pydantic" }, ] [package.dev-dependencies] @@ -483,9 +470,8 @@ dev = [ [package.metadata] requires-dist = [ { name = "pyarrow", specifier = ">=23.0.0" }, - { name = "pyiceberg", extras = ["pyiceberg-core"], specifier = ">=0.10.0" }, + { name = "pyiceberg", extras = ["pyiceberg-core"], specifier = ">=0.12.0" }, { name = "rustac", extras = ["arrow"], specifier = ">=0.9.3" }, - { name = "stac-pydantic", specifier = ">=3.4.0" }, ] [package.metadata.requires-dev] @@ -967,7 +953,7 @@ wheels = [ [[package]] name = "pyiceberg" -version = "0.10.0" +version = "0.12.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cachetools" }, @@ -979,22 +965,40 @@ dependencies = [ { name = "pyroaring" }, { name = "requests" }, { name = "rich" }, - { name = "sortedcontainers" }, { name = "strictyaml" }, { name = "tenacity" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a3/0e/90e61c38504f4fbd5ed79631f85da7d5ea5e5bf997bdeaa65b28ebf04cab/pyiceberg-0.10.0.tar.gz", hash = "sha256:2525afa5e7e5fc4e72b291f8e1cc219e982d2bda5ff17e62cd05b8d91c4139f5", size = 842633, upload-time = "2025-09-11T14:59:34.044Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a9/62/b6f7bed760d0896958d046ca3c188fd15467c6502bcc2dc301ac0554c1ce/pyiceberg-0.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2c799c9149e06ef9ece22945d5c198ffc69f5c04b314b59a43c2d4c1bb9ade84", size = 591127, upload-time = "2025-09-11T14:59:08.72Z" }, - { url = "https://files.pythonhosted.org/packages/4c/b2/294c74e70c68744a8246924fee350095cc46f97f81d1e37125011d8e1bcb/pyiceberg-0.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a8c7070fe1262f50694b12241b5373ee89c8aededda82ef325cb14e5a95cc461", size = 587041, upload-time = "2025-09-11T14:59:10.643Z" }, - { url = "https://files.pythonhosted.org/packages/7a/2f/9a9f0a01f0dae2cefc024a2bd84a00ff2a5d8d952f37053c46523c1dd7a6/pyiceberg-0.10.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e0d1a4896f546b1e115ece4212dd02b383eeb3c7ff5c072624b15f531b776f36", size = 1135929, upload-time = "2025-09-11T14:59:12.164Z" }, - { url = "https://files.pythonhosted.org/packages/e1/c2/51deddeec916d44a04cc26053179b560ffceba72e4561b6cf58a64aea209/pyiceberg-0.10.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1b0ef2f1880dd7549cc54ccb1a25f61ad5329e079cba372b4c239b0012aecac6", size = 1131851, upload-time = "2025-09-11T14:59:13.792Z" }, - { url = "https://files.pythonhosted.org/packages/ba/cc/e9cf3fa56d67306ba29352d56152907a91ca29eabc1a30d3177cee0d1418/pyiceberg-0.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:2127c795e451b971bd3f55cbda2d2c8200182bec3476e590e4a3453e60efda3c", size = 583472, upload-time = "2025-09-11T14:59:15.173Z" }, - { url = "https://files.pythonhosted.org/packages/03/61/f5042dd09cb91deed908a39acd5012f1ac6910ddf84ada889751732f0df8/pyiceberg-0.10.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:64cad9d1db08192605875a872152cbcaca147ea486cfa94773fa5f4f65d78a23", size = 629281, upload-time = "2025-09-11T14:59:17.585Z" }, - { url = "https://files.pythonhosted.org/packages/8e/50/960f7239eedd4b1bab2a611f5e100fffc138549c1213760a57cd24a5bac1/pyiceberg-0.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3e12cf585318f0f48d31a77b4149e0e5b4c41e03a24aa8612e060f20ff41eb10", size = 623424, upload-time = "2025-09-11T14:59:19.045Z" }, - { url = "https://files.pythonhosted.org/packages/f5/2b/756a74c80db6edd82c8d3f23c3ae13e7d6620300b87ef792c2a4d3935b30/pyiceberg-0.10.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6979dd741cee263c1235595f71888c73365f2725697411027c4bd81046db3294", size = 1377048, upload-time = "2025-09-11T14:59:20.541Z" }, - { url = "https://files.pythonhosted.org/packages/bb/35/9c18cb4ddc7d371db63714abb2f5e8414bc7a4d63f474644a2aea2933fe6/pyiceberg-0.10.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:13fd03ec3da6eb4d3b55ff94b647946a7749bede5d743c75b39deaad26421200", size = 1369921, upload-time = "2025-09-11T14:59:22.134Z" }, - { url = "https://files.pythonhosted.org/packages/7b/b3/c012dc6b5bc3d0a84821936789c753f5c44aec619b64fbcf7f90038d172e/pyiceberg-0.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:33367c84bcb0a2fbbe54cbbfe062691ab93b91a2e3d319bb546ec5b9b45b6057", size = 617722, upload-time = "2025-09-11T14:59:23.67Z" }, + { name = "zstandard" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/85/08/bde71e0bbcf1a62c92d7fa457b508691596c65fa7e52c1982c78c461cd1c/pyiceberg-0.12.0.tar.gz", hash = "sha256:19f165d298054f9436108691098b60fa0fa99d0eff5fb884700c43b29334a39d", size = 1212830, upload-time = "2026-09-01T17:28:42.378Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/07/aab0770651e9dd70902ff170d7db3875caa2f53c0df1103035022721d54d/pyiceberg-0.12.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ae611854967d691f3158ae06407622072a9c442e29f5fba073ecbedb4d0b8b66", size = 566935, upload-time = "2026-09-01T17:28:06.564Z" }, + { url = "https://files.pythonhosted.org/packages/6a/11/c14909196946ab59677d8c814bd9d811691b62a794bb27895d406dd34119/pyiceberg-0.12.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:63929b5664512c67bb89603c5ca8c568a08b4a9922ec86675b339214d362ea81", size = 567919, upload-time = "2026-09-01T17:28:07.642Z" }, + { url = "https://files.pythonhosted.org/packages/88/05/33f6ef4315e6210dcd7e98dbd4c367390ca24b71820f782df0fb5df49aee/pyiceberg-0.12.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cd913759564d08444567690c27a187b408d06e6c82a17ba4e4f98aaf90031e86", size = 773938, upload-time = "2026-09-01T17:28:08.672Z" }, + { url = "https://files.pythonhosted.org/packages/f0/38/11579d59d3d91288e4b805d13642417ebadae3548c554b2474cb8229b616/pyiceberg-0.12.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2dec70218c8e81f1630b9d7184b760cd161eff936127745355ba94100a4e7380", size = 771959, upload-time = "2026-09-01T17:28:10.099Z" }, + { url = "https://files.pythonhosted.org/packages/84/77/17168e83f5216a57b74c1c64875f5469239d16ea1ad60902aed7a2fdc2d3/pyiceberg-0.12.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:29fb5032276aaac1ad01f98125622012067be075cc70053a0dbd0d61e4a6e8e5", size = 770200, upload-time = "2026-09-01T17:28:11.432Z" }, + { url = "https://files.pythonhosted.org/packages/25/0c/88fcbf58f0c659b7f714ceed56ffd42b26f85fafbf5eea61056d9c10e654/pyiceberg-0.12.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0b432f97a7774a211980f8d135c12a75221d0696204a0dc67f9b750a5c1a3332", size = 770216, upload-time = "2026-09-01T17:28:12.637Z" }, + { url = "https://files.pythonhosted.org/packages/60/80/5e3c42eb35c88ed5b73bae31e1fc0cca69f7c09a604aafc09bec1feab68d/pyiceberg-0.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:bb4d04f93c1b98c19c365dc2bc8115e8396a1b2e1eb99ec9e04ade262c82c565", size = 564293, upload-time = "2026-09-01T17:28:13.888Z" }, + { url = "https://files.pythonhosted.org/packages/b1/45/9fc0692dd081ab34e53731b0cf3eba0f948e5a54948840fa06a4ed4cbf9b/pyiceberg-0.12.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0079c44d065fc70df09deb03dc7297314c31b8d9708559c0bec2b8f851ffbf21", size = 567622, upload-time = "2026-09-01T17:28:15.199Z" }, + { url = "https://files.pythonhosted.org/packages/0e/67/b11334fe6af2a5729bfd30c0e03dee78887e9874a634888aa81d91dc82e9/pyiceberg-0.12.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:baf35f45ed5ee1a14c8db8264de31fb942b9d0d21913262f0d444d2df45e8176", size = 568214, upload-time = "2026-09-01T17:28:16.459Z" }, + { url = "https://files.pythonhosted.org/packages/1d/24/eccd190e358514e7e0d9a5c7591e44e71eb0e2ab5cff6cb90adb9ecaa963/pyiceberg-0.12.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b20c36d08b4b12da572b10c20594be78eee7e51845040443c45231d2ad57c28a", size = 779432, upload-time = "2026-09-01T17:28:17.775Z" }, + { url = "https://files.pythonhosted.org/packages/ed/75/046692b5ae4330d251974a428fdd82c9d7840715580d332c7cbcd65a13db/pyiceberg-0.12.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:17dca4377c76b7b047a2e807f009595b8f0b090c792a84ecb427c2fa683a96e2", size = 781395, upload-time = "2026-09-01T17:28:18.906Z" }, + { url = "https://files.pythonhosted.org/packages/5f/38/f8b0dc8cbc53459c6780a3ebf78382960c78fd80db54d0eeb3f7a63c9c7f/pyiceberg-0.12.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5902178a7d46bc4b783a4026c474178a8cb4c9c413b47cdb2e520df4082ed255", size = 773096, upload-time = "2026-09-01T17:28:20.236Z" }, + { url = "https://files.pythonhosted.org/packages/d9/4e/ef4265f3b7108d1591ff233edf3ef6ca50d2639026b4a1cb62ddda3398bd/pyiceberg-0.12.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:cbb9f060d170b1b072e2ba2e821f5c76302e5731452f37fa63e104334f3feba1", size = 778402, upload-time = "2026-09-01T17:28:21.79Z" }, + { url = "https://files.pythonhosted.org/packages/f9/2c/858c329a4e93897568ab73e95892dde0bc32bf4a76a6133096ef6b660854/pyiceberg-0.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:0fcce46f5633491b50ebf8ed94fbf5c8d3b6bed76902cf28361368e9169e16f2", size = 564568, upload-time = "2026-09-01T17:28:23.048Z" }, + { url = "https://files.pythonhosted.org/packages/44/7d/c04a65b08ba272bfcbb31638222d71a9f1c7f12c0d8b659530e14817afaf/pyiceberg-0.12.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:498763380220a1d8881d52c218318e884be28c3ea5824cbbcb22a042c12a9ad3", size = 567243, upload-time = "2026-09-01T17:28:24.149Z" }, + { url = "https://files.pythonhosted.org/packages/7d/bc/73277e56a30234afed4405bbe874a5410dbf9fa6c22c2352cc2615f7ed78/pyiceberg-0.12.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:307e46f46ffd48e0b270acf10bc5892f09e8c9fa2c828e9ffbf32ad480504bae", size = 567739, upload-time = "2026-09-01T17:28:25.376Z" }, + { url = "https://files.pythonhosted.org/packages/a2/72/8e09e90fd556af1ea90b993287da7997423a99784f4b7ccc77bd96a28f39/pyiceberg-0.12.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e055cc459d7b6eba21bd62eedbda0d0845ade0c2161a04c0f8eb252ec3e2d7d1", size = 773284, upload-time = "2026-09-01T17:28:26.452Z" }, + { url = "https://files.pythonhosted.org/packages/83/f1/cb542e8a46690d9cd2112eafd052f7bfdcff33f2027a344126d07b5b681a/pyiceberg-0.12.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:893e35df750644dab19bb873522d15914335e22368a711703176ea187c26b655", size = 776034, upload-time = "2026-09-01T17:28:27.823Z" }, + { url = "https://files.pythonhosted.org/packages/52/62/9e41c64c9bd741da75b408379ce3175af9f2dbf453c0a2bd9e5bd404e068/pyiceberg-0.12.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f53afc4ae649d35d43eee2b32515f22bc64fa0d4921c359ed9e7744ca1101ae0", size = 768107, upload-time = "2026-09-01T17:28:28.94Z" }, + { url = "https://files.pythonhosted.org/packages/9a/39/18af56141c920e62dcd4dc4f91aa058c7361e8f2e8dd45f73cf3f0c25b64/pyiceberg-0.12.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fd57f73a55dc9439f183e13aa1d9c79ac4274564bdd412da735314e4e26b77ff", size = 774220, upload-time = "2026-09-01T17:28:30.094Z" }, + { url = "https://files.pythonhosted.org/packages/13/98/50a2a45e451df14ce0da864b1fd869a950c9cea0877800baa0ff287c5991/pyiceberg-0.12.0-cp313-cp313-win_amd64.whl", hash = "sha256:934c30733c3debf9b13bbcdb85c4cfdaf4b72a808f7af1dd027a38e4e2baef07", size = 563797, upload-time = "2026-09-01T17:28:31.388Z" }, + { url = "https://files.pythonhosted.org/packages/19/e3/49ca88aff0dd74560acdcbf1d23d33a1e4e3a2d6d17f03399e7401faf084/pyiceberg-0.12.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7bb480f6ac06e4afb3de1e1a46781d071e3f1cfcc5802de9e48e6289b3546c0a", size = 567458, upload-time = "2026-09-01T17:28:32.72Z" }, + { url = "https://files.pythonhosted.org/packages/5e/90/18a84508ec3bae4e90b5f631a4f4ad7de477729b8fa90014c46550b8e1e6/pyiceberg-0.12.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:16f2f03c20d01ced43198aedfffb620163e7debc21d33ff02cc18e107aa43e69", size = 568050, upload-time = "2026-09-01T17:28:33.963Z" }, + { url = "https://files.pythonhosted.org/packages/54/a4/e3281a6e98645c179652c7a9c04bdc3fb2d388f26a75dbf3225891804a11/pyiceberg-0.12.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:115ecdabd4c47d64b4eda1a4271a4c4eea0514411832f11adc7cdf6630e57a66", size = 772948, upload-time = "2026-09-01T17:28:35.211Z" }, + { url = "https://files.pythonhosted.org/packages/70/d7/0b4fcd024b938dceeb941626ef465cc1d17d023ca461db58a636701b1482/pyiceberg-0.12.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eb3db80cd510ddc34246059f4631661aa5b5d94c36b131499c56e746f22db3c5", size = 774232, upload-time = "2026-09-01T17:28:36.297Z" }, + { url = "https://files.pythonhosted.org/packages/97/bf/2c2d14235fcb1a28eba3e576e3472b77ee9408b52b13a2d33937a6189ea6/pyiceberg-0.12.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:82e0250f6c9baa11644efa44da5f968205c158c75126deb1b1730ce158cb3f76", size = 769045, upload-time = "2026-09-01T17:28:37.858Z" }, + { url = "https://files.pythonhosted.org/packages/ed/fb/c16e5aa6810a1e2dc0907b7a7d4605701b9da537d4175736cbec0c86ffb5/pyiceberg-0.12.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e73a396826d745b2a6ac0d3b5151c8cdcbb090e81bfc31f07139cb463e62dbe3", size = 772590, upload-time = "2026-09-01T17:28:39.132Z" }, + { url = "https://files.pythonhosted.org/packages/a4/57/2ae640b6220a321958484162cfbdd1a2772198bac0c4588a990b8afd22d5/pyiceberg-0.12.0-cp314-cp314-win_amd64.whl", hash = "sha256:ee7b572d209a39224a093661f462f95551b2bd3982a07aecd80c7dd560be15ea", size = 561552, upload-time = "2026-09-01T17:28:41.156Z" }, ] [package.optional-dependencies] @@ -1004,15 +1008,15 @@ pyiceberg-core = [ [[package]] name = "pyiceberg-core" -version = "0.6.0" +version = "0.10.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/60/80/06bd9159cacd80797122a88d65e8d3377fb76f00f1b23eeefe4ab0d85f4d/pyiceberg_core-0.6.0.tar.gz", hash = "sha256:ce2cac8cf8a85da6e682cec032165fcf387256257971f0f84bc6d50c0941f261", size = 457209, upload-time = "2025-07-30T09:20:23.447Z" } +sdist = { url = "https://files.pythonhosted.org/packages/64/0a/fa73e70a8af2c600fa8089a009c9be99587f4f62a1dd674acbb15f5dab91/pyiceberg_core-0.10.1.tar.gz", hash = "sha256:c5e600728071032a4027c4c36680e4806c98f443057a26523532a2f830db4c89", size = 883913, upload-time = "2026-08-01T18:34:38.955Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f6/8b/7ff908f6f18bc3d6351d9f4334d6a299eb7f1975b0bacb061d73b3292c1c/pyiceberg_core-0.6.0-cp39-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:2f228a54a2a69912378be18f98ea866bb4a08d265c875856f99cd81f2f7299ba", size = 55132736, upload-time = "2025-07-30T09:20:07.91Z" }, - { url = "https://files.pythonhosted.org/packages/57/09/ee720f4811fd4323a45d9d7bebcfd5d99283cf45092bccea87787a06bdff/pyiceberg_core-0.6.0-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:edb41a1f182774085b11352a1f44955d561e21453f00973021244471873fbbd7", size = 30041729, upload-time = "2025-07-30T09:20:13.056Z" }, - { url = "https://files.pythonhosted.org/packages/37/cd/94095aa2282ebe716e0a12130760b51076b1c921285574b1f88e5f63e234/pyiceberg_core-0.6.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5cf869d225d57254a54bc3778841cffea4193319bc0a849767a15e05e75c9b36", size = 30566511, upload-time = "2025-07-30T09:20:15.596Z" }, - { url = "https://files.pythonhosted.org/packages/87/62/7971cc8b090e51448da8d59e411be7b752a3a2abb1365e760871f27611e7/pyiceberg_core-0.6.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:18c12fe1ac5b4725b673cf0d1d0ab3e9475644ac0dae871a2e9a293c2622f0a8", size = 29570254, upload-time = "2025-07-30T09:20:18.371Z" }, - { url = "https://files.pythonhosted.org/packages/29/40/96bd273520075ee10718eeb609e92d44ee0b7701b5c225eae505a38fb22d/pyiceberg_core-0.6.0-cp39-abi3-win_amd64.whl", hash = "sha256:d3249eeae5e1d1f1d2c8bd8d6eced98da002afa7c48c751cb22d8dbd4b091a1e", size = 25815921, upload-time = "2025-07-30T09:20:20.781Z" }, + { url = "https://files.pythonhosted.org/packages/b6/8e/a22c7b1798023bc2a2bcdbe12930d06509be034ad7ec448cfdf5308fe3a7/pyiceberg_core-0.10.1-cp310-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:5ae7490fa3d03d32eab6e15116ddbeb0899cb3ba8af9332e302f2f3ca8e7667c", size = 24865938, upload-time = "2026-08-01T18:34:23.329Z" }, + { url = "https://files.pythonhosted.org/packages/3c/5a/f97796aff09011e0d91f6e8d2715933159547b36711c00f55c179214a4d2/pyiceberg_core-0.10.1-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bb6e7188243e1cf34d3897d6642078b3cc170935bb1955bc1c5096dd32f6719a", size = 11696981, upload-time = "2026-08-01T18:34:26.734Z" }, + { url = "https://files.pythonhosted.org/packages/76/72/7a259abb1b3bfee4216c9e307c6b5f96850a45bd88b4be58a46143dfc051/pyiceberg_core-0.10.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:413bb2e699d1957c98302c1bce5a0bf36fc5acf11797c48de5c08067df7b27b4", size = 14046735, upload-time = "2026-08-01T18:34:29.802Z" }, + { url = "https://files.pythonhosted.org/packages/82/a1/b4ffa500ea9681ebb1db173b4bd6c0bfdd8707e7ac6d5b7c0a9a49ddfa2f/pyiceberg_core-0.10.1-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a8974a5c93282455ed023e28f5291bb899b91731631d93f0ccf0411ad32efb71", size = 14502489, upload-time = "2026-08-01T18:34:32.808Z" }, + { url = "https://files.pythonhosted.org/packages/1a/c1/f0fdd495b8312b295726135c16a84e3b63a0e21977af465401a8d6f10925/pyiceberg_core-0.10.1-cp310-abi3-win_amd64.whl", hash = "sha256:884969c030be824d5ce7998d96215741d0e34351cbe995df6a156947b1eb7472", size = 13293439, upload-time = "2026-08-01T18:34:36.634Z" }, ] [[package]] @@ -1271,15 +1275,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] -[[package]] -name = "sortedcontainers" -version = "2.4.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, -] - [[package]] name = "sqlalchemy" version = "2.0.46" @@ -1329,20 +1324,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fc/a1/9c4efa03300926601c19c18582531b45aededfb961ab3c3585f1e24f120b/sqlalchemy-2.0.46-py3-none-any.whl", hash = "sha256:f9c11766e7e7c0a2767dda5acb006a118640c9fc0a4104214b96269bfb78399e", size = 1937882, upload-time = "2026-01-21T18:22:10.456Z" }, ] -[[package]] -name = "stac-pydantic" -version = "3.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "geojson-pydantic" }, - { name = "pydantic" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/30/54/95586c73f097df47340dffbe19ae3db4eda832af997e834c9396d4bdcc83/stac_pydantic-3.4.0.tar.gz", hash = "sha256:5e7a45d38df18c4148fe45469447288a5b2eb15b10737608da4fba3dccc50683", size = 22943, upload-time = "2025-07-17T11:17:28.83Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/27/c7/8ee49430a0a745559dab4205ad9f946a264e793061fe5d1456a4b7cd2f27/stac_pydantic-3.4.0-py3-none-any.whl", hash = "sha256:ce2e7b377db078abbb164f378e18d54b53cda2953e44b643cdfa3adc831ca1c8", size = 24851, upload-time = "2025-07-17T11:17:27.966Z" }, -] - [[package]] name = "strictyaml" version = "1.7.3" @@ -1495,3 +1476,77 @@ sdist = { url = "https://files.pythonhosted.org/packages/ce/4f/d6a5ff3b020c801c8 wheels = [ { url = "https://files.pythonhosted.org/packages/29/d1/3f62e4f9577b28c352c11623a03fb916096d5c131303d4861b4914481b6b/virtualenv-21.0.0-py3-none-any.whl", hash = "sha256:d44e70637402c7f4b10f48491c02a6397a3a187152a70cba0b6bc7642d69fb05", size = 5817167, upload-time = "2026-02-25T20:21:05.476Z" }, ] + +[[package]] +name = "zstandard" +version = "0.25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/aa/3e0508d5a5dd96529cdc5a97011299056e14c6505b678fd58938792794b1/zstandard-0.25.0.tar.gz", hash = "sha256:7713e1179d162cf5c7906da876ec2ccb9c3a9dcbdffef0cc7f70c3667a205f0b", size = 711513, upload-time = "2025-09-14T22:15:54.002Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/83/c3ca27c363d104980f1c9cee1101cc8ba724ac8c28a033ede6aab89585b1/zstandard-0.25.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:933b65d7680ea337180733cf9e87293cc5500cc0eb3fc8769f4d3c88d724ec5c", size = 795254, upload-time = "2025-09-14T22:16:26.137Z" }, + { url = "https://files.pythonhosted.org/packages/ac/4d/e66465c5411a7cf4866aeadc7d108081d8ceba9bc7abe6b14aa21c671ec3/zstandard-0.25.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a3f79487c687b1fc69f19e487cd949bf3aae653d181dfb5fde3bf6d18894706f", size = 640559, upload-time = "2025-09-14T22:16:27.973Z" }, + { url = "https://files.pythonhosted.org/packages/12/56/354fe655905f290d3b147b33fe946b0f27e791e4b50a5f004c802cb3eb7b/zstandard-0.25.0-cp311-cp311-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:0bbc9a0c65ce0eea3c34a691e3c4b6889f5f3909ba4822ab385fab9057099431", size = 5348020, upload-time = "2025-09-14T22:16:29.523Z" }, + { url = "https://files.pythonhosted.org/packages/3b/13/2b7ed68bd85e69a2069bcc72141d378f22cae5a0f3b353a2c8f50ef30c1b/zstandard-0.25.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:01582723b3ccd6939ab7b3a78622c573799d5d8737b534b86d0e06ac18dbde4a", size = 5058126, upload-time = "2025-09-14T22:16:31.811Z" }, + { url = "https://files.pythonhosted.org/packages/c9/dd/fdaf0674f4b10d92cb120ccff58bbb6626bf8368f00ebfd2a41ba4a0dc99/zstandard-0.25.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:5f1ad7bf88535edcf30038f6919abe087f606f62c00a87d7e33e7fc57cb69fcc", size = 5405390, upload-time = "2025-09-14T22:16:33.486Z" }, + { url = "https://files.pythonhosted.org/packages/0f/67/354d1555575bc2490435f90d67ca4dd65238ff2f119f30f72d5cde09c2ad/zstandard-0.25.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:06acb75eebeedb77b69048031282737717a63e71e4ae3f77cc0c3b9508320df6", size = 5452914, upload-time = "2025-09-14T22:16:35.277Z" }, + { url = "https://files.pythonhosted.org/packages/bb/1f/e9cfd801a3f9190bf3e759c422bbfd2247db9d7f3d54a56ecde70137791a/zstandard-0.25.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9300d02ea7c6506f00e627e287e0492a5eb0371ec1670ae852fefffa6164b072", size = 5559635, upload-time = "2025-09-14T22:16:37.141Z" }, + { url = "https://files.pythonhosted.org/packages/21/88/5ba550f797ca953a52d708c8e4f380959e7e3280af029e38fbf47b55916e/zstandard-0.25.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bfd06b1c5584b657a2892a6014c2f4c20e0db0208c159148fa78c65f7e0b0277", size = 5048277, upload-time = "2025-09-14T22:16:38.807Z" }, + { url = "https://files.pythonhosted.org/packages/46/c0/ca3e533b4fa03112facbe7fbe7779cb1ebec215688e5df576fe5429172e0/zstandard-0.25.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f373da2c1757bb7f1acaf09369cdc1d51d84131e50d5fa9863982fd626466313", size = 5574377, upload-time = "2025-09-14T22:16:40.523Z" }, + { url = "https://files.pythonhosted.org/packages/12/9b/3fb626390113f272abd0799fd677ea33d5fc3ec185e62e6be534493c4b60/zstandard-0.25.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6c0e5a65158a7946e7a7affa6418878ef97ab66636f13353b8502d7ea03c8097", size = 4961493, upload-time = "2025-09-14T22:16:43.3Z" }, + { url = "https://files.pythonhosted.org/packages/cb/d3/23094a6b6a4b1343b27ae68249daa17ae0651fcfec9ed4de09d14b940285/zstandard-0.25.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:c8e167d5adf59476fa3e37bee730890e389410c354771a62e3c076c86f9f7778", size = 5269018, upload-time = "2025-09-14T22:16:45.292Z" }, + { url = "https://files.pythonhosted.org/packages/8c/a7/bb5a0c1c0f3f4b5e9d5b55198e39de91e04ba7c205cc46fcb0f95f0383c1/zstandard-0.25.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:98750a309eb2f020da61e727de7d7ba3c57c97cf6213f6f6277bb7fb42a8e065", size = 5443672, upload-time = "2025-09-14T22:16:47.076Z" }, + { url = "https://files.pythonhosted.org/packages/27/22/503347aa08d073993f25109c36c8d9f029c7d5949198050962cb568dfa5e/zstandard-0.25.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:22a086cff1b6ceca18a8dd6096ec631e430e93a8e70a9ca5efa7561a00f826fa", size = 5822753, upload-time = "2025-09-14T22:16:49.316Z" }, + { url = "https://files.pythonhosted.org/packages/e2/be/94267dc6ee64f0f8ba2b2ae7c7a2df934a816baaa7291db9e1aa77394c3c/zstandard-0.25.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:72d35d7aa0bba323965da807a462b0966c91608ef3a48ba761678cb20ce5d8b7", size = 5366047, upload-time = "2025-09-14T22:16:51.328Z" }, + { url = "https://files.pythonhosted.org/packages/7b/a3/732893eab0a3a7aecff8b99052fecf9f605cf0fb5fb6d0290e36beee47a4/zstandard-0.25.0-cp311-cp311-win32.whl", hash = "sha256:f5aeea11ded7320a84dcdd62a3d95b5186834224a9e55b92ccae35d21a8b63d4", size = 436484, upload-time = "2025-09-14T22:16:55.005Z" }, + { url = "https://files.pythonhosted.org/packages/43/a3/c6155f5c1cce691cb80dfd38627046e50af3ee9ddc5d0b45b9b063bfb8c9/zstandard-0.25.0-cp311-cp311-win_amd64.whl", hash = "sha256:daab68faadb847063d0c56f361a289c4f268706b598afbf9ad113cbe5c38b6b2", size = 506183, upload-time = "2025-09-14T22:16:52.753Z" }, + { url = "https://files.pythonhosted.org/packages/8c/3e/8945ab86a0820cc0e0cdbf38086a92868a9172020fdab8a03ac19662b0e5/zstandard-0.25.0-cp311-cp311-win_arm64.whl", hash = "sha256:22a06c5df3751bb7dc67406f5374734ccee8ed37fc5981bf1ad7041831fa1137", size = 462533, upload-time = "2025-09-14T22:16:53.878Z" }, + { url = "https://files.pythonhosted.org/packages/82/fc/f26eb6ef91ae723a03e16eddb198abcfce2bc5a42e224d44cc8b6765e57e/zstandard-0.25.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7b3c3a3ab9daa3eed242d6ecceead93aebbb8f5f84318d82cee643e019c4b73b", size = 795738, upload-time = "2025-09-14T22:16:56.237Z" }, + { url = "https://files.pythonhosted.org/packages/aa/1c/d920d64b22f8dd028a8b90e2d756e431a5d86194caa78e3819c7bf53b4b3/zstandard-0.25.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:913cbd31a400febff93b564a23e17c3ed2d56c064006f54efec210d586171c00", size = 640436, upload-time = "2025-09-14T22:16:57.774Z" }, + { url = "https://files.pythonhosted.org/packages/53/6c/288c3f0bd9fcfe9ca41e2c2fbfd17b2097f6af57b62a81161941f09afa76/zstandard-0.25.0-cp312-cp312-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:011d388c76b11a0c165374ce660ce2c8efa8e5d87f34996aa80f9c0816698b64", size = 5343019, upload-time = "2025-09-14T22:16:59.302Z" }, + { url = "https://files.pythonhosted.org/packages/1e/15/efef5a2f204a64bdb5571e6161d49f7ef0fffdbca953a615efbec045f60f/zstandard-0.25.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dffecc361d079bb48d7caef5d673c88c8988d3d33fb74ab95b7ee6da42652ea", size = 5063012, upload-time = "2025-09-14T22:17:01.156Z" }, + { url = "https://files.pythonhosted.org/packages/b7/37/a6ce629ffdb43959e92e87ebdaeebb5ac81c944b6a75c9c47e300f85abdf/zstandard-0.25.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7149623bba7fdf7e7f24312953bcf73cae103db8cae49f8154dd1eadc8a29ecb", size = 5394148, upload-time = "2025-09-14T22:17:03.091Z" }, + { url = "https://files.pythonhosted.org/packages/e3/79/2bf870b3abeb5c070fe2d670a5a8d1057a8270f125ef7676d29ea900f496/zstandard-0.25.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:6a573a35693e03cf1d67799fd01b50ff578515a8aeadd4595d2a7fa9f3ec002a", size = 5451652, upload-time = "2025-09-14T22:17:04.979Z" }, + { url = "https://files.pythonhosted.org/packages/53/60/7be26e610767316c028a2cbedb9a3beabdbe33e2182c373f71a1c0b88f36/zstandard-0.25.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5a56ba0db2d244117ed744dfa8f6f5b366e14148e00de44723413b2f3938a902", size = 5546993, upload-time = "2025-09-14T22:17:06.781Z" }, + { url = "https://files.pythonhosted.org/packages/85/c7/3483ad9ff0662623f3648479b0380d2de5510abf00990468c286c6b04017/zstandard-0.25.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:10ef2a79ab8e2974e2075fb984e5b9806c64134810fac21576f0668e7ea19f8f", size = 5046806, upload-time = "2025-09-14T22:17:08.415Z" }, + { url = "https://files.pythonhosted.org/packages/08/b3/206883dd25b8d1591a1caa44b54c2aad84badccf2f1de9e2d60a446f9a25/zstandard-0.25.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aaf21ba8fb76d102b696781bddaa0954b782536446083ae3fdaa6f16b25a1c4b", size = 5576659, upload-time = "2025-09-14T22:17:10.164Z" }, + { url = "https://files.pythonhosted.org/packages/9d/31/76c0779101453e6c117b0ff22565865c54f48f8bd807df2b00c2c404b8e0/zstandard-0.25.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1869da9571d5e94a85a5e8d57e4e8807b175c9e4a6294e3b66fa4efb074d90f6", size = 4953933, upload-time = "2025-09-14T22:17:11.857Z" }, + { url = "https://files.pythonhosted.org/packages/18/e1/97680c664a1bf9a247a280a053d98e251424af51f1b196c6d52f117c9720/zstandard-0.25.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:809c5bcb2c67cd0ed81e9229d227d4ca28f82d0f778fc5fea624a9def3963f91", size = 5268008, upload-time = "2025-09-14T22:17:13.627Z" }, + { url = "https://files.pythonhosted.org/packages/1e/73/316e4010de585ac798e154e88fd81bb16afc5c5cb1a72eeb16dd37e8024a/zstandard-0.25.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:f27662e4f7dbf9f9c12391cb37b4c4c3cb90ffbd3b1fb9284dadbbb8935fa708", size = 5433517, upload-time = "2025-09-14T22:17:16.103Z" }, + { url = "https://files.pythonhosted.org/packages/5b/60/dd0f8cfa8129c5a0ce3ea6b7f70be5b33d2618013a161e1ff26c2b39787c/zstandard-0.25.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:99c0c846e6e61718715a3c9437ccc625de26593fea60189567f0118dc9db7512", size = 5814292, upload-time = "2025-09-14T22:17:17.827Z" }, + { url = "https://files.pythonhosted.org/packages/fc/5f/75aafd4b9d11b5407b641b8e41a57864097663699f23e9ad4dbb91dc6bfe/zstandard-0.25.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:474d2596a2dbc241a556e965fb76002c1ce655445e4e3bf38e5477d413165ffa", size = 5360237, upload-time = "2025-09-14T22:17:19.954Z" }, + { url = "https://files.pythonhosted.org/packages/ff/8d/0309daffea4fcac7981021dbf21cdb2e3427a9e76bafbcdbdf5392ff99a4/zstandard-0.25.0-cp312-cp312-win32.whl", hash = "sha256:23ebc8f17a03133b4426bcc04aabd68f8236eb78c3760f12783385171b0fd8bd", size = 436922, upload-time = "2025-09-14T22:17:24.398Z" }, + { url = "https://files.pythonhosted.org/packages/79/3b/fa54d9015f945330510cb5d0b0501e8253c127cca7ebe8ba46a965df18c5/zstandard-0.25.0-cp312-cp312-win_amd64.whl", hash = "sha256:ffef5a74088f1e09947aecf91011136665152e0b4b359c42be3373897fb39b01", size = 506276, upload-time = "2025-09-14T22:17:21.429Z" }, + { url = "https://files.pythonhosted.org/packages/ea/6b/8b51697e5319b1f9ac71087b0af9a40d8a6288ff8025c36486e0c12abcc4/zstandard-0.25.0-cp312-cp312-win_arm64.whl", hash = "sha256:181eb40e0b6a29b3cd2849f825e0fa34397f649170673d385f3598ae17cca2e9", size = 462679, upload-time = "2025-09-14T22:17:23.147Z" }, + { url = "https://files.pythonhosted.org/packages/35/0b/8df9c4ad06af91d39e94fa96cc010a24ac4ef1378d3efab9223cc8593d40/zstandard-0.25.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ec996f12524f88e151c339688c3897194821d7f03081ab35d31d1e12ec975e94", size = 795735, upload-time = "2025-09-14T22:17:26.042Z" }, + { url = "https://files.pythonhosted.org/packages/3f/06/9ae96a3e5dcfd119377ba33d4c42a7d89da1efabd5cb3e366b156c45ff4d/zstandard-0.25.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a1a4ae2dec3993a32247995bdfe367fc3266da832d82f8438c8570f989753de1", size = 640440, upload-time = "2025-09-14T22:17:27.366Z" }, + { url = "https://files.pythonhosted.org/packages/d9/14/933d27204c2bd404229c69f445862454dcc101cd69ef8c6068f15aaec12c/zstandard-0.25.0-cp313-cp313-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:e96594a5537722fdfb79951672a2a63aec5ebfb823e7560586f7484819f2a08f", size = 5343070, upload-time = "2025-09-14T22:17:28.896Z" }, + { url = "https://files.pythonhosted.org/packages/6d/db/ddb11011826ed7db9d0e485d13df79b58586bfdec56e5c84a928a9a78c1c/zstandard-0.25.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bfc4e20784722098822e3eee42b8e576b379ed72cca4a7cb856ae733e62192ea", size = 5063001, upload-time = "2025-09-14T22:17:31.044Z" }, + { url = "https://files.pythonhosted.org/packages/db/00/87466ea3f99599d02a5238498b87bf84a6348290c19571051839ca943777/zstandard-0.25.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:457ed498fc58cdc12fc48f7950e02740d4f7ae9493dd4ab2168a47c93c31298e", size = 5394120, upload-time = "2025-09-14T22:17:32.711Z" }, + { url = "https://files.pythonhosted.org/packages/2b/95/fc5531d9c618a679a20ff6c29e2b3ef1d1f4ad66c5e161ae6ff847d102a9/zstandard-0.25.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:fd7a5004eb1980d3cefe26b2685bcb0b17989901a70a1040d1ac86f1d898c551", size = 5451230, upload-time = "2025-09-14T22:17:34.41Z" }, + { url = "https://files.pythonhosted.org/packages/63/4b/e3678b4e776db00f9f7b2fe58e547e8928ef32727d7a1ff01dea010f3f13/zstandard-0.25.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8e735494da3db08694d26480f1493ad2cf86e99bdd53e8e9771b2752a5c0246a", size = 5547173, upload-time = "2025-09-14T22:17:36.084Z" }, + { url = "https://files.pythonhosted.org/packages/4e/d5/ba05ed95c6b8ec30bd468dfeab20589f2cf709b5c940483e31d991f2ca58/zstandard-0.25.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3a39c94ad7866160a4a46d772e43311a743c316942037671beb264e395bdd611", size = 5046736, upload-time = "2025-09-14T22:17:37.891Z" }, + { url = "https://files.pythonhosted.org/packages/50/d5/870aa06b3a76c73eced65c044b92286a3c4e00554005ff51962deef28e28/zstandard-0.25.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:172de1f06947577d3a3005416977cce6168f2261284c02080e7ad0185faeced3", size = 5576368, upload-time = "2025-09-14T22:17:40.206Z" }, + { url = "https://files.pythonhosted.org/packages/5d/35/398dc2ffc89d304d59bc12f0fdd931b4ce455bddf7038a0a67733a25f550/zstandard-0.25.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3c83b0188c852a47cd13ef3bf9209fb0a77fa5374958b8c53aaa699398c6bd7b", size = 4954022, upload-time = "2025-09-14T22:17:41.879Z" }, + { url = "https://files.pythonhosted.org/packages/9a/5c/36ba1e5507d56d2213202ec2b05e8541734af5f2ce378c5d1ceaf4d88dc4/zstandard-0.25.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1673b7199bbe763365b81a4f3252b8e80f44c9e323fc42940dc8843bfeaf9851", size = 5267889, upload-time = "2025-09-14T22:17:43.577Z" }, + { url = "https://files.pythonhosted.org/packages/70/e8/2ec6b6fb7358b2ec0113ae202647ca7c0e9d15b61c005ae5225ad0995df5/zstandard-0.25.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:0be7622c37c183406f3dbf0cba104118eb16a4ea7359eeb5752f0794882fc250", size = 5433952, upload-time = "2025-09-14T22:17:45.271Z" }, + { url = "https://files.pythonhosted.org/packages/7b/01/b5f4d4dbc59ef193e870495c6f1275f5b2928e01ff5a81fecb22a06e22fb/zstandard-0.25.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:5f5e4c2a23ca271c218ac025bd7d635597048b366d6f31f420aaeb715239fc98", size = 5814054, upload-time = "2025-09-14T22:17:47.08Z" }, + { url = "https://files.pythonhosted.org/packages/b2/e5/fbd822d5c6f427cf158316d012c5a12f233473c2f9c5fe5ab1ae5d21f3d8/zstandard-0.25.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4f187a0bb61b35119d1926aee039524d1f93aaf38a9916b8c4b78ac8514a0aaf", size = 5360113, upload-time = "2025-09-14T22:17:48.893Z" }, + { url = "https://files.pythonhosted.org/packages/8e/e0/69a553d2047f9a2c7347caa225bb3a63b6d7704ad74610cb7823baa08ed7/zstandard-0.25.0-cp313-cp313-win32.whl", hash = "sha256:7030defa83eef3e51ff26f0b7bfb229f0204b66fe18e04359ce3474ac33cbc09", size = 436936, upload-time = "2025-09-14T22:17:52.658Z" }, + { url = "https://files.pythonhosted.org/packages/d9/82/b9c06c870f3bd8767c201f1edbdf9e8dc34be5b0fbc5682c4f80fe948475/zstandard-0.25.0-cp313-cp313-win_amd64.whl", hash = "sha256:1f830a0dac88719af0ae43b8b2d6aef487d437036468ef3c2ea59c51f9d55fd5", size = 506232, upload-time = "2025-09-14T22:17:50.402Z" }, + { url = "https://files.pythonhosted.org/packages/d4/57/60c3c01243bb81d381c9916e2a6d9e149ab8627c0c7d7abb2d73384b3c0c/zstandard-0.25.0-cp313-cp313-win_arm64.whl", hash = "sha256:85304a43f4d513f5464ceb938aa02c1e78c2943b29f44a750b48b25ac999a049", size = 462671, upload-time = "2025-09-14T22:17:51.533Z" }, + { url = "https://files.pythonhosted.org/packages/3d/5c/f8923b595b55fe49e30612987ad8bf053aef555c14f05bb659dd5dbe3e8a/zstandard-0.25.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e29f0cf06974c899b2c188ef7f783607dbef36da4c242eb6c82dcd8b512855e3", size = 795887, upload-time = "2025-09-14T22:17:54.198Z" }, + { url = "https://files.pythonhosted.org/packages/8d/09/d0a2a14fc3439c5f874042dca72a79c70a532090b7ba0003be73fee37ae2/zstandard-0.25.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:05df5136bc5a011f33cd25bc9f506e7426c0c9b3f9954f056831ce68f3b6689f", size = 640658, upload-time = "2025-09-14T22:17:55.423Z" }, + { url = "https://files.pythonhosted.org/packages/5d/7c/8b6b71b1ddd517f68ffb55e10834388d4f793c49c6b83effaaa05785b0b4/zstandard-0.25.0-cp314-cp314-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:f604efd28f239cc21b3adb53eb061e2a205dc164be408e553b41ba2ffe0ca15c", size = 5379849, upload-time = "2025-09-14T22:17:57.372Z" }, + { url = "https://files.pythonhosted.org/packages/a4/86/a48e56320d0a17189ab7a42645387334fba2200e904ee47fc5a26c1fd8ca/zstandard-0.25.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:223415140608d0f0da010499eaa8ccdb9af210a543fac54bce15babbcfc78439", size = 5058095, upload-time = "2025-09-14T22:17:59.498Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ad/eb659984ee2c0a779f9d06dbfe45e2dc39d99ff40a319895df2d3d9a48e5/zstandard-0.25.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2e54296a283f3ab5a26fc9b8b5d4978ea0532f37b231644f367aa588930aa043", size = 5551751, upload-time = "2025-09-14T22:18:01.618Z" }, + { url = "https://files.pythonhosted.org/packages/61/b3/b637faea43677eb7bd42ab204dfb7053bd5c4582bfe6b1baefa80ac0c47b/zstandard-0.25.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ca54090275939dc8ec5dea2d2afb400e0f83444b2fc24e07df7fdef677110859", size = 6364818, upload-time = "2025-09-14T22:18:03.769Z" }, + { url = "https://files.pythonhosted.org/packages/31/dc/cc50210e11e465c975462439a492516a73300ab8caa8f5e0902544fd748b/zstandard-0.25.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e09bb6252b6476d8d56100e8147b803befa9a12cea144bbe629dd508800d1ad0", size = 5560402, upload-time = "2025-09-14T22:18:05.954Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ae/56523ae9c142f0c08efd5e868a6da613ae76614eca1305259c3bf6a0ed43/zstandard-0.25.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a9ec8c642d1ec73287ae3e726792dd86c96f5681eb8df274a757bf62b750eae7", size = 4955108, upload-time = "2025-09-14T22:18:07.68Z" }, + { url = "https://files.pythonhosted.org/packages/98/cf/c899f2d6df0840d5e384cf4c4121458c72802e8bda19691f3b16619f51e9/zstandard-0.25.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a4089a10e598eae6393756b036e0f419e8c1d60f44a831520f9af41c14216cf2", size = 5269248, upload-time = "2025-09-14T22:18:09.753Z" }, + { url = "https://files.pythonhosted.org/packages/1b/c0/59e912a531d91e1c192d3085fc0f6fb2852753c301a812d856d857ea03c6/zstandard-0.25.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f67e8f1a324a900e75b5e28ffb152bcac9fbed1cc7b43f99cd90f395c4375344", size = 5430330, upload-time = "2025-09-14T22:18:11.966Z" }, + { url = "https://files.pythonhosted.org/packages/a0/1d/7e31db1240de2df22a58e2ea9a93fc6e38cc29353e660c0272b6735d6669/zstandard-0.25.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9654dbc012d8b06fc3d19cc825af3f7bf8ae242226df5f83936cb39f5fdc846c", size = 5811123, upload-time = "2025-09-14T22:18:13.907Z" }, + { url = "https://files.pythonhosted.org/packages/f6/49/fac46df5ad353d50535e118d6983069df68ca5908d4d65b8c466150a4ff1/zstandard-0.25.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4203ce3b31aec23012d3a4cf4a2ed64d12fea5269c49aed5e4c3611b938e4088", size = 5359591, upload-time = "2025-09-14T22:18:16.465Z" }, + { url = "https://files.pythonhosted.org/packages/c2/38/f249a2050ad1eea0bb364046153942e34abba95dd5520af199aed86fbb49/zstandard-0.25.0-cp314-cp314-win32.whl", hash = "sha256:da469dc041701583e34de852d8634703550348d5822e66a0c827d39b05365b12", size = 444513, upload-time = "2025-09-14T22:18:20.61Z" }, + { url = "https://files.pythonhosted.org/packages/3a/43/241f9615bcf8ba8903b3f0432da069e857fc4fd1783bd26183db53c4804b/zstandard-0.25.0-cp314-cp314-win_amd64.whl", hash = "sha256:c19bcdd826e95671065f8692b5a4aa95c52dc7a02a4c5a0cac46deb879a017a2", size = 516118, upload-time = "2025-09-14T22:18:17.849Z" }, + { url = "https://files.pythonhosted.org/packages/f0/ef/da163ce2450ed4febf6467d77ccb4cd52c4c30ab45624bad26ca0a27260c/zstandard-0.25.0-cp314-cp314-win_arm64.whl", hash = "sha256:d7541afd73985c630bafcd6338d2518ae96060075f9463d7dc14cfb33514383d", size = 476940, upload-time = "2025-09-14T22:18:19.088Z" }, +] From d55926eeb1793a3a8b1c1852a783c4fd31e970dc Mon Sep 17 00:00:00 2001 From: hrodmn Date: Fri, 11 Sep 2026 11:17:46 -0500 Subject: [PATCH 23/23] feat: eliminate IcestacCatalog, defer to pyiceberg --- README.md | 109 +++++-- main.py | 53 +-- src/icestac/catalog.py | 79 ----- src/icestac/constants.py | 1 + src/icestac/errors.py | 2 - src/icestac/load.py | 70 ---- src/icestac/schema.py | 170 +++++----- src/icestac/write.py | 79 +++++ tests/conftest.py | 13 +- tests/test_catalog.py | 277 ---------------- tests/test_load.py | 552 -------------------------------- tests/test_main.py | 13 +- tests/test_schema.py | 17 +- tests/test_write.py | 673 +++++++++++++++++++++++++++++++++++++++ 14 files changed, 982 insertions(+), 1126 deletions(-) delete mode 100644 src/icestac/catalog.py delete mode 100644 src/icestac/errors.py delete mode 100644 src/icestac/load.py create mode 100644 src/icestac/write.py delete mode 100644 tests/test_catalog.py delete mode 100644 tests/test_load.py create mode 100644 tests/test_write.py diff --git a/README.md b/README.md index 12adab0..6d5baaa 100644 --- a/README.md +++ b/README.md @@ -1,73 +1,109 @@ # icestac -Store STAC Items in Apache Iceberg, with one table per collection. +icestac maps STAC concepts onto Apache Iceberg. -## Load items +The current implementation covers STAC Items with schema utilities for the flattened item layout defined by [STAC GeoParquet](https://radiantearth.github.io/stac-geoparquet-spec/), a small write API, and runnable PyIceberg examples. -The input is an Arrow table (`pyarrow.Table` or `arro3.core.Table`) in rustac's flattened STAC format. +Each collection gets one Iceberg table named with its STAC collection ID. Clients can locate an item table from a collection ID without a metadata lookup. Each row represents one item, and an item's identity is scoped to its collection. Requesting an item by collection and ID should return one current record, not multiple versions. + +The `collections` table name is reserved for planned collection metadata. Collection metadata is not implemented yet. + +## Storage conventions + +Use PyIceberg for catalog and table administration, including namespace and table creation, partitioning, sorting, and table properties. icestac does not choose a default table layout. + +The schema utilities and `put_items` handle the STAC-facing parts. They work with an Arrow table (`pyarrow.Table` or `arro3.core.Table`) using the [STAC GeoParquet](https://radiantearth.github.io/stac-geoparquet-spec/) column mapping: item properties are top-level columns, while geometry is WKB and links and assets remain nested structures. This describes the in-memory Arrow input, not a compliant STAC GeoParquet file; file compliance also requires GeoParquet and STAC metadata. `get_schema_from_items` checks the required fields and their basic Arrow types, then creates an Iceberg schema. It does not perform full STAC validation. + +The inferred schema marks `id` as an Iceberg identifier field. Iceberg identifier fields are schema metadata, not uniqueness constraints. Starting with a table that has unique `(collection, id)` pairs and writing through `put_items` preserves the one-current-row convention. Native writes that bypass `put_items`, such as appends, can create duplicate IDs and violate it. + +## Put items + +Create a table for one collection with native PyIceberg, then pass its items to `put_items`: ```python from pyiceberg.catalog import load_catalog -from icestac.catalog import IcestacCatalog +from icestac.constants import DEFAULT_NAMESPACE from icestac.schema import get_schema_from_items +from icestac.write import put_items # items is an Arrow table containing items from one collection. collection_id = "my-collection" -catalog = IcestacCatalog(catalog=load_catalog()) +catalog = load_catalog() +catalog.create_namespace_if_not_exists(DEFAULT_NAMESPACE) schema = get_schema_from_items(items) -catalog.create_item_table(collection_id=collection_id, iceberg_schema=schema) -catalog.load_items(collection_id=collection_id, items=items) +table = catalog.create_table( + identifier=(DEFAULT_NAMESPACE, collection_id), + schema=schema, +) +put_items(table, items) ``` -Use the collection ID from your items. IDs containing periods are unsupported because icestac uses them in dotted table identifiers. Tables live in the `icestac` namespace by default. +`put_items` accepts items from one collection and checks that their `collection` values match the collection named by the table. It rejects duplicate IDs within an input batch. Matching items are complete replacements, so omitted optional fields are cleared, including nested link fields. New IDs are inserted. + +Collection IDs can contain periods. Pass catalog identifiers as tuples such as `(namespace, collection_id)` instead of interpolated dotted strings; a catalog backend may still reject a particular ID. Do not use `collections` as an item table name. -Loading checks the collection ID and upserts on STAC `id`. To append without replacing existing items, pass `method="append"`. +For append semantics, use native PyIceberg: + +```python +table.append(df=items) +``` ### Schema evolution -New fields raise an error unless you pass `evolve_schema=True`: +New fields raise an error unless you pass `evolve_schema=True`. Schema evolution and the replacement write happen in one Iceberg transaction: ```python -catalog.load_items(collection_id=collection_id, items=items, evolve_schema=True) +put_items(table, items, evolve_schema=True) ``` -Compatible schema changes and the write commit in one transaction. To update a schema without loading items, use PyIceberg: +To update a schema without writing items, use native PyIceberg: ```python -table = catalog.catalog.load_table((catalog.namespace, collection_id)) +from pyiceberg.catalog import load_catalog + +from icestac.constants import DEFAULT_NAMESPACE +from icestac.schema import get_schema_from_items + +catalog = load_catalog() +table = catalog.load_table((DEFAULT_NAMESPACE, collection_id)) with table.update_schema() as update: update.union_by_name(get_schema_from_items(items)) ``` -### Table layout +### Concurrent writes + +Each batch, including optional schema evolution, commits as one Iceberg transaction. A concurrent commit conflict raises `CommitFailedException`; refresh the table and retry from current state rather than replaying a stale replacement. A `CommitStateUnknownException` means the commit outcome is unknown, so refresh and reconcile before retrying. + +## Table layout -Tables use monthly `datetime` partitions and no sort order by default. Pass native PyIceberg objects to choose another layout. For example, partition by year and sort by `datetime`: +Choose the layout when creating the table. For example, partition by month of `datetime`, sort by `datetime`, and set a Parquet row group limit: ```python from pyiceberg.partitioning import PartitionField, PartitionSpec +from pyiceberg.table import TableProperties from pyiceberg.table.sorting import SortField, SortOrder -from pyiceberg.transforms import YearTransform +from pyiceberg.transforms import MonthTransform +schema = get_schema_from_items(items) datetime_id = schema.find_field("datetime").field_id -catalog.create_item_table( - collection_id=collection_id, - iceberg_schema=schema, +table = catalog.create_table( + identifier=(DEFAULT_NAMESPACE, collection_id), + schema=schema, partition_spec=PartitionSpec( PartitionField( source_id=datetime_id, field_id=1000, - transform=YearTransform(), - name="datetime_year", + transform=MonthTransform(), + name="datetime_month", ) ), sort_order=SortOrder(SortField(source_id=datetime_id)), + properties={TableProperties.PARQUET_ROW_GROUP_LIMIT: "50000"}, ) ``` -Use this instead of the earlier `create_item_table` call. Build field references from the schema you pass to it. Pass `PartitionSpec()` for an unpartitioned table. - -Interval items with a null `datetime` go into a null partition. To partition them by their start time, use the `start_datetime` field with `MonthTransform()`. +Omit `partition_spec` for an unpartitioned table. Interval items with a null `datetime` go into a null partition. To partition them by their start time, use the `start_datetime` field with `MonthTransform()`. Geometry uses WKB in an Iceberg binary column. The output files lack the metadata required for GeoParquet and STAC GeoParquet compliance. @@ -93,12 +129,14 @@ The checked-in `.pyiceberg.yaml` points to these services. The credentials (`adm uv run main.py ``` -The demo downloads January–August 2026 HLS STAC GeoParquet from public S3 into `data/`, reuses cached files, and upserts each month. Each batch must fit in memory. +The demo downloads January–August 2026 HLS STAC GeoParquet from public S3 into `data/`, reuses cached files, and writes each month's items to the `HLSS30_2.0` collection table. Each batch must fit in memory. -It renames collection `HLSS30_2.0` to `HLSS30_2_0`, sorts items by DuckDB's Hilbert index of their bbox lower-left coordinates, and caps Parquet row groups at 50,000 rows. New tables record the Hilbert sort order. Compatible schema changes use `evolve_schema=True`. +It keeps the original `HLSS30_2.0` collection ID, partitions by month of `datetime`, sorts by DuckDB's Hilbert index of each bbox's lower-left coordinates, and caps Parquet row groups at 50,000 rows. Compatible schema changes use `evolve_schema=True`; rerunning the demo reuses the existing table and replaces the cached batches again. ### Query with DuckDB +The query selects items overlapping the contiguous United States bounding box (longitude -125 to -66, latitude 24 to 50) from April 1, 2026 inclusive through July 1, 2026 exclusive; the bbox predicates test bounding-box overlap, not exact geometry intersection. + ```sql INSTALL iceberg; LOAD iceberg; INSTALL httpfs; LOAD httpfs; @@ -118,11 +156,18 @@ ATTACH 'icestac' AS catalog ( AUTHORIZATION_TYPE 'none' ); -SELECT id, datetime, collection, geometry -FROM catalog.icestac.HLSS30_2_0 -LIMIT 10; +SELECT count(*) FROM catalog.icestac."HLSS30_2.0"; + +EXPLAIN ANALYZE SELECT count(*) +FROM catalog.icestac."HLSS30_2.0" +WHERE bbox.xmin <= -90.0 + AND bbox.xmax >= -100.0 + AND bbox.ymin <= 50.0 + AND bbox.ymax >= 40.0 + AND datetime >= TIMESTAMPTZ '2026-04-01 00:00:00+00' + AND datetime < TIMESTAMPTZ '2026-07-01 00:00:00+00' +; -SELECT count(*) FROM catalog.icestac.HLSS30_2_0; ``` ### Delete the demo table @@ -132,7 +177,9 @@ SELECT count(*) FROM catalog.icestac.HLSS30_2_0; ```python from pyiceberg.catalog import load_catalog -load_catalog().purge_table(("icestac", "HLSS30_2_0")) +from icestac.constants import DEFAULT_NAMESPACE + +load_catalog().purge_table((DEFAULT_NAMESPACE, "HLSS30_2.0")) ``` To remove only the catalog entry and keep the files, use `drop_table(...)` instead. diff --git a/main.py b/main.py index 34c18e3..f760ea9 100644 --- a/main.py +++ b/main.py @@ -6,11 +6,14 @@ from obstore.store import LocalStore, S3Store from pyiceberg.catalog import load_catalog from pyiceberg.exceptions import TableAlreadyExistsError +from pyiceberg.partitioning import PartitionField, PartitionSpec from pyiceberg.table import TableProperties from pyiceberg.table.sorting import SortField, SortOrder +from pyiceberg.transforms import MonthTransform from rustac import DuckdbClient -from icestac.catalog import IcestacCatalog +from icestac.constants import DEFAULT_NAMESPACE +from icestac.write import put_items from icestac.schema import get_schema_from_items logger = logging.getLogger("icestac-demo") @@ -20,15 +23,15 @@ MAX_ROW_GROUP_SIZE = 50_000 -def read_hls_items(client: DuckdbClient, path: Path, collection_id: str) -> pa.Table: - """Read an HLS batch, rename its collection, and sort by bbox Hilbert index.""" +def read_hls_items(client: DuckdbClient, path: Path) -> pa.Table: + """Read an HLS batch and sort it by bbox Hilbert index.""" # Keep GeoParquet geometry as WKB for Iceberg's binary column. client.execute("SET enable_geoparquet_conversion = false") client.execute("SET TimeZone = 'UTC'") items = pa.table( client.query_to_table( """ - SELECT * REPLACE (? AS collection), + SELECT *, CASE WHEN isfinite(bbox.xmin) AND isfinite(bbox.ymin) THEN ST_Hilbert( bbox.xmin, bbox.ymin, @@ -39,7 +42,7 @@ def read_hls_items(client: DuckdbClient, path: Path, collection_id: str) -> pa.T FROM read_parquet(?, hive_partitioning = false) ORDER BY hilbert_idx """, - [collection_id, str(path)], + [str(path)], ) ) if not len(items): @@ -56,7 +59,7 @@ async def run() -> None: format="%(asctime)s %(levelname)s:%(name)s:%(message)s", datefmt="%Y-%m-%dT%H:%M:%S%z", ) - catalog = IcestacCatalog(catalog=load_catalog()) + catalog = load_catalog() client = DuckdbClient() data_dir = Path("data") data_dir.mkdir(exist_ok=True) @@ -67,13 +70,13 @@ async def run() -> None: region="us-west-2", skip_signature=True, ) - source_collection_id = "HLSS30_2.0" - collection_id = "HLSS30_2_0" + collection_id = "HLSS30_2.0" + table = None for month in range(1, 9): path = ( - f"{source_collection_id}/year=2026/month={month}/" - f"{source_collection_id}-2026-{month}.parquet" + f"{collection_id}/year=2026/month={month}/" + f"{collection_id}-2026-{month}.parquet" ) try: local_store.head(path) @@ -83,28 +86,38 @@ async def run() -> None: await local_store.put_async(path, response) logger.info("Reading and sorting %s", path) - items = read_hls_items(client, data_dir / path, collection_id) + items = read_hls_items(client, data_dir / path) if month == 1: schema = get_schema_from_items(items) + datetime_id = schema.find_field("datetime").field_id + catalog.create_namespace_if_not_exists(DEFAULT_NAMESPACE) try: - table = catalog.create_item_table( - iceberg_schema=schema, - collection_id=collection_id, + table = catalog.create_table( + identifier=(DEFAULT_NAMESPACE, collection_id), + schema=schema, + partition_spec=PartitionSpec( + PartitionField( + source_id=datetime_id, + field_id=1000, + transform=MonthTransform(), + name="datetime_month", + ) + ), sort_order=SortOrder( SortField(source_id=schema.find_field("hilbert_idx").field_id) ), + properties={ + TableProperties.PARQUET_ROW_GROUP_LIMIT: str(MAX_ROW_GROUP_SIZE) + }, ) except TableAlreadyExistsError: - table = catalog.catalog.load_table((catalog.namespace, collection_id)) + table = catalog.load_table((DEFAULT_NAMESPACE, collection_id)) logger.info("Using existing table %s", collection_id) - with table.transaction() as transaction: - transaction.set_properties( - {TableProperties.PARQUET_ROW_GROUP_LIMIT: str(MAX_ROW_GROUP_SIZE)} - ) logger.info("Loading %s items for 2026-%02d", len(items), month) - catalog.load_items(collection_id, items, evolve_schema=True) + assert table is not None + put_items(table, items, evolve_schema=True) if __name__ == "__main__": diff --git a/src/icestac/catalog.py b/src/icestac/catalog.py deleted file mode 100644 index d6f3b21..0000000 --- a/src/icestac/catalog.py +++ /dev/null @@ -1,79 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -from pyiceberg.catalog import Catalog -from pyiceberg.partitioning import PartitionField, PartitionSpec -from pyiceberg.schema import Schema as IcebergSchema -from pyiceberg.table import Table, TableProperties -from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder -from pyiceberg.transforms import MonthTransform - -from icestac.constants import DEFAULT_NAMESPACE -from icestac.errors import InvalidCollectionIdError -from icestac.load import Method, load_items -from icestac.schema import IcestacItem, ItemsInput - - -def validate_collection_id(collection_id: str) -> None: - """Ensure a collection ID can be used as an Iceberg table name.""" - - if "." in collection_id: - raise InvalidCollectionIdError(collection_id) - - -@dataclass -class IcestacCatalog: - """Manage collection item tables through a PyIceberg catalog.""" - - catalog: Catalog - namespace: str = DEFAULT_NAMESPACE - - def create_item_table( - self, - collection_id: str, - iceberg_schema: IcebergSchema, - partition_spec: PartitionSpec | None = None, - sort_order: SortOrder = UNSORTED_SORT_ORDER, - ) -> Table: - """Create an Iceberg item table for a collection.""" - validate_collection_id(collection_id) - IcestacItem.validate_schema(iceberg_schema) - self.catalog.create_namespace_if_not_exists(self.namespace) - - # TODO: check if collection record is present in collections table - - if partition_spec is None: - partition_spec = PartitionSpec( - PartitionField( - source_id=iceberg_schema.find_field("datetime").field_id, - field_id=1000, - transform=MonthTransform(), - name="datetime_month", - ) - ) - - return self.catalog.create_table( - identifier=f"{self.namespace}.{collection_id}", - schema=iceberg_schema, - partition_spec=partition_spec, - sort_order=sort_order, - properties={TableProperties.COMMIT_NUM_RETRIES: "0"}, - ) - - def load_items( - self, - collection_id: str, - items: ItemsInput, - method: Method = "upsert", - evolve_schema: bool = False, - ) -> None: - """Load items, optionally evolving their collection table schema.""" - load_items( - items, - table=self.catalog.load_table( - identifier=f"{self.namespace}.{collection_id}" - ), - method=method, - evolve_schema=evolve_schema, - ) diff --git a/src/icestac/constants.py b/src/icestac/constants.py index 1012ecf..4df4b90 100644 --- a/src/icestac/constants.py +++ b/src/icestac/constants.py @@ -1 +1,2 @@ DEFAULT_NAMESPACE = "icestac" +COLLECTIONS_TABLE_NAME = "collections" diff --git a/src/icestac/errors.py b/src/icestac/errors.py deleted file mode 100644 index 80f58f7..0000000 --- a/src/icestac/errors.py +++ /dev/null @@ -1,2 +0,0 @@ -class InvalidCollectionIdError(Exception): - """Invalid collection id""" diff --git a/src/icestac/load.py b/src/icestac/load.py deleted file mode 100644 index 7297246..0000000 --- a/src/icestac/load.py +++ /dev/null @@ -1,70 +0,0 @@ -import warnings -from typing import Literal - -from pyiceberg.table import Table, TableProperties -from pyiceberg.table.upsert_util import create_match_filter, has_duplicate_rows - -from icestac.schema import ItemsInput, prepare_arrow_table - -Method = Literal["append", "upsert"] - - -def load_items( - items: ItemsInput, - table: Table, - method: Method = "upsert", - evolve_schema: bool = False, -) -> None: - """Load STAC items, optionally evolving the Iceberg schema by name. - - Upserts are complete replacements: omitted optional fields are written as - null rather than retained from the previous item. Upsert commit retries - are disabled because PyIceberg cannot safely replay the match against - refreshed table state. A known ``CommitFailedException`` - is propagated for a fresh retry; a ``CommitStateUnknownException`` is also - propagated and requires refreshing/reconciling before retrying the load. - """ - if method not in ("append", "upsert"): - raise ValueError(f"Unsupported load method: {method}") - - arrow_table = prepare_arrow_table(items) - collection_ids = set(arrow_table.column("collection").unique().to_pylist()) - expected_collection_id = table.name()[-1] - if collection_ids != {expected_collection_id}: - raise ValueError( - f"Items for {expected_collection_id!r} contain collection ids " - f"{sorted(map(str, collection_ids))}" - ) - - if method == "upsert": - # PyIceberg retries a staged merge against refreshed metadata without - # rerunning the original upsert lookup, which can duplicate an ID. - table.metadata.properties[TableProperties.COMMIT_NUM_RETRIES] = "0" - - with table.transaction() as transaction: - if evolve_schema: - with transaction.update_schema() as update: - update.union_by_name(arrow_table.schema) - - if method == "upsert": - # Overwrite matching IDs so omitted optional fields are cleared. - # PyIceberg handles schema alignment, including staged evolution. - if has_duplicate_rows(arrow_table, ["id"]): - raise ValueError( - "Duplicate rows found in source dataset based on the key " - "columns. No upsert executed" - ) - with warnings.catch_warnings(): - # New IDs legitimately have no existing records to delete. - warnings.filterwarnings( - "ignore", - message="^Delete operation did not match any records$", - category=UserWarning, - module=r"^pyiceberg\.table$", - ) - transaction.overwrite( - df=arrow_table, - overwrite_filter=create_match_filter(arrow_table, ["id"]), - ) - else: - transaction.append(df=arrow_table) diff --git a/src/icestac/schema.py b/src/icestac/schema.py index 17de122..131ed12 100644 --- a/src/icestac/schema.py +++ b/src/icestac/schema.py @@ -1,7 +1,7 @@ import pyarrow as pa from arro3.core import Schema as ArrowSchema from arro3.core import Table as ArrowTable -from pyiceberg.io.pyarrow import _pyarrow_to_schema_without_ids +from pyiceberg.io.pyarrow import _pyarrow_to_schema_without_ids, schema_to_pyarrow from pyiceberg.schema import Schema as IcebergSchema from pyiceberg.schema import assign_fresh_schema_ids from pyiceberg.types import ( @@ -14,6 +14,10 @@ ) ItemsInput = pa.Table | ArrowTable +REQUIRED_FIELDS = frozenset( + {"geometry", "type", "id", "datetime", "links", "collection", "assets"} +) +NON_NULLABLE_FIELDS = frozenset({"type", "id", "links", "collection", "assets"}) LINK_TYPE = pa.list_( pa.struct( [ @@ -27,87 +31,90 @@ EMPTY_ITEMS_ERROR = "Cannot infer or load a schema from an empty Arrow table" -class IcestacItem: - """Structural fields required by the flattened STAC representation.""" +def enforce_required_fields(schema: ArrowSchema) -> ArrowSchema: + """Make required fields non-null and give empty links a concrete type.""" + arrow_schema = pa.schema(schema) + fields = [] + for field in arrow_schema: + if field.name in NON_NULLABLE_FIELDS: + field = field.with_nullable(False) + if ( + field.name == "links" + and pa.types.is_list(field.type) + and pa.types.is_null(field.type.value_type) + ): + field = field.with_type(LINK_TYPE) + fields.append(field) + return ArrowSchema.from_arrow(pa.schema(fields, metadata=arrow_schema.metadata)) + - @classmethod - def get_required_fields(cls) -> set[str]: - """Return required top-level fields, including flattened datetime.""" - return {"geometry", "type", "id", "datetime", "links", "collection", "assets"} +def validate_schema(schema: pa.Schema | ArrowSchema | IcebergSchema) -> None: + """Validate the structural fields and types of a flattened STAC schema.""" + if isinstance(schema, IcebergSchema): + schema_fields = set(schema.column_names) + else: + schema_fields = set(schema.names) + missing_fields = REQUIRED_FIELDS - schema_fields - @classmethod - def get_non_nullable_fields(cls) -> set[str]: - """Return required fields that cannot contain null values.""" - return {"type", "id", "links", "collection", "assets"} + if missing_fields: + raise ValueError( + f"Schema is missing required STAC fields: {sorted(missing_fields)}" + ) - @classmethod - def enforce_required_fields(cls, schema: ArrowSchema) -> ArrowSchema: - """Make required fields non-null and give empty links a concrete type.""" - non_nullable_fields = cls.get_non_nullable_fields() + if isinstance(schema, IcebergSchema): + fields = {field.name: field.field_type for field in schema.fields} + expected = { + "type": StringType, + "id": StringType, + "collection": StringType, + "geometry": BinaryType, + "datetime": (TimestampType, TimestamptzType), + "links": ListType, + "assets": StructType, + } + invalid = [ + name + for name, field_type in expected.items() + if not isinstance(fields[name], field_type) + ] + else: arrow_schema = pa.schema(schema) - fields = [] - for field in arrow_schema: - if field.name in non_nullable_fields: - field = field.with_nullable(False) - if ( - field.name == "links" - and pa.types.is_list(field.type) - and pa.types.is_null(field.type.value_type) - ): - field = field.with_type(LINK_TYPE) - fields.append(field) - return ArrowSchema.from_arrow(pa.schema(fields, metadata=arrow_schema.metadata)) - - @classmethod - def validate_schema(cls, schema: pa.Schema | ArrowSchema | IcebergSchema) -> None: - """Validate required fields and the types used by the STAC representation.""" - if isinstance(schema, IcebergSchema): - schema_fields = set(schema.column_names) - else: - schema_fields = set(schema.names) - missing_fields = cls.get_required_fields() - schema_fields - - if missing_fields: - raise ValueError( - f"Schema is missing required STAC fields: {sorted(missing_fields)}" - ) - - if isinstance(schema, IcebergSchema): - fields = {field.name: field.field_type for field in schema.fields} - expected = { - "id": StringType, - "collection": StringType, - "geometry": BinaryType, - "datetime": (TimestampType, TimestamptzType), - "links": ListType, - "assets": StructType, - } - invalid = [ - name - for name, field_type in expected.items() - if not isinstance(fields[name], field_type) - ] - else: - arrow_schema = pa.schema(schema) - invalid = [] - for name, predicate in ( - ("id", pa.types.is_string), - ("collection", pa.types.is_string), - ("geometry", pa.types.is_binary), - ("datetime", pa.types.is_timestamp), - ("links", pa.types.is_list), - ("assets", pa.types.is_struct), + invalid = [] + for name, predicate in ( + ("type", pa.types.is_string), + ("id", pa.types.is_string), + ("collection", pa.types.is_string), + ("geometry", pa.types.is_binary), + ("datetime", pa.types.is_timestamp), + ("links", pa.types.is_list), + ("assets", pa.types.is_struct), + ): + field_type = arrow_schema.field(name).type + if name in ("type", "id", "collection") and pa.types.is_dictionary( + field_type ): - field_type = arrow_schema.field(name).type - if name in ("id", "collection") and pa.types.is_dictionary(field_type): - field_type = field_type.value_type - if not predicate(field_type): - invalid.append(name) + field_type = field_type.value_type + if not predicate(field_type): + invalid.append(name) - if invalid: - raise ValueError( - "Unsupported types for STAC fields: " + ", ".join(sorted(invalid)) - ) + if invalid: + raise ValueError( + "Unsupported types for STAC fields: " + ", ".join(sorted(invalid)) + ) + + +def _align_empty_links(items: pa.Table, schema: IcebergSchema) -> pa.Table: + """Align empty links with the table's nested schema before writing.""" + if not all(not links for links in items["links"].to_pylist()): + return items + + links_type = schema_to_pyarrow(schema, include_field_ids=False).field("links").type + links = items["links"].cast(links_type) + links_index = items.schema.get_field_index("links") + links_field = ( + items.schema.field(links_index).with_type(links_type).with_nullable(False) + ) + return items.set_column(links_index, links_field, links) def prepare_arrow_table(items: ItemsInput) -> pa.Table: @@ -121,8 +128,8 @@ def prepare_arrow_table(items: ItemsInput) -> pa.Table: if len(table) == 0: raise ValueError(EMPTY_ITEMS_ERROR) - IcestacItem.validate_schema(table.schema) - schema = IcestacItem.enforce_required_fields(ArrowSchema.from_arrow(table.schema)) + validate_schema(table.schema) + schema = enforce_required_fields(ArrowSchema.from_arrow(table.schema)) return table.cast(pa.schema(schema)) @@ -130,4 +137,9 @@ def get_schema_from_items(items: ItemsInput) -> IcebergSchema: """Derive an Iceberg schema from a structurally valid Arrow table.""" arrow_table = prepare_arrow_table(items) schema_without_ids = _pyarrow_to_schema_without_ids(arrow_table.schema) - return assign_fresh_schema_ids(schema_without_ids) + schema = assign_fresh_schema_ids(schema_without_ids) + return IcebergSchema( + *schema.fields, + schema_id=schema.schema_id, + identifier_field_ids=[schema.find_field("id").field_id], + ) diff --git a/src/icestac/write.py b/src/icestac/write.py new file mode 100644 index 0000000..6b6d4fe --- /dev/null +++ b/src/icestac/write.py @@ -0,0 +1,79 @@ +import warnings +from pyiceberg.table import Table, TableProperties +from pyiceberg.table.upsert_util import create_match_filter, has_duplicate_rows + +from icestac.constants import COLLECTIONS_TABLE_NAME +from icestac.schema import ItemsInput, _align_empty_links, prepare_arrow_table + + +def put_items( + table: Table, + items: ItemsInput, + *, + evolve_schema: bool = False, +) -> None: + """Replace or insert a batch of STAC items in an Iceberg table. + + Matching IDs are complete replacements: omitted optional fields are + cleared. Commit retries are disabled because PyIceberg cannot safely replay + the match against refreshed table state. Commit failures are propagated; + an unknown commit outcome requires caller reconciliation before retrying. + """ + + arrow_table = prepare_arrow_table(items) + expected_collection_id = table.name()[-1] + if expected_collection_id == COLLECTIONS_TABLE_NAME: + raise ValueError( + f"Cannot load items into reserved table {COLLECTIONS_TABLE_NAME!r}" + ) + + collection_ids = set(arrow_table.column("collection").unique().to_pylist()) + if collection_ids != {expected_collection_id}: + raise ValueError( + f"Items for {expected_collection_id!r} contain collection ids " + f"{sorted(map(str, collection_ids))}" + ) + + # PyIceberg retries a staged merge against refreshed metadata without + # rerunning the original lookup, which can duplicate an ID. + retry_metadata = table.metadata + previous_retries = retry_metadata.properties.get(TableProperties.COMMIT_NUM_RETRIES) + retry_metadata.properties[TableProperties.COMMIT_NUM_RETRIES] = "0" + + try: + with table.transaction() as transaction: + if evolve_schema: + with transaction.update_schema() as update: + update.union_by_name(arrow_table.schema) + + arrow_table = _align_empty_links( + arrow_table, transaction.table_metadata.schema() + ) + # Reject duplicate incoming IDs before staging any write. + if has_duplicate_rows(arrow_table, ["id"]): + raise ValueError( + "Duplicate rows found in source dataset based on the key " + "columns. No write executed" + ) + with warnings.catch_warnings(): + # New IDs legitimately have no existing records to delete. + warnings.filterwarnings( + "ignore", + message="^Delete operation did not match any records$", + category=UserWarning, + module=r"^pyiceberg\.table$", + ) + transaction.overwrite( + df=arrow_table, + overwrite_filter=create_match_filter(arrow_table, ["id"]), + ) + finally: + # A refresh during commit means another metadata object owns the + # policy; never overwrite it with this call's temporary value. + if table.metadata is retry_metadata: + properties = retry_metadata.properties + if properties.get(TableProperties.COMMIT_NUM_RETRIES) == "0": + if previous_retries is None: + properties.pop(TableProperties.COMMIT_NUM_RETRIES, None) + else: + properties[TableProperties.COMMIT_NUM_RETRIES] = previous_retries diff --git a/tests/conftest.py b/tests/conftest.py index 435c154..dc8e569 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,9 +6,9 @@ import pyarrow as pa import pytest import rustac -from pyiceberg.catalog import load_catalog +from pyiceberg.catalog import Catalog, load_catalog -from icestac.catalog import IcestacCatalog +from icestac.constants import DEFAULT_NAMESPACE @pytest.fixture @@ -19,7 +19,7 @@ def temp_warehouse(): @pytest.fixture -def test_catalog(temp_warehouse: Path) -> Generator[IcestacCatalog, None, None]: +def test_catalog(temp_warehouse: Path) -> Generator[Catalog, None, None]: """Create a temporary SQL catalog for testing.""" catalog = load_catalog( "test_catalog", @@ -29,12 +29,11 @@ def test_catalog(temp_warehouse: Path) -> Generator[IcestacCatalog, None, None]: "warehouse": str(temp_warehouse), }, ) - icestac_catalog = IcestacCatalog(catalog=catalog) - - yield icestac_catalog + catalog.create_namespace_if_not_exists(DEFAULT_NAMESPACE) + yield catalog gc.collect() - icestac_catalog.catalog.close() + catalog.close() @pytest.fixture diff --git a/tests/test_catalog.py b/tests/test_catalog.py deleted file mode 100644 index 9845eba..0000000 --- a/tests/test_catalog.py +++ /dev/null @@ -1,277 +0,0 @@ -from typing import Any - -import pyarrow -import pytest -from arro3.core import Schema as ArrowSchema -from pyiceberg.exceptions import TableAlreadyExistsError -from pyiceberg.partitioning import PartitionField, PartitionSpec -from pyiceberg.schema import Schema as IcebergSchema -from pyiceberg.table.sorting import SortField, SortOrder -from pyiceberg.transforms import IdentityTransform, MonthTransform -from icestac.catalog import IcestacCatalog -from icestac.errors import InvalidCollectionIdError -from icestac.schema import IcestacItem, get_schema_from_items -from tests.helpers import items_to_arrow, items_to_list - - -def test_catalog_constructor_does_not_create_namespace( - test_catalog: IcestacCatalog, -) -> None: - """Constructing a catalog wrapper does not mutate a read-only backend.""" - assert not test_catalog.catalog.namespace_exists(test_catalog.namespace) - - -def test_create_item_table( - test_catalog: IcestacCatalog, - test_collection_id: str, - items: pyarrow.Table, -) -> None: - expected_items = items_to_list(items) - iceberg_schema = get_schema_from_items(items) - table = test_catalog.create_item_table( - iceberg_schema=iceberg_schema, - collection_id=test_collection_id, - ) - - assert table.schema().find_field("datetime") - assert len(table.spec().fields) == 1 - assert isinstance(table.spec().fields[0].transform, MonthTransform) - assert not table.sort_order().fields - - # Ensure data has required fields marked as non-nullable to match table schema - enforced_schema = IcestacItem.enforce_required_fields( - ArrowSchema.from_arrow(items.schema) - ) - arrow_table = items.cast(pyarrow.schema(enforced_schema)) - - table.upsert( - df=arrow_table, - join_cols=["id"], - ) - - # Verify records were inserted - result = table.scan().to_arrow() - assert len(result) == len(expected_items) - assert result.column("id").to_pylist() == [item["id"] for item in expected_items] - - with pytest.raises(TableAlreadyExistsError): - test_catalog.create_item_table( - iceberg_schema=iceberg_schema, - collection_id=test_collection_id, - ) - - -def test_create_item_table_bad_collection_id( - test_catalog: IcestacCatalog, - items: pyarrow.Table, -) -> None: - iceberg_schema = get_schema_from_items(items) - with pytest.raises(InvalidCollectionIdError): - test_catalog.create_item_table( - iceberg_schema=iceberg_schema, - collection_id="bad.collection", - ) - - -def test_create_item_table_custom_layout( - test_catalog: IcestacCatalog, - test_collection_id: str, - sample_stac_item: dict[str, Any], -) -> None: - items = items_to_arrow([sample_stac_item]) - iceberg_schema = get_schema_from_items(items) - title_id = iceberg_schema.find_field("title").field_id - - table = test_catalog.create_item_table( - collection_id=test_collection_id, - iceberg_schema=iceberg_schema, - partition_spec=PartitionSpec( - PartitionField( - source_id=title_id, - field_id=1000, - transform=IdentityTransform(), - name="title", - ) - ), - sort_order=SortOrder(SortField(source_id=title_id)), - ) - - assert [field.name for field in table.spec().fields] == ["title"] - assert isinstance(table.spec().fields[0].transform, IdentityTransform) - assert ( - table.spec().fields[0].source_id == table.schema().find_field("title").field_id - ) - assert ( - table.sort_order().fields[0].source_id - == table.schema().find_field("title").field_id - ) - - -def test_create_item_table_nested_layout( - test_catalog: IcestacCatalog, - test_collection_id: str, - sample_stac_item: dict[str, Any], -) -> None: - items = items_to_arrow([sample_stac_item]) - iceberg_schema = get_schema_from_items(items) - xmin_id = iceberg_schema.find_field("bbox.xmin").field_id - - table = test_catalog.create_item_table( - collection_id=test_collection_id, - iceberg_schema=iceberg_schema, - partition_spec=PartitionSpec( - PartitionField( - source_id=xmin_id, - field_id=1000, - transform=IdentityTransform(), - name="bbox_xmin", - ) - ), - sort_order=SortOrder(SortField(source_id=xmin_id)), - ) - - table_xmin_id = table.schema().find_field("bbox.xmin").field_id - table_ymax_id = table.schema().find_field("bbox.ymax").field_id - assert table_xmin_id != table_ymax_id - assert table.spec().fields[0].source_id == table_xmin_id - assert table.sort_order().fields[0].source_id == table_xmin_id - - -def test_create_item_table_unpartitioned( - test_catalog: IcestacCatalog, - test_collection_id: str, - sample_stac_item: dict[str, Any], -) -> None: - items = items_to_arrow([sample_stac_item]) - table = test_catalog.create_item_table( - collection_id=test_collection_id, - iceberg_schema=get_schema_from_items(items), - partition_spec=PartitionSpec(), - ) - - assert not table.spec().fields - - -def test_create_item_table_rejects_invalid_schema( - test_catalog: IcestacCatalog, - test_collection_id: str, - sample_stac_item: dict[str, Any], -) -> None: - items = items_to_arrow([sample_stac_item]) - iceberg_schema = get_schema_from_items(items) - missing_id = IcebergSchema( - *(field for field in iceberg_schema.fields if field.name != "id") - ) - - with pytest.raises(ValueError, match="missing required STAC fields.*'id'"): - test_catalog.create_item_table( - collection_id=test_collection_id, - iceberg_schema=missing_id, - ) - - -def test_create_item_table_rejects_unknown_partition_source( - test_catalog: IcestacCatalog, - test_collection_id: str, - sample_stac_item: dict[str, Any], -) -> None: - with pytest.raises(ValueError): - test_catalog.create_item_table( - collection_id=test_collection_id, - iceberg_schema=get_schema_from_items(items_to_arrow([sample_stac_item])), - partition_spec=PartitionSpec( - PartitionField( - source_id=9999, - field_id=1000, - transform=IdentityTransform(), - name="missing", - ) - ), - ) - - -def test_create_item_table_rejects_unknown_sort_source( - test_catalog: IcestacCatalog, - test_collection_id: str, - sample_stac_item: dict[str, Any], -) -> None: - with pytest.raises(ValueError): - test_catalog.create_item_table( - collection_id=test_collection_id, - iceberg_schema=get_schema_from_items(items_to_arrow([sample_stac_item])), - sort_order=SortOrder(SortField(source_id=9999)), - ) - - -def test_load_items_rejects_a_different_collection( - test_catalog: IcestacCatalog, - test_collection_id: str, - sample_stac_item: dict[str, Any], -) -> None: - items = items_to_arrow([sample_stac_item]) - iceberg_schema = get_schema_from_items(items) - table = test_catalog.create_item_table( - iceberg_schema=iceberg_schema, - collection_id=test_collection_id, - ) - different_collection = items.set_column( - items.schema.get_field_index("collection"), - "collection", - pyarrow.array(["different-collection"] * len(items)), - ) - - with pytest.raises(ValueError, match="different-collection"): - test_catalog.load_items(test_collection_id, different_collection) - - assert len(table.scan().to_arrow()) == 0 - - -def test_load_items( - test_catalog: IcestacCatalog, - test_collection_id: str, - items: pyarrow.Table, -) -> None: - expected_items = items_to_list(items) - iceberg_schema = get_schema_from_items(items) - table = test_catalog.create_item_table( - iceberg_schema=iceberg_schema, - collection_id=test_collection_id, - ) - - test_catalog.load_items( - collection_id=test_collection_id, - items=items, - method="upsert", - ) - - table.refresh() - - # Verify records were inserted - result = table.scan().to_arrow() - assert len(result) == len(expected_items) - assert result.column("id").to_pylist() == [item["id"] for item in expected_items] - - -def test_catalog_load_items_evolves_schema( - test_catalog: IcestacCatalog, - test_collection_id: str, - sample_stac_item: dict[str, Any], -) -> None: - initial = items_to_arrow([sample_stac_item]) - table = test_catalog.create_item_table( - iceberg_schema=get_schema_from_items(initial), - collection_id=test_collection_id, - ) - evolved_item = { - **sample_stac_item, - "properties": {**sample_stac_item["properties"], "new_field": True}, - } - - test_catalog.load_items( - collection_id=test_collection_id, - items=items_to_arrow([evolved_item]), - evolve_schema=True, - ) - - table.refresh() - assert table.schema().find_field("new_field") diff --git a/tests/test_load.py b/tests/test_load.py deleted file mode 100644 index 3757dbd..0000000 --- a/tests/test_load.py +++ /dev/null @@ -1,552 +0,0 @@ -from copy import deepcopy -from typing import Any - -import pyarrow as pa -import pytest -import rustac -from pyiceberg.exceptions import CommitFailedException - -from icestac.catalog import IcestacCatalog -from icestac.load import Method, load_items -from icestac.schema import get_schema_from_items, prepare_arrow_table -from tests.helpers import items_to_arrow, items_to_list - - -@pytest.mark.filterwarnings( - "error:Delete operation did not match any records:UserWarning" -) -def test_load_items_upsert_default( - test_catalog: IcestacCatalog, - test_collection_id: str, - items: pa.Table, -) -> None: - """Test loading items with default upsert method.""" - expected_items = items_to_list(items) - - # Create the table - iceberg_schema = get_schema_from_items(items) - table = test_catalog.create_item_table( - iceberg_schema=iceberg_schema, - collection_id=test_collection_id, - ) - - # Load items (default method is upsert) - load_items(items, table) - - # Verify records were inserted - result = table.scan().to_arrow() - assert len(result) == len(expected_items) - assert sorted(result.column("id").to_pylist()) == sorted( - item["id"] for item in expected_items - ) - - -def test_load_items_upsert_explicit( - test_catalog: IcestacCatalog, - items: pa.Table, -) -> None: - """Test loading items with explicit upsert method.""" - expected_items = items_to_list(items) - - # Create the table - iceberg_schema = get_schema_from_items(items) - table = test_catalog.create_item_table( - iceberg_schema=iceberg_schema, - collection_id=expected_items[0]["collection"], - ) - - # Load items with explicit upsert method - load_items(items, table, method="upsert") - - # Verify records were inserted - result = table.scan().to_arrow() - assert len(result) == len(expected_items) - - -def test_load_items_upsert_updates_existing( - test_catalog: IcestacCatalog, - items: pa.Table, -) -> None: - """Test that upsert updates existing records with same ID.""" - expected_items = items_to_list(items) - - # Create the table - iceberg_schema = get_schema_from_items(items) - table = test_catalog.create_item_table( - iceberg_schema=iceberg_schema, - collection_id=expected_items[0]["collection"], - ) - - # Load initial items - load_items(items, table, method="upsert") - - # Modify items (same IDs but different data) - modified_items = [] - for item in expected_items: - modified_item = item.copy() - modified_item["properties"] = item["properties"].copy() - modified_item["properties"]["title"] = f"Updated {item['properties']['title']}" - modified_items.append(modified_item) - - # Load modified items with upsert - load_items(items_to_arrow(modified_items), table, method="upsert") - - # Verify only the original record count exists and they have updated titles - result = table.scan().to_arrow() - assert len(result) == len(expected_items) - - # Check that titles were updated - titles = result.column("title").to_pylist() - assert all(title.startswith("Updated") for title in titles) - - -def test_upsert_reconstructs_evolved_existing_item( - test_catalog: IcestacCatalog, - sample_stac_item: dict[str, Any], -) -> None: - """An existing item can gain a top-level property during replacement.""" - initial = items_to_arrow([sample_stac_item]) - table = test_catalog.create_item_table( - iceberg_schema=get_schema_from_items(initial), - collection_id=sample_stac_item["collection"], - ) - load_items(initial, table) - - replacement = deepcopy(sample_stac_item) - replacement["properties"]["processing:software"] = {"version": "1.0"} - load_items(items_to_arrow([replacement]), table, evolve_schema=True) - - reconstructed = rustac.from_arrow(table.scan().to_arrow())["features"] - assert len(reconstructed) == 1 - assert reconstructed[0]["id"] == replacement["id"] - assert reconstructed[0]["properties"] == replacement["properties"] - - -def test_upsert_reconstructs_nested_asset_and_property_evolution( - test_catalog: IcestacCatalog, - sample_stac_item: dict[str, Any], -) -> None: - """Nested asset fields and flattened properties evolve in one replacement.""" - initial = items_to_arrow([sample_stac_item]) - table = test_catalog.create_item_table( - iceberg_schema=get_schema_from_items(initial), - collection_id=sample_stac_item["collection"], - ) - load_items(initial, table) - - replacement = deepcopy(sample_stac_item) - replacement["properties"]["processing:software"] = {"version": "1.0"} - replacement["assets"]["data"]["roles"] = ["data"] - replacement["assets"]["thumbnail"] = { - "href": "https://example.com/thumbnail.jpg", - "type": "image/jpeg", - } - load_items(items_to_arrow([replacement]), table, evolve_schema=True) - - reconstructed = rustac.from_arrow(table.scan().to_arrow())["features"][0] - assert reconstructed["properties"]["processing:software"] == {"version": "1.0"} - assert reconstructed["assets"] == replacement["assets"] - - -def test_upsert_clears_omitted_optional_fields( - test_catalog: IcestacCatalog, - sample_stac_item: dict[str, Any], -) -> None: - """Omitted optional values are cleared instead of being retained.""" - initial = items_to_arrow([sample_stac_item]) - table = test_catalog.create_item_table( - iceberg_schema=get_schema_from_items(initial), - collection_id=sample_stac_item["collection"], - ) - load_items(initial, table) - - replacement = deepcopy(sample_stac_item) - replacement["properties"] = {"datetime": sample_stac_item["properties"]["datetime"]} - replacement["assets"]["data"] = {"href": "https://example.com/replacement.tif"} - load_items(items_to_arrow([replacement]), table) - - reconstructed = rustac.from_arrow(table.scan().to_arrow())["features"][0] - assert "title" not in reconstructed["properties"] - assert reconstructed["assets"] == { - "data": {"href": replacement["assets"]["data"]["href"]} - } - - -def test_competing_upserts_fail_without_duplicate_ids( - test_catalog: IcestacCatalog, - sample_stac_item: dict[str, Any], -) -> None: - """A stale competing upsert fails instead of committing a duplicate.""" - initial = items_to_arrow([sample_stac_item]) - table = test_catalog.create_item_table( - iceberg_schema=get_schema_from_items(initial), - collection_id=sample_stac_item["collection"], - ) - first = test_catalog.catalog.load_table(table.name()) - second = test_catalog.catalog.load_table(table.name()) - arrow_table = prepare_arrow_table(items_to_arrow([sample_stac_item])) - - first_transaction = first.transaction() - second_transaction = second.transaction() - first_transaction.upsert(df=arrow_table, join_cols=["id"]) - second_transaction.upsert(df=arrow_table, join_cols=["id"]) - - first_transaction.commit_transaction() - with pytest.raises(CommitFailedException): - second_transaction.commit_transaction() - - second.refresh() - result = second.scan().to_arrow() - assert result.column("id").to_pylist() == [sample_stac_item["id"]] - - -def test_load_items_append( - test_catalog: IcestacCatalog, - items: pa.Table, -) -> None: - """Test loading items with append method.""" - expected_items = items_to_list(items) - - # Create the table - iceberg_schema = get_schema_from_items(items) - table = test_catalog.create_item_table( - iceberg_schema=iceberg_schema, - collection_id=expected_items[0]["collection"], - ) - - # Load items with append method - load_items(items, table, method="append") - - # Verify records were inserted - result = table.scan().to_arrow() - assert len(result) == len(expected_items) - - -def test_load_items_append_creates_duplicates( - test_catalog: IcestacCatalog, - items: pa.Table, -) -> None: - """Test that append creates duplicate records when IDs overlap.""" - expected_items = items_to_list(items) - - # Create the table - iceberg_schema = get_schema_from_items(items) - table = test_catalog.create_item_table( - iceberg_schema=iceberg_schema, - collection_id=expected_items[0]["collection"], - ) - - # Load items twice with append - load_items(items, table, method="append") - load_items(items, table, method="append") - - # Verify we have double the records (append doesn't deduplicate) - result = table.scan().to_arrow() - assert len(result) == len(expected_items) * 2 - - -def test_load_items_multiple_batches( - test_catalog: IcestacCatalog, - items: pa.Table, -) -> None: - """Test loading items in multiple batches with different methods.""" - expected_items = items_to_list(items) - - # Create the table - iceberg_schema = get_schema_from_items(items) - table = test_catalog.create_item_table( - iceberg_schema=iceberg_schema, - collection_id=expected_items[0]["collection"], - ) - - # Load first batch - load_items(items_to_arrow(expected_items[:2]), table, method="upsert") - - result = table.scan().to_arrow() - assert len(result) == min(2, len(expected_items)) - - # Load second batch - if len(expected_items) > 2: - load_items(items_to_arrow(expected_items[2:]), table, method="upsert") - - result = table.scan().to_arrow() - assert len(result) == len(expected_items) - - -def test_load_items_rejects_invalid_method( - test_catalog: IcestacCatalog, - sample_stac_item: dict[str, Any], -) -> None: - initial = items_to_arrow([sample_stac_item]) - iceberg_schema = get_schema_from_items(initial) - table = test_catalog.create_item_table( - iceberg_schema=iceberg_schema, - collection_id=sample_stac_item["collection"], - ) - - with pytest.raises(ValueError, match="Unsupported load method"): - load_items(initial, table, method="insert") # type: ignore[arg-type] - - -def test_load_items_supports_interval_datetime( - test_catalog: IcestacCatalog, - sample_stac_item: dict[str, Any], -) -> None: - interval_item = deepcopy(sample_stac_item) - interval_item["properties"] = { - "datetime": None, - "start_datetime": "2024-01-01T00:00:00Z", - "end_datetime": "2024-01-02T00:00:00Z", - } - interval_items = items_to_arrow([interval_item]) - iceberg_schema = get_schema_from_items(interval_items) - table = test_catalog.create_item_table( - iceberg_schema=iceberg_schema, - collection_id=interval_item["collection"], - ) - - load_items(interval_items, table) - - assert table.scan().to_arrow().column("id").to_pylist() == [interval_item["id"]] - - -def test_load_items_different_schema( - test_catalog: IcestacCatalog, - sample_stac_item: dict[str, Any], -) -> None: - initial = items_to_arrow([sample_stac_item]) - iceberg_schema = get_schema_from_items(initial) - table = test_catalog.create_item_table( - iceberg_schema=iceberg_schema, - collection_id=sample_stac_item["collection"], - ) - load_items(initial, table) - - item_new_schema = deepcopy(sample_stac_item) - item_new_schema["properties"]["new_field"] = True - with pytest.raises(ValueError, match="Update the schema first"): - load_items(items_to_arrow([item_new_schema]), table) - - -@pytest.mark.parametrize("method", ["append", "upsert"]) -def test_load_items_evolves_schema( - test_catalog: IcestacCatalog, - sample_stac_item: dict[str, Any], - method: Method, -) -> None: - initial = items_to_arrow([sample_stac_item]) - table = test_catalog.create_item_table( - iceberg_schema=get_schema_from_items(initial), - collection_id=sample_stac_item["collection"], - ) - load_items(initial, table) - - evolved_item = deepcopy(sample_stac_item) - evolved_item["id"] = "evolved-item" - evolved_item["properties"]["processing:software"] = { - "Atmospheric Correction": "6.0" - } - load_items(items_to_arrow([evolved_item]), table, method=method, evolve_schema=True) - - table.refresh() - assert table.schema().find_field("processing:software.Atmospheric Correction") - assert len(table.scan().to_arrow()) == 2 - - -def test_populated_link_fields_survive_reconstruction( - test_catalog: IcestacCatalog, - sample_stac_item: dict[str, Any], -) -> None: - item = deepcopy(sample_stac_item) - item["links"] = [ - { - "href": "https://example.com/query", - "rel": "data", - "type": "application/json", - "title": "Query", - "method": "POST", - "headers": {"content-type": "application/json"}, - "body": {"limit": 1}, - "merge": True, - } - ] - input_items = items_to_arrow([item]) - table = test_catalog.create_item_table( - iceberg_schema=get_schema_from_items(input_items), - collection_id=item["collection"], - ) - - load_items(input_items, table) - - reconstructed = rustac.from_arrow(table.scan().to_arrow())["features"][0] - assert reconstructed["links"] == item["links"] - - -def test_populated_links_require_evolution_after_empty_schema( - test_catalog: IcestacCatalog, - sample_stac_item: dict[str, Any], -) -> None: - empty_links = deepcopy(sample_stac_item) - empty_links["links"] = [] - empty_links_table = items_to_arrow([empty_links]) - table = test_catalog.create_item_table( - iceberg_schema=get_schema_from_items(empty_links_table), - collection_id=empty_links["collection"], - ) - load_items(empty_links_table, table) - - populated = deepcopy(sample_stac_item) - populated["id"] = "populated-links" - populated["links"] = [ - { - "href": "https://example.com/query", - "rel": "data", - "method": "POST", - "headers": {"content-type": "application/json"}, - "body": {"limit": 1}, - "merge": True, - } - ] - populated_table = items_to_arrow([populated]) - with pytest.raises(ValueError, match="Update the schema first"): - load_items(populated_table, table) - - load_items(populated_table, table, evolve_schema=True) - reconstructed = rustac.from_arrow(table.scan().to_arrow())["features"] - assert ( - next(item for item in reconstructed if item["id"] == populated["id"])["links"] - == populated["links"] - ) - - -def test_arrow_temporal_semantics_are_callers_responsibility( - test_catalog: IcestacCatalog, - sample_stac_item: dict[str, Any], -) -> None: - invalid = deepcopy(sample_stac_item) - invalid["properties"] = { - "datetime": None, - "start_datetime": "2024-01-02T00:00:00Z", - "end_datetime": "2024-01-01T00:00:00Z", - } - arrow_items = items_to_arrow([invalid]) - table = test_catalog.create_item_table( - iceberg_schema=get_schema_from_items(arrow_items), - collection_id=sample_stac_item["collection"], - ) - - load_items(arrow_items, table) - - assert table.scan().to_arrow().column("id").to_pylist() == [sample_stac_item["id"]] - - -def test_arrow_structural_edge_cases_follow_arrow_schema( - sample_stac_item: dict[str, Any], -) -> None: - arrow = items_to_arrow([sample_stac_item]) - geometry_index = arrow.schema.get_field_index("geometry") - null_geometry = arrow.set_column( - geometry_index, - "geometry", - pa.array([None], type=pa.binary()), - ) - - assert get_schema_from_items(null_geometry).find_field("geometry") - - -def test_load_items_accepts_arro3_table( - test_catalog: IcestacCatalog, - sample_stac_item: dict[str, Any], -) -> None: - items = rustac.to_arrow([sample_stac_item]) - table = test_catalog.create_item_table( - iceberg_schema=get_schema_from_items(items), - collection_id=sample_stac_item["collection"], - ) - - load_items(items, table) - - assert table.scan().to_arrow().column("id").to_pylist() == [sample_stac_item["id"]] - - -@pytest.mark.parametrize("method", ["append", "upsert"]) -def test_load_items_accepts_dictionary_columns( - test_catalog: IcestacCatalog, - items: pa.Table, - method: Method, -) -> None: - """Dictionary-encoded columns can be written without manual re-encoding.""" - for name in ("id", "collection", "title"): - items = items.set_column( - items.schema.get_field_index(name), name, items[name].dictionary_encode() - ) - table = test_catalog.create_item_table( - iceberg_schema=get_schema_from_items(items), - collection_id=items["collection"][0].as_py(), - ) - - load_items(items, table, method=method) - - result = table.scan().to_arrow() - assert sorted(zip(result["id"].to_pylist(), result["title"].to_pylist())) == sorted( - zip(items["id"].to_pylist(), items["title"].to_pylist()) - ) - - -def test_load_items_rejects_non_arrow_inputs( - test_catalog: IcestacCatalog, - sample_stac_item: dict[str, Any], -) -> None: - initial = items_to_arrow([sample_stac_item]) - table = test_catalog.create_item_table( - iceberg_schema=get_schema_from_items(initial), - collection_id=sample_stac_item["collection"], - ) - - with pytest.raises(TypeError, match="pyarrow.Table"): - load_items(sample_stac_item, table) - - -def test_load_items_rejects_invalid_arrow_schema( - test_catalog: IcestacCatalog, - items: pa.Table, -) -> None: - table = test_catalog.create_item_table( - iceberg_schema=get_schema_from_items(items), - collection_id="test-collection", - ) - id_index = items.schema.get_field_index("id") - invalid = items.set_column( - id_index, - "id", - pa.array([1, 2, 3], type=pa.int64()), - ) - - with pytest.raises(ValueError, match="Unsupported types for STAC fields: id"): - load_items(invalid, table) - assert len(table.scan().to_arrow()) == 0 - - -def test_load_items_does_not_evolve_schema_when_write_fails( - test_catalog: IcestacCatalog, - sample_stac_item: dict[str, Any], -) -> None: - initial = items_to_arrow([sample_stac_item]) - table = test_catalog.create_item_table( - iceberg_schema=get_schema_from_items(initial), - collection_id=sample_stac_item["collection"], - ) - load_items(initial, table) - before = rustac.from_arrow(table.scan().to_arrow())["features"] - - evolved_item = deepcopy(sample_stac_item) - evolved_item["properties"]["new_field"] = True - - evolved = items_to_arrow([evolved_item, evolved_item]) - with pytest.raises(ValueError, match="Duplicate rows"): - load_items(evolved, table, evolve_schema=True) - - table.refresh() - assert rustac.from_arrow(table.scan().to_arrow())["features"] == before - with pytest.raises(ValueError, match="Could not find field"): - table.schema().find_field("new_field") diff --git a/tests/test_main.py b/tests/test_main.py index 2d560ef..c36d1f4 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -26,7 +26,7 @@ def test_demo_loads_cached_batches_and_can_rerun( ) -> None: """Load real Parquet through DuckDB and Iceberg, including schema evolution.""" monkeypatch.chdir(tmp_path) - monkeypatch.setattr(main, "load_catalog", lambda: test_catalog.catalog) + monkeypatch.setattr(main, "load_catalog", lambda: test_catalog) monkeypatch.setattr(main, "DuckdbClient", lambda: duckdb_client) for month in range(1, 9): batch = [] @@ -44,7 +44,7 @@ def test_demo_loads_cached_batches_and_can_rerun( ) path.parent.mkdir(parents=True) pq.write_table(items_to_arrow(batch), path) - prepared = main.read_hls_items(duckdb_client, path, "HLSS30_2_0") + prepared = main.read_hls_items(duckdb_client, path) keys = prepared["hilbert_idx"].to_pylist() assert keys == sorted(keys) assert len(set(keys)) == 3 @@ -55,11 +55,14 @@ def test_demo_loads_cached_batches_and_can_rerun( asyncio.run(main.run()) asyncio.run(main.run()) - table = test_catalog.catalog.load_table(("icestac", "HLSS30_2_0")) + table = test_catalog.load_table(("icestac", "HLSS30_2.0")) result = table.scan().to_arrow() assert len(result) == 24 assert len(result["id"].unique()) == 24 - assert result["collection"].unique().to_pylist() == ["HLSS30_2_0"] + assert result["collection"].unique().to_pylist() == ["HLSS30_2.0"] + assert table.schema().identifier_field_ids == [ + table.schema().find_field("id").field_id + ] assert result["new_field"].null_count == 15 assert table.properties[TableProperties.PARQUET_ROW_GROUP_LIMIT] == "50000" assert ( @@ -91,4 +94,4 @@ def test_read_hls_items_rejects_invalid_batches( pq.write_table(items, path) with pytest.raises(ValueError, match="No items found|bbox coordinates"): - main.read_hls_items(duckdb_client, path, "test-collection") + main.read_hls_items(duckdb_client, path) diff --git a/tests/test_schema.py b/tests/test_schema.py index 6f7c205..3df4438 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -4,7 +4,7 @@ from pyiceberg.types import ListType import rustac -from icestac.schema import IcestacItem, get_schema_from_items +from icestac.schema import get_schema_from_items, validate_schema def test_get_schema_from_items(items: pa.Table) -> None: @@ -14,6 +14,7 @@ def test_get_schema_from_items(items: pa.Table) -> None: assert isinstance(schema, Schema) assert schema.find_field("title").field_id > 0 assert schema.find_field("id").required + assert schema.identifier_field_ids == [schema.find_field("id").field_id] assert str(schema.find_field("geometry").field_type) == "binary" nested_paths = ( @@ -74,6 +75,14 @@ def test_get_schema_from_items_rejects_invalid_field_type(items: pa.Table) -> No get_schema_from_items(invalid) +def test_get_schema_from_items_rejects_non_string_type(items: pa.Table) -> None: + index = items.schema.get_field_index("type") + invalid = items.set_column(index, "type", pa.array([1, 2, 3], type=pa.int64())) + + with pytest.raises(ValueError, match="Unsupported types for STAC fields: type"): + get_schema_from_items(invalid) + + def test_get_schema_from_items_rejects_null_required_field(items: pa.Table) -> None: index = items.schema.get_field_index("id") invalid = items.set_column( @@ -98,7 +107,7 @@ def test_get_schema_from_items_rejects_null_links(items: pa.Table) -> None: def test_validate_schema_valid(items: pa.Table) -> None: """A valid inferred schema passes structural validation.""" - IcestacItem.validate_schema(get_schema_from_items(items)) + validate_schema(get_schema_from_items(items)) def test_validate_schema_missing_required_field() -> None: @@ -112,7 +121,7 @@ def test_validate_schema_missing_required_field() -> None: ) with pytest.raises(ValueError, match="missing required STAC fields.*'id'"): - IcestacItem.validate_schema(schema) + validate_schema(schema) def test_validate_schema_rejects_invalid_field_type() -> None: @@ -129,4 +138,4 @@ def test_validate_schema_rejects_invalid_field_type() -> None: ) with pytest.raises(ValueError, match="Unsupported types for STAC fields: id"): - IcestacItem.validate_schema(schema) + validate_schema(schema) diff --git a/tests/test_write.py b/tests/test_write.py new file mode 100644 index 0000000..128c96f --- /dev/null +++ b/tests/test_write.py @@ -0,0 +1,673 @@ +from concurrent.futures import ThreadPoolExecutor +from copy import deepcopy +from threading import Barrier +from typing import Any + +import pyarrow as pa +import pytest +import rustac +from pyiceberg.catalog import Catalog +from pyiceberg.exceptions import CommitFailedException +from pyiceberg.table import Table, TableProperties + +from icestac.constants import COLLECTIONS_TABLE_NAME, DEFAULT_NAMESPACE +from icestac.schema import get_schema_from_items +from icestac.write import put_items +from tests.helpers import items_to_arrow, items_to_list + + +@pytest.mark.filterwarnings( + "error:Delete operation did not match any records:UserWarning" +) +def test_put_items_inserts_items( + test_catalog: Catalog, + test_collection_id: str, + items: pa.Table, +) -> None: + """Insert a batch into a native Iceberg table.""" + expected_items = items_to_list(items) + + # Create the table + iceberg_schema = get_schema_from_items(items) + assert iceberg_schema.identifier_field_ids == [ + iceberg_schema.find_field("id").field_id + ] + table = test_catalog.create_table( + schema=iceberg_schema, + identifier=(DEFAULT_NAMESPACE, test_collection_id), + ) + + # Put items through the standalone writer. + put_items(table, items) + + # Verify records were inserted + result = table.scan().to_arrow() + assert len(result) == len(expected_items) + assert sorted(result.column("id").to_pylist()) == sorted( + item["id"] for item in expected_items + ) + + +def test_put_items_accepts_dotted_collection_ids( + test_catalog: Catalog, + sample_stac_item: dict[str, Any], +) -> None: + item = deepcopy(sample_stac_item) + item["collection"] = "foo.bar" + items = items_to_arrow([item]) + table = test_catalog.create_table( + schema=get_schema_from_items(items), + identifier=(DEFAULT_NAMESPACE, item["collection"]), + ) + + put_items(table, items) + + assert table.name() == (DEFAULT_NAMESPACE, "foo.bar") + assert table.scan().to_arrow().column("collection").to_pylist() == ["foo.bar"] + + +def test_put_items_rejects_different_collection( + test_catalog: Catalog, + sample_stac_item: dict[str, Any], +) -> None: + items = items_to_arrow([sample_stac_item]) + table = test_catalog.create_table( + schema=get_schema_from_items(items), + identifier=(DEFAULT_NAMESPACE, sample_stac_item["collection"]), + ) + different = deepcopy(sample_stac_item) + different["collection"] = "other-collection" + + with pytest.raises(ValueError, match="other-collection"): + put_items(table, items_to_arrow([different])) + + assert len(table.scan().to_arrow()) == 0 + + +def test_put_items_rejects_reserved_collections_table( + test_catalog: Catalog, + sample_stac_item: dict[str, Any], +) -> None: + initial = items_to_arrow([sample_stac_item]) + table = test_catalog.create_table( + schema=get_schema_from_items(initial), + identifier=(DEFAULT_NAMESPACE, COLLECTIONS_TABLE_NAME), + ) + evolved = deepcopy(sample_stac_item) + evolved["properties"]["new_field"] = True + + with pytest.raises(ValueError, match="reserved table 'collections'"): + put_items(table, items_to_arrow([evolved]), evolve_schema=True) + + table.refresh() + assert len(table.scan().to_arrow()) == 0 + with pytest.raises(ValueError, match="Could not find field"): + table.schema().find_field("new_field") + + +def test_put_items_inserts_items_explicitly( + test_catalog: Catalog, + items: pa.Table, +) -> None: + """Insert a batch through the standalone writer.""" + expected_items = items_to_list(items) + + # Create the table + iceberg_schema = get_schema_from_items(items) + table = test_catalog.create_table( + schema=iceberg_schema, + identifier=(DEFAULT_NAMESPACE, expected_items[0]["collection"]), + ) + + # Put items through the standalone writer. + put_items(table, items) + + # Verify records were inserted + result = table.scan().to_arrow() + assert len(result) == len(expected_items) + + +def test_put_items_replaces_existing_items( + test_catalog: Catalog, + items: pa.Table, +) -> None: + """Replace existing records with the same ID.""" + expected_items = items_to_list(items) + + # Create the table + iceberg_schema = get_schema_from_items(items) + table = test_catalog.create_table( + schema=iceberg_schema, + identifier=(DEFAULT_NAMESPACE, expected_items[0]["collection"]), + ) + + # Put initial items + put_items(table, items) + + # Modify items (same IDs but different data) + modified_items = [] + for item in expected_items: + modified_item = item.copy() + modified_item["properties"] = item["properties"].copy() + modified_item["properties"]["title"] = f"Updated {item['properties']['title']}" + modified_items.append(modified_item) + + # Replace the existing items. + put_items(table, items_to_arrow(modified_items)) + + # Verify only the original record count exists and they have updated titles + result = table.scan().to_arrow() + assert len(result) == len(expected_items) + + # Check that titles were updated + titles = result.column("title").to_pylist() + assert all(title.startswith("Updated") for title in titles) + + +def test_put_items_reconstructs_evolved_existing_item( + test_catalog: Catalog, + sample_stac_item: dict[str, Any], +) -> None: + """An existing item can gain a top-level property during replacement.""" + initial = items_to_arrow([sample_stac_item]) + table = test_catalog.create_table( + schema=get_schema_from_items(initial), + identifier=(DEFAULT_NAMESPACE, sample_stac_item["collection"]), + ) + put_items(table, initial) + + replacement = deepcopy(sample_stac_item) + replacement["properties"]["processing:software"] = {"version": "1.0"} + put_items(table, items_to_arrow([replacement]), evolve_schema=True) + + reconstructed = rustac.from_arrow(table.scan().to_arrow())["features"] + assert len(reconstructed) == 1 + assert reconstructed[0]["id"] == replacement["id"] + assert reconstructed[0]["properties"] == replacement["properties"] + + +def test_put_items_reconstructs_nested_asset_and_property_evolution( + test_catalog: Catalog, + sample_stac_item: dict[str, Any], +) -> None: + """Nested asset fields and flattened properties evolve in one replacement.""" + initial = items_to_arrow([sample_stac_item]) + table = test_catalog.create_table( + schema=get_schema_from_items(initial), + identifier=(DEFAULT_NAMESPACE, sample_stac_item["collection"]), + ) + put_items(table, initial) + + replacement = deepcopy(sample_stac_item) + replacement["properties"]["processing:software"] = {"version": "1.0"} + replacement["assets"]["data"]["roles"] = ["data"] + replacement["assets"]["thumbnail"] = { + "href": "https://example.com/thumbnail.jpg", + "type": "image/jpeg", + } + put_items(table, items_to_arrow([replacement]), evolve_schema=True) + + reconstructed = rustac.from_arrow(table.scan().to_arrow())["features"][0] + assert reconstructed["properties"]["processing:software"] == {"version": "1.0"} + assert reconstructed["assets"] == replacement["assets"] + + +def test_put_items_clears_omitted_optional_fields( + test_catalog: Catalog, + sample_stac_item: dict[str, Any], +) -> None: + """Omitted optional values are cleared instead of being retained.""" + initial = items_to_arrow([sample_stac_item]) + table = test_catalog.create_table( + schema=get_schema_from_items(initial), + identifier=(DEFAULT_NAMESPACE, sample_stac_item["collection"]), + ) + put_items(table, initial) + + replacement = deepcopy(sample_stac_item) + replacement["properties"] = {"datetime": sample_stac_item["properties"]["datetime"]} + replacement["assets"]["data"] = {"href": "https://example.com/replacement.tif"} + put_items(table, items_to_arrow([replacement])) + + reconstructed = rustac.from_arrow(table.scan().to_arrow())["features"][0] + assert "title" not in reconstructed["properties"] + assert reconstructed["assets"] == { + "data": {"href": replacement["assets"]["data"]["href"]} + } + + +@pytest.mark.parametrize("populate_first", [False, True]) +def test_competing_replacement_writes_do_not_duplicate_ids( + test_catalog: Catalog, + sample_stac_item: dict[str, Any], + monkeypatch: pytest.MonkeyPatch, + populate_first: bool, +) -> None: + """Concurrent replacement writes fail rather than replay stale matches.""" + initial = items_to_arrow([sample_stac_item]) + table = test_catalog.create_table( + schema=get_schema_from_items(initial), + identifier=(DEFAULT_NAMESPACE, sample_stac_item["collection"]), + ) + if populate_first: + put_items(table, initial) + first = test_catalog.load_table(table.name()) + second = test_catalog.load_table(table.name()) + assert TableProperties.COMMIT_NUM_RETRIES not in first.metadata.properties + + first_item = deepcopy(sample_stac_item) + first_item["properties"]["title"] = "First" + second_item = deepcopy(sample_stac_item) + second_item["properties"]["title"] = "Second" + barrier = Barrier(2) + original_commit = Table._do_commit + + def synchronized_commit(self, updates, requirements): + barrier.wait(timeout=10) + return original_commit(self, updates, requirements) + + monkeypatch.setattr(Table, "_do_commit", synchronized_commit) + + def write(item: dict[str, Any], target: Table) -> Exception | None: + try: + put_items(target, items_to_arrow([item])) + except Exception as error: # noqa: BLE001 + return error + return None + + with ThreadPoolExecutor(max_workers=2) as executor: + errors = list( + executor.map( + lambda args: write(*args), + ((first_item, first), (second_item, second)), + ) + ) + + assert sum(error is not None for error in errors) == 1 + assert any(isinstance(error, CommitFailedException) for error in errors) + table.refresh() + result = table.scan().to_arrow() + assert len(result) == 1 + assert result.column("id").to_pylist() == [sample_stac_item["id"]] + + +def test_failed_put_restores_retry_policy( + test_catalog: Catalog, + sample_stac_item: dict[str, Any], + monkeypatch: pytest.MonkeyPatch, +) -> None: + items = items_to_arrow([sample_stac_item]) + retry_property = TableProperties.COMMIT_NUM_RETRIES + table = test_catalog.create_table( + schema=get_schema_from_items(items), + identifier=(DEFAULT_NAMESPACE, sample_stac_item["collection"]), + properties={retry_property: "7"}, + ) + + def fail_commit(self, updates, requirements): + raise CommitFailedException("controlled commit failure") + + monkeypatch.setattr(Table, "_do_commit", fail_commit) + with pytest.raises(CommitFailedException, match="controlled commit failure"): + put_items(table, items) + + assert table.metadata.properties[retry_property] == "7" + + +def test_successful_put_restores_retry_policy( + test_catalog: Catalog, + sample_stac_item: dict[str, Any], +) -> None: + items = items_to_arrow([sample_stac_item]) + retry_property = TableProperties.COMMIT_NUM_RETRIES + table = test_catalog.create_table( + schema=get_schema_from_items(items), + identifier=(DEFAULT_NAMESPACE, sample_stac_item["collection"]), + properties={retry_property: "7"}, + ) + + put_items(table, items) + + assert table.metadata.properties[retry_property] == "7" + + +def test_failed_put_does_not_overwrite_refreshed_retry_policy( + test_catalog: Catalog, + sample_stac_item: dict[str, Any], + monkeypatch: pytest.MonkeyPatch, +) -> None: + items = items_to_arrow([sample_stac_item]) + retry_property = TableProperties.COMMIT_NUM_RETRIES + table = test_catalog.create_table( + schema=get_schema_from_items(items), + identifier=(DEFAULT_NAMESPACE, sample_stac_item["collection"]), + properties={retry_property: "7"}, + ) + + def refresh_then_fail(self, updates, requirements): + self.refresh() + raise CommitFailedException("controlled commit failure") + + monkeypatch.setattr(Table, "_do_commit", refresh_then_fail) + with pytest.raises(CommitFailedException, match="controlled commit failure"): + put_items(table, items) + + assert table.metadata.properties[retry_property] == "7" + + +def test_put_items_multiple_batches( + test_catalog: Catalog, + items: pa.Table, +) -> None: + """Put multiple batches into one table.""" + expected_items = items_to_list(items) + + # Create the table + iceberg_schema = get_schema_from_items(items) + table = test_catalog.create_table( + schema=iceberg_schema, + identifier=(DEFAULT_NAMESPACE, expected_items[0]["collection"]), + ) + + # Put the first batch + put_items(table, items_to_arrow(expected_items[:2])) + + result = table.scan().to_arrow() + assert len(result) == min(2, len(expected_items)) + + # Put the second batch + if len(expected_items) > 2: + put_items(table, items_to_arrow(expected_items[2:])) + + result = table.scan().to_arrow() + assert len(result) == len(expected_items) + + +def test_put_items_supports_interval_datetime( + test_catalog: Catalog, + sample_stac_item: dict[str, Any], +) -> None: + interval_item = deepcopy(sample_stac_item) + interval_item["properties"] = { + "datetime": None, + "start_datetime": "2024-01-01T00:00:00Z", + "end_datetime": "2024-01-02T00:00:00Z", + } + interval_items = items_to_arrow([interval_item]) + iceberg_schema = get_schema_from_items(interval_items) + table = test_catalog.create_table( + schema=iceberg_schema, + identifier=(DEFAULT_NAMESPACE, interval_item["collection"]), + ) + + put_items(table, interval_items) + + assert table.scan().to_arrow().column("id").to_pylist() == [interval_item["id"]] + + +def test_put_items_different_schema( + test_catalog: Catalog, + sample_stac_item: dict[str, Any], +) -> None: + initial = items_to_arrow([sample_stac_item]) + iceberg_schema = get_schema_from_items(initial) + table = test_catalog.create_table( + schema=iceberg_schema, + identifier=(DEFAULT_NAMESPACE, sample_stac_item["collection"]), + ) + put_items(table, initial) + + item_new_schema = deepcopy(sample_stac_item) + item_new_schema["properties"]["new_field"] = True + with pytest.raises(ValueError, match="Update the schema first"): + put_items(table, items_to_arrow([item_new_schema])) + + +def test_put_items_evolves_schema( + test_catalog: Catalog, + sample_stac_item: dict[str, Any], +) -> None: + initial = items_to_arrow([sample_stac_item]) + table = test_catalog.create_table( + schema=get_schema_from_items(initial), + identifier=(DEFAULT_NAMESPACE, sample_stac_item["collection"]), + ) + put_items(table, initial) + + evolved_item = deepcopy(sample_stac_item) + evolved_item["id"] = "evolved-item" + evolved_item["properties"]["processing:software"] = { + "Atmospheric Correction": "6.0" + } + put_items(table, items_to_arrow([evolved_item]), evolve_schema=True) + + table.refresh() + assert table.schema().find_field("processing:software.Atmospheric Correction") + assert len(table.scan().to_arrow()) == 2 + + +def test_populated_link_fields_survive_reconstruction( + test_catalog: Catalog, + sample_stac_item: dict[str, Any], +) -> None: + item = deepcopy(sample_stac_item) + item["links"] = [ + { + "href": "https://example.com/query", + "rel": "data", + "type": "application/json", + "title": "Query", + "method": "POST", + "headers": {"content-type": "application/json"}, + "body": {"limit": 1}, + "merge": True, + } + ] + input_items = items_to_arrow([item]) + table = test_catalog.create_table( + schema=get_schema_from_items(input_items), + identifier=(DEFAULT_NAMESPACE, item["collection"]), + ) + + put_items(table, input_items) + + reconstructed = rustac.from_arrow(table.scan().to_arrow())["features"][0] + assert reconstructed["links"] == item["links"] + + +def test_put_items_clears_extended_links_without_schema_evolution( + test_catalog: Catalog, + sample_stac_item: dict[str, Any], +) -> None: + populated = deepcopy(sample_stac_item) + populated["links"] = [ + { + "href": "https://example.com/query", + "rel": "data", + "method": "POST", + "headers": {"content-type": "application/json"}, + "body": {"limit": 1}, + "merge": True, + } + ] + populated_table = items_to_arrow([populated]) + table = test_catalog.create_table( + schema=get_schema_from_items(populated_table), + identifier=(DEFAULT_NAMESPACE, populated["collection"]), + ) + put_items(table, populated_table) + + replacement = deepcopy(populated) + replacement["links"] = [] + put_items(table, items_to_arrow([replacement])) + + reconstructed = rustac.from_arrow(table.scan().to_arrow())["features"][0] + assert reconstructed["links"] == [] + + +def test_populated_links_require_evolution_after_empty_schema( + test_catalog: Catalog, + sample_stac_item: dict[str, Any], +) -> None: + empty_links = deepcopy(sample_stac_item) + empty_links["links"] = [] + empty_links_table = items_to_arrow([empty_links]) + table = test_catalog.create_table( + schema=get_schema_from_items(empty_links_table), + identifier=(DEFAULT_NAMESPACE, empty_links["collection"]), + ) + put_items(table, empty_links_table) + + populated = deepcopy(sample_stac_item) + populated["id"] = "populated-links" + populated["links"] = [ + { + "href": "https://example.com/query", + "rel": "data", + "method": "POST", + "headers": {"content-type": "application/json"}, + "body": {"limit": 1}, + "merge": True, + } + ] + populated_table = items_to_arrow([populated]) + with pytest.raises(ValueError, match="Update the schema first"): + put_items(table, populated_table) + + put_items(table, populated_table, evolve_schema=True) + reconstructed = rustac.from_arrow(table.scan().to_arrow())["features"] + assert ( + next(item for item in reconstructed if item["id"] == populated["id"])["links"] + == populated["links"] + ) + + +def test_arrow_temporal_semantics_are_callers_responsibility( + test_catalog: Catalog, + sample_stac_item: dict[str, Any], +) -> None: + invalid = deepcopy(sample_stac_item) + invalid["properties"] = { + "datetime": None, + "start_datetime": "2024-01-02T00:00:00Z", + "end_datetime": "2024-01-01T00:00:00Z", + } + arrow_items = items_to_arrow([invalid]) + table = test_catalog.create_table( + schema=get_schema_from_items(arrow_items), + identifier=(DEFAULT_NAMESPACE, sample_stac_item["collection"]), + ) + + put_items(table, arrow_items) + + assert table.scan().to_arrow().column("id").to_pylist() == [sample_stac_item["id"]] + + +def test_arrow_structural_edge_cases_follow_arrow_schema( + sample_stac_item: dict[str, Any], +) -> None: + arrow = items_to_arrow([sample_stac_item]) + geometry_index = arrow.schema.get_field_index("geometry") + null_geometry = arrow.set_column( + geometry_index, + "geometry", + pa.array([None], type=pa.binary()), + ) + + assert get_schema_from_items(null_geometry).find_field("geometry") + + +def test_put_items_accepts_arro3_table( + test_catalog: Catalog, + sample_stac_item: dict[str, Any], +) -> None: + items = rustac.to_arrow([sample_stac_item]) + table = test_catalog.create_table( + schema=get_schema_from_items(items), + identifier=(DEFAULT_NAMESPACE, sample_stac_item["collection"]), + ) + + put_items(table, items) + + assert table.scan().to_arrow().column("id").to_pylist() == [sample_stac_item["id"]] + + +def test_put_items_accepts_dictionary_columns( + test_catalog: Catalog, + items: pa.Table, +) -> None: + """Dictionary-encoded columns can be written without manual re-encoding.""" + for name in ("id", "collection", "title"): + items = items.set_column( + items.schema.get_field_index(name), name, items[name].dictionary_encode() + ) + table = test_catalog.create_table( + schema=get_schema_from_items(items), + identifier=(DEFAULT_NAMESPACE, items["collection"][0].as_py()), + ) + + put_items(table, items) + + result = table.scan().to_arrow() + assert sorted(zip(result["id"].to_pylist(), result["title"].to_pylist())) == sorted( + zip(items["id"].to_pylist(), items["title"].to_pylist()) + ) + + +def test_put_items_rejects_non_arrow_inputs( + test_catalog: Catalog, + sample_stac_item: dict[str, Any], +) -> None: + initial = items_to_arrow([sample_stac_item]) + table = test_catalog.create_table( + schema=get_schema_from_items(initial), + identifier=(DEFAULT_NAMESPACE, sample_stac_item["collection"]), + ) + + with pytest.raises(TypeError, match="pyarrow.Table"): + put_items(table, sample_stac_item) + + +def test_put_items_rejects_invalid_arrow_schema( + test_catalog: Catalog, + items: pa.Table, +) -> None: + table = test_catalog.create_table( + schema=get_schema_from_items(items), + identifier=(DEFAULT_NAMESPACE, "test-collection"), + ) + id_index = items.schema.get_field_index("id") + invalid = items.set_column( + id_index, + "id", + pa.array([1, 2, 3], type=pa.int64()), + ) + + with pytest.raises(ValueError, match="Unsupported types for STAC fields: id"): + put_items(table, invalid) + assert len(table.scan().to_arrow()) == 0 + + +def test_put_items_does_not_evolve_schema_when_write_fails( + test_catalog: Catalog, + sample_stac_item: dict[str, Any], +) -> None: + initial = items_to_arrow([sample_stac_item]) + table = test_catalog.create_table( + schema=get_schema_from_items(initial), + identifier=(DEFAULT_NAMESPACE, sample_stac_item["collection"]), + ) + put_items(table, initial) + before = rustac.from_arrow(table.scan().to_arrow())["features"] + + evolved_item = deepcopy(sample_stac_item) + evolved_item["properties"]["new_field"] = True + + evolved = items_to_arrow([evolved_item, evolved_item]) + with pytest.raises(ValueError, match="Duplicate rows"): + put_items(table, evolved, evolve_schema=True) + + table.refresh() + assert rustac.from_arrow(table.scan().to_arrow())["features"] == before + with pytest.raises(ValueError, match="Could not find field"): + table.schema().find_field("new_field")