diff --git a/README.md b/README.md index cf0fb10..773e3ce 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,9 @@ print(client.get_status(job_id)) [Read the full quickstart](https://tributo.readthedocs.io/en/latest/getting-started/quickstart/). +For a sealed Ray runtime containing the first-party dependency closure, see +the [runtime image guide](https://tributo.readthedocs.io/en/latest/how-to/runtime-images/). + --- ## Architecture @@ -88,6 +91,9 @@ uv sync # With XGBoost training + ONNX export uv sync --extra training +# With BayesOpt search for Ray Tune +uv sync --extra tune + # With data formats (Lance / Iceberg) uv sync --extra data @@ -117,14 +123,15 @@ dialect or backend you use: | Local/S3 Iceberg and Lance | Ray Data / Daft public readers | `tributo[data,data-daft]` | Alpha; real dual-engine Conformance | | PostgreSQL structured table | Ray Data / Daft SQL readers | `tributo[postgresql,data-daft]` | Alpha; real PostgreSQL Conformance | | HDFS Parquet/CSV | Ray Data + PyArrow Hadoop filesystem | Ray runtime with HDFS libraries | Adapter present; cluster gate pending | -| ClickHouse | independent local `daft-clickhouse` wheel | `tributo[clickhouse]` plus the connector wheel | Adapter present; package/infrastructure gates pending | -| Doris | independent `ray-doris` / local `daft-doris` wheel | `tributo[mysql]` or `tributo[doris-flight]` plus the connector wheel | Adapters present; package/infrastructure gates pending | +| ClickHouse | `daft-clickhouse==1.0` | `tributo[clickhouse]` or `uv sync --extra clickhouse` | Adapter only; the full image contains the v1.0 package, while real-database Conformance remains the support gate | +| Doris | `ray-doris==1.0` / `daft-doris==1.0` | `tributo[mysql]` or `tributo[doris-flight]` | Adapter only; Ray routes require `ray-doris`, Daft routes require `daft-doris`, and real-database Conformance remains the support gate | | ORC / Hive external tables | no locked public reader path | — | Unsupported | -Provider/binding presence is not a support claim. ClickHouse, Doris, HDFS, and Hive are -reported as available only after their locked external dependencies and real -infrastructure gates pass. Tributo never installs optional providers or bindings at -runtime. +Provider/binding presence is not a support claim. The canonical full image +contains the locked v1.0 ClickHouse and Doris connector packages, but those +paths remain adapter-only until their real-database Conformance gates pass. +HDFS and Hive still require their own external dependency and infrastructure +gates. Tributo never installs optional providers or bindings at runtime. --- @@ -204,6 +211,9 @@ uv run tributo serve streaming status ### Hyperparameter tuning with Ray Tune Random search / BayesOpt with FIFO / ASHA / HyperBand schedulers. +BayesOpt requires the optional `tune` extra (`uv sync --extra tune` or +`python -m pip install "tributo[tune]"`); the full runtime image already +contains it. Tune trials execute setup and fit only: they report the configured metric and checkpoint without publishing production Bundles. After selecting parameters, run the Trainer explicitly to publish the single production Bundle. diff --git a/ci/test-suites.json b/ci/test-suites.json index aa4b77e..0f155e8 100644 --- a/ci/test-suites.json +++ b/ci/test-suites.json @@ -57,6 +57,15 @@ "patterns": ["src/**"], "domains": ["code", "docs", "storage"] }, + { + "name": "runtime-image", + "patterns": [ + "docker/**", + "tools/tributo-runtime-full.json", + "tools/build_tributo_image.py" + ], + "domains": ["code", "storage"] + }, { "name": "tests", "patterns": ["tests/**", "scripts/**", "tools/**"], @@ -160,6 +169,45 @@ "log_contract": "pytest terminal output; skips and missing JUnit evidence fail the suite", "rationale": "Dedicated bounded shard for integration-named modules that exercise local contracts without external infrastructure." }, + { + "id": "runtime-image", + "owner": "core", + "domain": "runtime-image", + "tier": "manual_external", + "workflow": "external", + "owns_tests": true, + "entrypoint": ["bash", "scripts/run_runtime_image_it.sh"], + "args": [], + "extras": ["dev"], + "test_paths": [ + "docker/tributo-runtime/Dockerfile", + "scripts/run_runtime_image_it.sh", + "tests/integrations/docker-compose.runtime-image.yml", + "tests/integrations/jobs/runtime_image_gate_job.py", + "tests/tools/test_build_tributo_image.py", + "tools/build_tributo_image.py", + "tools/tributo-runtime-full.json" + ], + "trigger_domains": [], + "trigger_paths": [ + "docker/tributo-runtime/**", + "scripts/run_runtime_image_it.sh", + "tests/integrations/docker-compose.runtime-image.yml", + "tests/integrations/jobs/runtime_image_gate_job.py", + "tests/integrations/runtime-profiles.json", + "tests/tools/test_build_tributo_image.py", + "tools/build_tributo_image.py", + "tools/tributo-runtime-full.json", + "pyproject.toml", + "uv.lock" + ], + "requires": ["docker", "multi_node_ray", "ray_jobs"], + "budget_seconds": 1800, + "ci_allowed": false, + "forbid_skips": true, + "log_contract": "/tmp/tributo-runtime-image-it-*/compose.log and image attestations", + "rationale": "The pinned full image and its Ray Jobs worker closure require one isolated multi-node Docker gate." + }, { "id": "documentation-api", "owner": "docs", diff --git a/docker/tributo-runtime/Dockerfile b/docker/tributo-runtime/Dockerfile new file mode 100644 index 0000000..1b8fe1d --- /dev/null +++ b/docker/tributo-runtime/Dockerfile @@ -0,0 +1,118 @@ +ARG BASE_IMAGE=tributo-ray-base:2.55.1-py312 +ARG UV_IMAGE=tributo-uv:0.11.23 +ARG TRIBUTO_BASE_IMAGE=rayproject/ray:2.55.1-py312@sha256:911245f2478ad2e9f67ac13978dc2a75bcae0498b9f188b10bba703324b78379 +ARG TRIBUTO_PLATFORM + +FROM ${UV_IMAGE} AS uv +FROM ${BASE_IMAGE} + +ARG BASE_IMAGE +ARG TRIBUTO_BASE_IMAGE +ARG TRIBUTO_PLATFORM +ARG TRIBUTO_MANIFEST_SHA256=unsealed +ARG TRIBUTO_RUNTIME_EXTRAS +ARG TRIBUTO_VERSION=1.0.0 + +LABEL org.opencontainers.image.title="tributo-runtime-full" \ + org.opencontainers.image.version="${TRIBUTO_VERSION}" \ + org.tributo.base-image="${TRIBUTO_BASE_IMAGE}" \ + org.tributo.ray-version="2.55.1" \ + org.tributo.python-version="3.12" \ + org.tributo.platform="${TRIBUTO_PLATFORM}" \ + org.tributo.runtime-extras="${TRIBUTO_RUNTIME_EXTRAS}" \ + org.tributo.manifest-sha256="${TRIBUTO_MANIFEST_SHA256}" + +COPY --from=uv /uv /uvx /usr/local/bin/ + +USER root +RUN mkdir -p /opt/tributo /opt/tributo-image /workspace && \ + chown -R ray:users /opt/tributo /opt/tributo-image /workspace + +# Some locked Alpha dependencies, notably econml on linux/arm64, do not ship +# a compatible wheel and must build their C extensions in the image. +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends \ + build-essential && \ + rm -rf /var/lib/apt/lists/* + +USER ray + +ENV PATH="/opt/tributo/.venv/bin:${PATH}" \ + PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + UV_HTTP_RETRIES=10 \ + UV_HTTP_TIMEOUT=600 \ + UV_LINK_MODE=copy \ + UV_PROJECT_ENVIRONMENT=/opt/tributo/.venv + +COPY --chown=ray:users pyproject.toml uv.lock README.md /opt/tributo/ +COPY --chown=ray:users src /opt/tributo/src + +WORKDIR /opt/tributo + +RUN --mount=type=cache,target=/home/ray/.cache/uv,uid=1000,gid=100 \ + env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY \ + -u http_proxy -u https_proxy -u all_proxy \ + uv sync \ + --extra data \ + --extra data-daft \ + --extra vector-index \ + --extra postgresql \ + --extra clickhouse \ + --extra mysql \ + --extra doris-flight \ + --extra s3 \ + --extra model-export \ + --extra model-export-torch \ + --extra hf \ + --extra model-export-hf \ + --extra training \ + --extra tune \ + --extra explainability \ + --extra identity \ + --extra streaming \ + --extra grpc \ + --extra registry \ + --extra graph \ + --extra causal \ + --extra streaming-inference \ + --no-default-groups \ + --no-dev \ + --locked + +# Ray runtime_env may need pip when an algorithm uses the offline wheelhouse +# distribution mode. Bootstrap it in the image without contacting an index. +RUN python -m ensurepip --upgrade + +# The builder always supplies this named context, including an empty directory +# when no external connector wheels were requested. +COPY --from=external-wheelhouse / /opt/tributo-image/external-wheelhouse/ + +# Keep the manifest value in a RUN instruction as well as an OCI label. This +# makes the sealing value part of the BuildKit cache key for the second build. +RUN printf '%s\n' "${TRIBUTO_MANIFEST_SHA256}" > /opt/tributo-image/manifest-seal + +RUN if find /opt/tributo-image/external-wheelhouse -type f -name '*.whl' -print -quit | grep -q .; then \ + python -m pip install --no-index --no-deps \ + /opt/tributo-image/external-wheelhouse/*.whl; \ + fi && \ + pip_check_output=""; \ + pip_check_status=0; \ + pip_check_output="$(python -m pip check 2>&1)" || pip_check_status=$?; \ + if [ "${pip_check_status}" -ne 0 ]; then \ + if [ "${TRIBUTO_PLATFORM}" != "linux/arm64" ] || \ + ! printf '%s\n' "${pip_check_output}" | grep -Eq \ + '^nvidia-cusparselt-cu13 0[.]8[.]1 is not supported on this platform[[:space:]]*$'; then \ + printf '%s\n' "${pip_check_output}"; \ + exit "${pip_check_status}"; \ + fi; \ + fi + +COPY --chown=ray:users tools/generate_distributions.py /opt/tributo-image/ + +RUN python /opt/tributo-image/generate_distributions.py + +WORKDIR /workspace + +# Keep an explicit final non-root boundary if later instructions add a root step. +USER ray diff --git a/docker/tributo-runtime/Dockerfile.dockerignore b/docker/tributo-runtime/Dockerfile.dockerignore new file mode 100644 index 0000000..2eb6696 --- /dev/null +++ b/docker/tributo-runtime/Dockerfile.dockerignore @@ -0,0 +1,8 @@ +** +!pyproject.toml +!uv.lock +!README.md +!src/ +!src/** +!tools/ +!tools/generate_distributions.py diff --git a/docs/STABILITY.md b/docs/STABILITY.md index f26e6d6..a601e33 100644 --- a/docs/STABILITY.md +++ b/docs/STABILITY.md @@ -34,6 +34,18 @@ This page provides module-level guidance and deprecation notes. | `tributo.exceptions` — `EngineNotAvailableError` | `alpha` | Candidate bounded-ingestion error | | `tributo.cli` | `beta` | Command-line interface | +### Runtime image tooling + +The image builder and emitted files are repository tooling rather than public +Python API symbols. Their contract is intentionally Alpha and is +covered by the external `runtime-image` suite. + +| Surface | Level | Notes | +|--------|-------|-------| +| `tools/build_tributo_image.py` | `alpha` | JSON-only Buildx builder for the pinned image validated for CPU execution by default, with explicit `linux/amd64`/`linux/arm64` targeting; it performs dependency-closure discovery, manifest sealing, and fail-closed import checks | +| `tools/tributo-runtime-full.json` | `alpha` | Multi-architecture pinned Ray/uv image references, native-platform default, complete first-party runtime-extra closure including locked v1.0 ClickHouse/Doris connectors, and optional external wheelhouse support | +| `manifest.json` / `image-profile.json` | `alpha` | Build attestations consumed for immutable image selection and algorithm artifact compatibility; not a registry or deployment API | + ### Training (tributo.training.*) Callbacks without a public `failure_policy` are best-effort in every normal diff --git a/docs/architecture/call-chain-inventory.md b/docs/architecture/call-chain-inventory.md index d825e63..b672cbe 100644 --- a/docs/architecture/call-chain-inventory.md +++ b/docs/architecture/call-chain-inventory.md @@ -279,9 +279,11 @@ credential-safe descriptor validation and atomic registration WriteBinding selection; native dependency import occurs at factory/execute ``` -Selected optional integrations (`ray-doris`, `daft-doris`, `daft-clickhouse`) also have -thin built-in descriptors and explicit install diagnostics. Their adapters are -not support claims until their external packages and infrastructure gates pass. +Selected optional integrations (`ray-doris==1.0`, `daft-doris==1.0`, +`daft-clickhouse==1.0`) also have thin built-in descriptors and explicit install +diagnostics. The canonical full runtime locks these packages into the image, +but their adapters are not support claims until database infrastructure gates +pass. Ray and Daft routes remain explicit and are not interchangeable. --- diff --git a/docs/data/index.md b/docs/data/index.md index 8994037..443e8ae 100644 --- a/docs/data/index.md +++ b/docs/data/index.md @@ -56,14 +56,15 @@ selects `binding_id` explicitly. | Local/S3 Lance | Native reader | Native reader | Verified | | PostgreSQL structured table | Native SQL reader | Native SQL reader | Verified | | HDFS Parquet/CSV | Native reader with PyArrow HDFS | No locked public reader | Adapted; cluster gate pending | -| ClickHouse | No selected Binding | External `daft-clickhouse` wheel | Adapter only; external package and database gates pending | -| Doris | `ray-doris` | External `daft-doris` wheel | Adapter only; external package and database gates pending | +| ClickHouse | No selected Binding | `daft-clickhouse==1.0` | Adapter only; install with `tributo[clickhouse]` or `uv sync --extra clickhouse`; real-database Conformance remains the support gate | +| Doris | `ray-doris==1.0` | `daft-doris==1.0` | Adapter only; Ray routes use `ray-doris`, Daft routes use `daft-doris`, and the full runtime image contains both v1.0 packages | | ORC or Hive external table | No locked public reader | No locked public reader | Unsupported, fail-closed | “Verified” means the current combination has semantic Conformance and real storage or database evidence. It does not turn every engine/source combination into a supported path. See the [support matrix](../reference/support-matrix.md) -for the exact boundary. +for the exact boundary. Daft and Ray Doris are explicit engine routes; a +missing `ray-doris` package does not silently fall back to Daft. Credentials belong to runtime configuration. They must not appear in dataset identifiers, logical plans, receipts, logs, or public errors. Bounded providers diff --git a/docs/getting-started/index.md b/docs/getting-started/index.md index 72bb63f..ac57a65 100644 --- a/docs/getting-started/index.md +++ b/docs/getting-started/index.md @@ -7,6 +7,7 @@ Choose one execution boundary and follow its guide from start to finish. installation quickstart +../how-to/runtime-images ``` - Use the local quickstart to validate a formal algorithm and Bundle without diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md index afa67d6..4d169cc 100644 --- a/docs/getting-started/installation.md +++ b/docs/getting-started/installation.md @@ -19,7 +19,11 @@ Pydantic, ONNX Runtime, PyArrow, pandas, and S3 filesystem support. | Ray Data table formats | `python -m pip install "tributo[data]"` | | Daft ingestion | `python -m pip install "tributo[data,data-daft]"` | | PostgreSQL ingestion | `python -m pip install "tributo[postgresql]"` | +| ClickHouse via Daft | `python -m pip install "tributo[clickhouse]"` | +| Doris via Daft/Ray Data | `python -m pip install "tributo[mysql]"` | +| Doris Flight via Daft/Ray Data | `python -m pip install "tributo[doris-flight]"` | | Distributed training | `python -m pip install "tributo[training]"` | +| BayesOpt search for Ray Tune | `python -m pip install "tributo[tune]"` | | Explainability | `python -m pip install "tributo[explainability]"` | | Lance vector indexing | `python -m pip install "tributo[vector-index]"` | | Torch model export | `python -m pip install "tributo[model-export-torch]"` | @@ -30,7 +34,14 @@ Pydantic, ONNX Runtime, PyArrow, pandas, and S3 filesystem support. An extra installs dependencies. It does not turn a protocol, adapter, or reserved problem type into a verified implementation. Check the -[support matrix](../reference/support-matrix.md) before deployment. +[support matrix](../reference/support-matrix.md) before deployment. Ray Tune +itself is included by the core Ray dependency; the `tune` extra adds the +optional BayesOpt search implementation. The `clickhouse` extra installs +`daft-clickhouse==1.0`; `mysql` installs `daft-doris==1.0` and +`ray-doris==1.0` for their explicit engine routes, while `doris-flight` adds +their Flight dependencies. The equivalent uv commands are +`uv sync --extra clickhouse`, `uv sync --extra mysql`, and +`uv sync --extra doris-flight`. ## Prepare a source checkout diff --git a/docs/how-to/runtime-images.md b/docs/how-to/runtime-images.md new file mode 100644 index 0000000..67276b4 --- /dev/null +++ b/docs/how-to/runtime-images.md @@ -0,0 +1,118 @@ +# Build the Tributo runtime image + +Tributo provides one directly buildable full runtime image validated for CPU +execution with Ray 2.55.1: Linux, Python 3.12, and every first-party runtime +extra, including the Alpha modules. By default the builder selects the host's +native architecture. The image is dependency-closed and does not install +development or test dependencies. On Linux, the locked PyTorch closure may +include transitive CUDA/NVIDIA distributions; their presence does not make +this a GPU image or a GPU support claim. + +## Prerequisites + +Run the commands from the repository root with Docker Buildx, Python 3.12, and +the repository's locked `uv` environment available. The pinned Ray and `uv` +base images are declared in +[`tools/tributo-runtime-full.json`](../../tools/tributo-runtime-full.json). + +The default target is the native host architecture (`linux/arm64` on Apple +Silicon and `linux/amd64` on an x86_64 host). To build a different target, +specify it explicitly, for example `--platform linux/amd64`. A cross-platform +build is not a native runtime validation; its Ray Jobs gate must run on a +matching native host. + +The builder downloads the digest-pinned Ray and `uv` inputs through the DaoCloud +mirrors recorded in the JSON configuration, verifies the digest, and gives each +image a local tag before BuildKit starts. Docker image operations explicitly +remove both upper- and lower-case `HTTP_PROXY`, `HTTPS_PROXY`, and `ALL_PROXY`, +so PandaFan is not used for registry traffic. The locked Python wheels keep +their `uv.lock` URLs and are not rewritten to a different index. + +## Build and attest the image + +```bash +uv run --locked --no-sync python tools/build_tributo_image.py \ + --config tools/tributo-runtime-full.json \ + --output-dir dist/tributo-runtime-full +``` + +The builder performs two Buildx builds. The first discovers the installed +distribution closure. The second seals a canonical SHA-256 manifest digest in +the image label `org.tributo.manifest-sha256` and verifies that the closure did +not change. It also runs `pip check`, imports every declared runtime/Alpha +module for the selected architecture, and checks `tributo --help`. + +The pinned `linux/arm64` Ray base exposes one known pip metadata baseline: +`nvidia-cusparselt-cu13 0.8.1 is not supported on this platform`. +The builder records that exact line in the manifest and accepts no other +`pip check` error; all other dependency failures stop the build. + +The output directory contains: + +- `manifest.json`: pinned image, base image, runtime extras, dependency + closure, mirror/local image source records, Alpha capability list, and image + digest; +- `image-profile.json`: the validated `ImageProfile` consumed by algorithm + artifact compatibility and preflight checks; it does not change an existing + Ray cluster's image; +- `installed-distributions.json`: the normalized image dependency inventory; +- `capabilities.json`: the Alpha capability declaration; +- `build.log`: the command/evidence log for this build. + +The builder never pushes an image. Publishing requires a separate reviewed +workflow. The canonical configuration resolves `daft-clickhouse==1.0`, +`daft-doris==1.0`, and `ray-doris==1.0` through the locked Tributo extras, so +the normal image build does not require a local connector wheelhouse. The +`external_wheelhouse` option remains available for packages outside the lock: +it is copied into a named build context and installed with +`pip --no-index --no-deps`; every wheel is recorded by filename, +package/version, size, and SHA-256. Such a variant is an attested, +image-specific extension and does not change the Tributo lockfile. + +The connector extras can also be installed outside Docker: + +```bash +pip install "tributo[clickhouse]" +pip install "tributo[mysql]" # Daft Doris + Ray Doris over MySQL +pip install "tributo[doris-flight]" # Daft/Ray Doris Flight dependencies +``` + +The equivalent uv commands are `uv sync --extra clickhouse`, +`uv sync --extra mysql`, and `uv sync --extra doris-flight`. A Doris test that +selects `engine="tributo.daft"` needs `daft-doris`; a Ray Doris or generic +training path that selects `engine="tributo.ray_data"` needs `ray-doris`. +Tributo does not silently switch between those explicit engine routes. + +## Run the image gate + +The gate builds the image for the native host architecture, starts a unique +two-node Docker Ray cluster, and submits +`tests/integrations/jobs/runtime_image_gate_job.py` through the Ray Jobs API: + +```bash +bash scripts/run_runtime_image_it.sh +``` + +An explicit target is available when the test host matches it: + +```bash +TRIBUTO_IMAGE_PLATFORM=linux/amd64 bash scripts/run_runtime_image_it.sh +``` + +The job imports all runtime and Alpha modules on the driver and a Ray worker, +checks Ray Data materialization, and verifies the Ray/Tributo versions. The +runner writes compose logs and attestation files under a unique temporary +directory, then removes only that Compose project and its volumes. + +## Runtime boundary + +The full image includes the first-party data, model export, training, +Tune, serving, streaming, registry, vector-index, explainability, graph, and +causal dependency closure. It does not include development/test dependencies, +GPU compatibility or a GPU execution gate, HDFS/ORC/Hive connectors, or a +KubeRay control plane. Linux PyTorch dependencies may still contain +CUDA/NVIDIA distributions; GPU drivers, GPU scheduling, and NCCL validation +are outside this image contract. +Kubernetes deployment remains the responsibility of the Ray/KubeRay +environment. Tributo selects and validates an immutable image profile; it does +not change a Ray cluster's image per submitted job. diff --git a/docs/reference/support-matrix.md b/docs/reference/support-matrix.md index b25b065..a50cdb2 100644 --- a/docs/reference/support-matrix.md +++ b/docs/reference/support-matrix.md @@ -18,6 +18,7 @@ automation eligibility. | Lance vector index | Vector-index API, storage profiles, and Ray Jobs request contract | `unit` contracts | `lance-vector-cluster` | | Tune and explainability | Capability declarations and typed job configuration | `unit` contracts | `tune-cluster` and `explainability-cluster` | | Distributed algorithms | Algorithm descriptors, execution profiles, and portable receipts | `unit` and `unit-integration-contracts` | `distributed-algorithm-cluster` | +| Full runtime image for CPU validation | Pinned Ray/uv base, locked first-party extras, Alpha import closure, and `ImageProfile` attestation | `unit` contracts | `runtime-image` | External suite names describe required evidence, not GitHub Actions jobs. Quarantined MLflow, serving, streaming, and legacy standalone tests do not @@ -44,8 +45,8 @@ compatible profile, while the generated `Validated profiles` column remains | PostgreSQL structured table reads | Verified | Ray uses a single public SQL read and fails closed on parallel shard requirements; Daft may use native partition hints | | ClickHouse/Doris raw SQL | Unsupported | Legacy shapes return a credential-free migration error; use structured table input or execute SQL outside Tributo ingestion | | HDFS Parquet/CSV reads | Adapter only | Ray binding exists; real HDFS/JVM/worker gate is pending | -| ClickHouse reads | Adapter only | Requires an externally installed `daft-clickhouse` wheel and real-database Conformance; provider partition discovery is distinct from engine auto-routing | -| Doris reads | Adapter only | Requires `ray-doris` or an externally installed `daft-doris` wheel and real-database Conformance; tablet planning remains provider/binding-owned | +| ClickHouse reads | Adapter only | Uses locked `daft-clickhouse==1.0` through `tributo[clickhouse]`; real-database Conformance is still required, and provider partition discovery is distinct from engine auto-routing | +| Doris reads | Adapter only | Ray routes use locked `ray-doris==1.0`; Daft routes use locked `daft-doris==1.0`; real-database Conformance is still required and tablet planning remains provider/binding-owned | | ORC and Hive external-table reads | Not implemented | Locked Ray/Daft versions expose no validated public reader | | Third-party ingestion Provider/Binding SPI | Implemented | Installed packages use `tributo.ingestion_providers` plus `tributo.ingestion_bindings`; bad plugins are isolated, duplicate routes never replace built-ins, and Binding selection can constrain filesystem, catalog, and storage format | | Lance output | Implemented as a generic ResultSink path | User Predictor owns vector semantics; the sink does not pool, normalize, or automatically invoke the separate vector-index workflow | @@ -152,5 +153,16 @@ tensors; it does not apply DNN/PU preprocessing implicitly. | Third-party Provider `normalize()+open()` | Independent Provider SPI retained; canonical Gateway requires `plan()` plus `EngineBinding`, and does not fallback or emit the removed adapter warning | | Third-party bounded source | `ProviderSourceConfig` plus versioned Provider/Binding descriptors; no consumer-module source branches | +## Runtime images + +| Capability | Status | Boundary | +| --- | --- | --- | +| Full Tributo runtime image for CPU validation | Alpha, directly buildable | Linux `arm64`/`amd64`, defaulting to the native host architecture; Python 3.12, Ray 2.55.1, locked dependency closure, and all first-party runtime extras including Alpha modules. Linux PyTorch resolution may include transitive CUDA/NVIDIA distributions, but no GPU support is claimed | +| Custom connector wheelhouse variant | Alpha, optional extension | An external wheelhouse remains available for packages outside the locked v1.0 connector set; it is not required for the canonical ClickHouse/Doris image and does not change the Tributo lockfile | +| Runtime image attestation | Alpha | `manifest.json`, `image-profile.json`, normalized distribution inventory, and sealed `org.tributo.manifest-sha256` label | +| Runtime image Ray Jobs gate | Alpha, validation gate | Requires a unique two-node Docker Ray cluster on a native host architecture; verifies driver/worker imports, Ray Data, Jobs API submission, and v1.0 ClickHouse/Doris package presence. `linux/amd64` requires a matching native host | +| GPU runtime image | Not implemented | The Linux dependency closure may contain transitive CUDA/NVIDIA distributions from PyTorch, but no GPU driver, scheduling, NCCL, or GPU compatibility contract has been validated | +| HDFS/ORC/Hive runtime additions | Not included | No validated provider/runtime contract in the current image | + For symbol-level compatibility promises, consult the [API stability inventory](../STABILITY.md). diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 852f3b1..ad2aaf0 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -18,6 +18,7 @@ auth autodoc backend backends +Buildx backpressure backpropagation BaseModel @@ -27,6 +28,7 @@ BayesOpt benchmarked boto botocore +buildable bundle BundleModelReference BundleRef @@ -136,6 +138,7 @@ JSON json Kafka Kubernetes +KubeRay kwargs Lance pylance @@ -147,6 +150,7 @@ Lifecycle lifecycle logit logits +lockfile lookalike Makefile macOS diff --git a/pyproject.toml b/pyproject.toml index 21257e4..3a2158d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,14 +72,19 @@ postgresql = [ ] clickhouse = [ "clickhouse-connect[arrow,async]>=1.5,<1.6", + "daft-clickhouse[clickhouse]==1.0", ] mysql = [ "PyMySQL>=1.2,<1.3", + "daft-doris[doris]==1.0", + "ray-doris==1.0", ] doris-flight = [ "PyMySQL>=1.2,<1.3", "adbc-driver-manager>=1.6,<2", "adbc-driver-flightsql>=1.6,<2", + "daft-doris[doris-flight]==1.0", + "ray-doris[flight]==1.0", ] s3 = [ "boto3>=1.42.91", @@ -109,6 +114,10 @@ training = [ "onnxmltools>=1.13.0", "skl2onnx>=1.17.0", ] +tune = [ + # Ray Tune's optional BayesOpt integration is validated against this exact version. + "bayesian-optimization==1.4.3", +] explainability = [ "shap>=0.52.0,<0.53.0", "numpy>=2.0.0,<3.0.0", diff --git a/scripts/run_runtime_image_it.sh b/scripts/run_runtime_image_it.sh new file mode 100644 index 0000000..8fb79b2 --- /dev/null +++ b/scripts/run_runtime_image_it.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +COMPOSE_FILE="$ROOT/tests/integrations/docker-compose.runtime-image.yml" +PROJECT="tributo-runtime-image-it-$(date +%s)-$$" +OUTPUT_DIR=$(mktemp -d "${TMPDIR:-/tmp}/tributo-runtime-image-it.XXXXXX") +IMAGE="tributo-runtime-full:local" +DOCKER_ENV=( + env + -u HTTP_PROXY + -u HTTPS_PROXY + -u ALL_PROXY + -u http_proxy + -u https_proxy + -u all_proxy +) + +docker_compose() { + "${DOCKER_ENV[@]}" docker compose "$@" +} + +export COMPOSE_PROJECT_NAME="$PROJECT" +export TRIBUTO_RUNTIME_IMAGE="$IMAGE" +export TRIBUTO_IMAGE_SOURCE_ROOT="$ROOT" + +cleanup() { + # Preserve the status that triggered EXIT/INT/TERM; cleanup is best effort. + status=$? + docker_compose --project-name "$PROJECT" --file "$COMPOSE_FILE" logs --no-color \ + >"$OUTPUT_DIR/compose.log" 2>&1 || true + docker_compose --project-name "$PROJECT" --file "$COMPOSE_FILE" down \ + --volumes --remove-orphans || true + echo "Runtime image gate artifacts: $OUTPUT_DIR" >&2 + exit "$status" +} +trap cleanup EXIT INT TERM + +BUILD_COMMAND=( + env + -u HTTP_PROXY + -u HTTPS_PROXY + -u ALL_PROXY + -u http_proxy + -u https_proxy + -u all_proxy + "UV_CACHE_DIR=${UV_CACHE_DIR:-/tmp/tributo-image-uv-cache}" + uv run + --locked + --no-sync + python + "$ROOT/tools/build_tributo_image.py" + --repo-root + "$ROOT" + --config + "$ROOT/tools/tributo-runtime-full.json" + --output-dir + "$OUTPUT_DIR/image" +) +if [[ -n "${TRIBUTO_IMAGE_PLATFORM:-}" ]]; then + BUILD_COMMAND+=(--platform "$TRIBUTO_IMAGE_PLATFORM") +fi +"${BUILD_COMMAND[@]}" + +runtime_platform="$(python -c 'import json, sys; print(json.load(open(sys.argv[1], encoding="utf-8"))["platform"])' "$OUTPUT_DIR/image/manifest.json")" +case "$runtime_platform" in + linux/amd64|linux/arm64) ;; + *) + echo "Invalid runtime image platform in manifest: $runtime_platform" >&2 + exit 1 + ;; +esac +export TRIBUTO_RUNTIME_PLATFORM="$runtime_platform" + +docker_compose --project-name "$PROJECT" --file "$COMPOSE_FILE" up \ + --detach --wait --wait-timeout 180 +docker_compose --project-name "$PROJECT" --file "$COMPOSE_FILE" exec --no-TTY ray-head \ + ray job submit \ + --address http://127.0.0.1:8265 \ + --working-dir /workspace/tributo-src \ + -- python tests/integrations/jobs/runtime_image_gate_job.py diff --git a/src/tributo/data/bindings/__init__.py b/src/tributo/data/bindings/__init__.py index e5af101..c4082b9 100644 --- a/src/tributo/data/bindings/__init__.py +++ b/src/tributo/data/bindings/__init__.py @@ -28,14 +28,17 @@ _DAFT_LANCE_INSTALL_HINT = "pip install 'tributo[data,data-daft]'" _DATA_INSTALL_HINT = "pip install 'tributo[data]'" _DAFT_CLICKHOUSE_INSTALL_HINT = ( - "Install a local daft-clickhouse[clickhouse] wheel, then run " - "pip install 'tributo[clickhouse]'" + "Install daft-clickhouse==1.0 with Tributo: " + "pip install 'tributo[clickhouse]' (or uv sync --extra clickhouse)" ) _DAFT_DORIS_INSTALL_HINT = ( - "Install a local daft-doris[doris] wheel (or daft-doris[doris-flight] for Flight), " - "then run pip install 'tributo[mysql]' (or 'tributo[doris-flight]' for Flight)" + "Install daft-doris==1.0 with Tributo: pip install 'tributo[mysql]' " + "(or uv sync --extra mysql); use 'tributo[doris-flight]' for Flight" +) +_RAY_DORIS_INSTALL_HINT = ( + "Install ray-doris==1.0 with Tributo: pip install 'tributo[mysql]' " + "(or uv sync --extra mysql); use 'tributo[doris-flight]' for Flight" ) -_RAY_DORIS_INSTALL_HINT = "pip install 'ray-doris[mysql,flight]'" _POSTGRESQL_INSTALL_HINT = "pip install 'tributo[postgresql]'" @@ -283,7 +286,7 @@ def _daft_clickhouse_descriptor() -> BindingDescriptor: factory=DaftClickHouseBinding, capabilities=frozenset({SourceCapability.PROJECTION}), distribution_name="daft-clickhouse", - distribution_version=_distribution_version("daft-clickhouse") or "0.1.0a1", + distribution_version=_distribution_version("daft-clickhouse") or "1.0", engine_version_spec=_DAFT_SQL_VERSION_SPEC, dependency_distributions=("clickhouse-connect",), supported_read_hints=frozenset( @@ -306,7 +309,7 @@ def _daft_doris_descriptor() -> BindingDescriptor: factory=DaftDorisBinding, capabilities=frozenset({SourceCapability.PROJECTION}), distribution_name="daft-doris", - distribution_version=_distribution_version("daft-doris") or "0.1.0a1", + distribution_version=_distribution_version("daft-doris") or "1.0", engine_version_spec=_DAFT_SQL_VERSION_SPEC, dependency_distributions=("PyMySQL",), supported_read_hints=frozenset( @@ -329,8 +332,9 @@ def _ray_doris_descriptor() -> BindingDescriptor: factory=RayDorisBinding, capabilities=frozenset({SourceCapability.PROJECTION}), distribution_name="ray-doris", - distribution_version=_distribution_version("ray-doris") or "0.1.0a1", + distribution_version=_distribution_version("ray-doris") or "1.0", engine_version_spec=_RAY_VERSION_SPEC, + dependency_distributions=("PyMySQL",), supported_read_hints=frozenset( { ReadHint.TARGET_PARALLELISM, @@ -634,7 +638,7 @@ def default_engine_bindings() -> EngineBindings: _RAY_VERSION_SPEC, _RAY_DORIS_INSTALL_HINT, None, - ("ray-doris",), + ("ray-doris", "PyMySQL"), ), ( _ray_postgresql_descriptor, diff --git a/tests/data/test_engine_binding.py b/tests/data/test_engine_binding.py index 9ec9aa5..346766c 100644 --- a/tests/data/test_engine_binding.py +++ b/tests/data/test_engine_binding.py @@ -733,8 +733,8 @@ def test_daft_sql_descriptors_use_new_package_identity( builtin_bindings, "_distribution_version", lambda name: { - "daft-clickhouse": "0.1.0a1", - "daft-doris": "0.1.0a1", + "daft-clickhouse": "1.0", + "daft-doris": "1.0", }.get(name), ) @@ -766,7 +766,7 @@ def test_missing_daft_sql_packages_report_new_install_hints( with pytest.raises( EngineNotAvailableError, - match=r"daft_clickhouse\.daft\.clickhouse.*local daft-clickhouse.*tributo\[clickhouse\]", + match=r"daft_clickhouse\.daft\.clickhouse.*daft-clickhouse.*tributo\[clickhouse\]", ): bindings.resolve( BindingKey( @@ -778,7 +778,7 @@ def test_missing_daft_sql_packages_report_new_install_hints( ) with pytest.raises( EngineNotAvailableError, - match=r"daft_doris\.daft\.doris.*local daft-doris.*tributo\[mysql\]", + match=r"daft_doris\.daft\.doris.*daft-doris.*tributo\[mysql\]", ): bindings.resolve( BindingKey( diff --git a/tests/integration/test_it_component_versions.py b/tests/integration/test_it_component_versions.py index 727514f..7e90202 100644 --- a/tests/integration/test_it_component_versions.py +++ b/tests/integration/test_it_component_versions.py @@ -80,3 +80,48 @@ def test_dockerfiles_and_compose_consume_the_version_contract() -> None: assert "mlflow-data:/mlflow" in compose assert "--artifacts-destination" in compose assert "--default-artifact-root" not in compose + + +def test_full_runtime_profile_matches_the_image_builder_contract() -> None: + profiles = json.loads( + (ROOT / "tests/integrations/runtime-profiles.json").read_text() + )["profiles"] + profile = profiles["runtime-full"] + dockerfile = (ROOT / profile["dockerfile"]).read_text() + config = json.loads((ROOT / "tools/tributo-runtime-full.json").read_text()) + locked = _locked_versions() + + assert profile["base_image"] == config["base_image"] + assert profile["base_image_mirror"] == config["base_image_mirror"] + assert profile["uv_image"] == config["uv_image"] + assert profile["uv_image_mirror"] == config["uv_image_mirror"] + assert ( + profile["base_image"].rsplit("@", 1)[1] + == profile["base_image_mirror"].rsplit("@", 1)[1] + ) + assert ( + profile["uv_image"].rsplit("@", 1)[1] + == profile["uv_image_mirror"].rsplit("@", 1)[1] + ) + assert profile["extras"] == config["runtime_extras"] + assert profile["python_version"] == "3.12" + assert profile["version_contract"]["ray"] == "2.55.1" + for package, contract_key in { + "daft-clickhouse": "daft_clickhouse", + "daft-doris": "daft_doris", + "ray-doris": "ray_doris", + }.items(): + assert profile["version_contract"][contract_key] == locked[package] + for extra in config["runtime_extras"]: + assert f"--extra {extra}" in dockerfile + assert "--no-default-groups" in dockerfile + assert "--no-dev" in dockerfile + assert "TRIBUTO_BASE_IMAGE" in dockerfile + assert "COPY --from=external-wheelhouse" in dockerfile + + runner = (ROOT / "scripts/run_runtime_image_it.sh").read_text() + assert 'case "$runtime_platform" in' in runner + assert 'export TRIBUTO_RUNTIME_PLATFORM="$runtime_platform"' in runner + assert "-u HTTP_PROXY" in runner + assert "-u http_proxy" in runner + assert "--wait-timeout 180" in runner diff --git a/tests/integrations/Dockerfile.data-ingestion b/tests/integrations/Dockerfile.data-ingestion index cc02290..7a920fd 100644 --- a/tests/integrations/Dockerfile.data-ingestion +++ b/tests/integrations/Dockerfile.data-ingestion @@ -39,6 +39,6 @@ RUN --mount=type=cache,target=/home/ray/.cache/uv,uid=1000,gid=100 \ # Python's bundled ensurepip so offline Wheelhouse jobs never need an index. RUN python -m ensurepip --upgrade -RUN python -c "import importlib.metadata as m, sys; assert sys.version_info[:2] == (3, 12); assert m.version('pip'); assert m.version('ray') == '2.55.1'; assert m.version('daft').startswith('0.7.'); assert m.version('pylance') == '9.0.0'; assert m.version('lance-ray') == '0.5.0'; assert m.version('pyarrow') == '19.0.1'" +RUN python -c "import importlib.metadata as m, sys; assert sys.version_info[:2] == (3, 12); assert m.version('pip'); assert m.version('ray') == '2.55.1'; assert m.version('daft').startswith('0.7.'); assert m.version('pylance') == '9.0.0'; assert m.version('lance-ray') == '0.5.0'; assert m.version('pyarrow') == '19.0.1'; assert m.version('daft-clickhouse') == '1.0'; assert m.version('daft-doris') == '1.0'; assert m.version('ray-doris') == '1.0'" WORKDIR /workspace/tributo-src diff --git a/tests/integrations/README.md b/tests/integrations/README.md index 94c706a..e29f68f 100644 --- a/tests/integrations/README.md +++ b/tests/integrations/README.md @@ -6,10 +6,11 @@ project. They never reuse a host Ray runtime, an existing MLflow server, or a pre-existing container. `ci/test-suites.json` owns their impact rules and reports required evidence, but no GitHub Actions event executes these suites. -The data-ingestion image installs database drivers through Tributo extras. The -unpublished `daft-clickhouse` and `daft-doris` connectors are external wheel -prerequisites and are not resolved by Tributo's `uv.lock`; the relevant runtime -must install the local wheel before running the corresponding E2E entry point. +The data-ingestion image installs database drivers and the v1.0 database +connectors through Tributo extras. `daft-clickhouse==1.0`, +`daft-doris==1.0`, and `ray-doris==1.0` are resolved by `uv.lock` and included +in the canonical full runtime image. A custom external wheelhouse is only for +packages outside that locked set. --- @@ -20,8 +21,9 @@ must install the local wheel before running the corresponding E2E entry point. | Model-export golden path | `../integration/test_walking_skeleton.py` | `manual_external` | Ray Data Parquet → XGBoost → ONNX + UBJ S3 Bundle → BundleReader → batch inference → Ray Serve HTTP, plus MLflow provenance | Docker only; the runner creates all infrastructure | | First-party export conformance | `../training/exporters/test_first_party_conformance.py` | `manual_external` | XGBoost, Torch, Hugging Face, quantizer, validator, and checkpoint-source contracts in the pinned Linux image | Docker only; executed by the model-export runner | | S3/MinIO contract | `../integration/test_export_s3.py`, `../integration/test_minio_compat.py` | `ci_fast` Moto / `manual_external` MinIO | Manifest-last publication, Lease/CAS, alias, GC, path-style access, and conditional writes | Ephemeral Moto in CI; run-owned MinIO externally | +| Runtime image gate | `../../scripts/run_runtime_image_it.sh`, `jobs/runtime_image_gate_job.py` | `manual_external` | Pinned full image, first-party and Alpha imports on driver/worker, Ray Data, Ray Jobs, and image attestations | Docker Buildx + two-node Ray cluster on a matching native host architecture | | MLflow Hook | `test_e2e_mlflow.py` | `manual_external` | Committed Bundle upload, replay deduplication, explicit run reuse, and failure semantics | Isolated model-export runner | -| ClickHouse E2E | `test_e2e_clickhouse.py` | `quarantine` | ClickHouse table → Daft ClickHouse Binding → explicit Daft-to-Ray adapter → XGBoost distributed training → MLflow → ONNX | Lifecycle and ownership contract pending | +| ClickHouse E2E | `test_e2e_clickhouse.py` | `quarantine` | ClickHouse table → Daft ClickHouse Binding → explicit Daft-to-Ray adapter → XGBoost distributed training → MLflow → ONNX | Full image or `tributo[clickhouse]`; lifecycle and ownership contract pending | | Dual-engine Docker | `test_data_ingestion_dual_engine.py` | `manual_external` | Local Parquet, full ETL chain, typed handles, worker-version evidence | Docker Ray cluster + Daft | | Lance vector index | `test_lance_vector_index.py` | `manual_external` | Distributed IVF_FLAT/IVF_PQ build, append coverage, global Top-K, fallback, optimization, compaction, Ray Jobs, and S3 result delivery | Docker Ray cluster + Lance-Ray + MinIO | | File conformance | `../integration/test_data_ingestion_conformance.py` | `manual_external` | Local/MinIO Parquet and CSV through Ray Data and Daft | Local Ray runtime + MinIO | diff --git a/tests/integrations/docker-compose.runtime-image.yml b/tests/integrations/docker-compose.runtime-image.yml new file mode 100644 index 0000000..4e151b1 --- /dev/null +++ b/tests/integrations/docker-compose.runtime-image.yml @@ -0,0 +1,46 @@ +x-ray-common: &ray-common + image: ${TRIBUTO_RUNTIME_IMAGE:?TRIBUTO_RUNTIME_IMAGE is required} + pull_policy: never + init: true + shm_size: 2gb + working_dir: /workspace/tributo-src + environment: + PYTHONDONTWRITEBYTECODE: "1" + PYTHONPATH: /workspace/tributo-src/src:/workspace/tributo-src + RAY_ACCEL_ENV_VAR_OVERRIDE_ON_ZERO: "0" + TRIBUTO_RAY_DASHBOARD_URL: http://127.0.0.1:8265 + TRIBUTO_RUNTIME_PLATFORM: ${TRIBUTO_RUNTIME_PLATFORM:?TRIBUTO_RUNTIME_PLATFORM is required} + volumes: + - ${TRIBUTO_IMAGE_SOURCE_ROOT:?TRIBUTO_IMAGE_SOURCE_ROOT is required}:/workspace/tributo-src:ro + +services: + ray-head: + <<: *ray-common + command: + - ray + - start + - --head + - --port=6379 + - --dashboard-host=0.0.0.0 + - --num-cpus=1 + - --block + healthcheck: + test: + - CMD-SHELL + - ray status --address=127.0.0.1:6379 >/dev/null 2>&1 + interval: 5s + timeout: 5s + retries: 24 + start_period: 10s + + ray-worker: + <<: *ray-common + command: + - ray + - start + - --address=ray-head:6379 + - --num-cpus=2 + - --block + depends_on: + ray-head: + condition: service_healthy diff --git a/tests/integrations/jobs/runtime_image_gate_job.py b/tests/integrations/jobs/runtime_image_gate_job.py new file mode 100644 index 0000000..aea531e --- /dev/null +++ b/tests/integrations/jobs/runtime_image_gate_job.py @@ -0,0 +1,78 @@ +"""Ray Jobs API payload for the full runtime image gate.""" + +from __future__ import annotations + +import importlib +import importlib.metadata +import json +import os +import platform +import sys + +import ray + +from tools.runtime_image_contract import ( + REQUIRED_DISTRIBUTION_VERSIONS, + REQUIRED_DISTRIBUTIONS, + REQUIRED_IMPORTS, +) + + +@ray.remote +def worker_probe() -> dict[str, object]: + """Verify that a Ray worker sees the same installed runtime closure.""" + for name in REQUIRED_DISTRIBUTIONS: + importlib.metadata.version(name) + for name, version in REQUIRED_DISTRIBUTION_VERSIONS.items(): + assert importlib.metadata.version(name) == version + for name in REQUIRED_IMPORTS: + importlib.import_module(name) + return { + "python": f"{sys.version_info.major}.{sys.version_info.minor}", + "machine": platform.machine(), + "ray": importlib.metadata.version("ray"), + "tributo": importlib.metadata.version("tributo"), + "daft-clickhouse": importlib.metadata.version("daft-clickhouse"), + "daft-doris": importlib.metadata.version("daft-doris"), + "ray-doris": importlib.metadata.version("ray-doris"), + } + + +def main() -> None: + assert sys.version_info[:2] == (3, 12) + expected_machine = { + "linux/amd64": "x86_64", + "linux/arm64": "aarch64", + }[os.environ["TRIBUTO_RUNTIME_PLATFORM"]] + assert platform.machine() == expected_machine + assert importlib.metadata.version("ray") == "2.55.1" + assert importlib.metadata.version("tributo") == "1.0.0" + for name in REQUIRED_DISTRIBUTIONS: + importlib.metadata.version(name) + for name, version in REQUIRED_DISTRIBUTION_VERSIONS.items(): + assert importlib.metadata.version(name) == version + for name in REQUIRED_IMPORTS: + importlib.import_module(name) + + ray.init(address="auto", namespace="tributo-runtime-image-gate") + try: + worker_result = ray.get(worker_probe.remote()) + assert worker_result == { + "python": "3.12", + "machine": expected_machine, + "ray": "2.55.1", + "tributo": "1.0.0", + "daft-clickhouse": "1.0", + "daft-doris": "1.0", + "ray-doris": "1.0", + } + dataset = ray.data.from_items([{"value": 1}, {"value": 2}]) + assert dataset.count() == 2 + assert ray.cluster_resources().get("CPU", 0) >= 1 + print(json.dumps({"status": "passed", "worker": worker_result}, sort_keys=True)) + finally: + ray.shutdown() + + +if __name__ == "__main__": + main() diff --git a/tests/integrations/runtime-profiles.json b/tests/integrations/runtime-profiles.json index 5735c29..1353440 100644 --- a/tests/integrations/runtime-profiles.json +++ b/tests/integrations/runtime-profiles.json @@ -20,10 +20,58 @@ "uv_image": "ghcr.io/astral-sh/uv:0.11.23@sha256:d0a0a753ab981624b49c97abc98821c1c09f4ca69d1ef5cee69c501be3d88479", "version_contract": { "daft_prefix": "0.7.", + "daft_clickhouse": "1.0", + "daft_doris": "1.0", "lance_ray": "0.5.0", "pyarrow": "19.0.1", "pylance": "9.0.0", - "ray": "2.55.1" + "ray": "2.55.1", + "ray_doris": "1.0" + } + }, + "runtime-full": { + "base_image": "rayproject/ray:2.55.1-py312@sha256:911245f2478ad2e9f67ac13978dc2a75bcae0498b9f188b10bba703324b78379", + "base_image_mirror": "docker.m.daocloud.io/rayproject/ray:2.55.1-py312@sha256:911245f2478ad2e9f67ac13978dc2a75bcae0498b9f188b10bba703324b78379", + "dockerfile": "docker/tributo-runtime/Dockerfile", + "extras": [ + "data", + "data-daft", + "vector-index", + "postgresql", + "clickhouse", + "mysql", + "doris-flight", + "s3", + "model-export", + "model-export-torch", + "hf", + "model-export-hf", + "training", + "tune", + "explainability", + "identity", + "streaming", + "grpc", + "registry", + "graph", + "causal", + "streaming-inference" + ], + "minio_image": "minio/minio:RELEASE.2025-09-07T16-13-09Z@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e", + "python_version": "3.12", + "runtime_repository": "tributo-runtime", + "tool_image": "python:3.12.13-alpine3.22@sha256:a190708a2dec1bd18b1decb539f8e8f5407abaa9bf39cacda583f7f8c11db322", + "uv_image": "ghcr.io/astral-sh/uv:0.11.23@sha256:d0a0a753ab981624b49c97abc98821c1c09f4ca69d1ef5cee69c501be3d88479", + "uv_image_mirror": "ghcr.m.daocloud.io/astral-sh/uv:0.11.23@sha256:d0a0a753ab981624b49c97abc98821c1c09f4ca69d1ef5cee69c501be3d88479", + "version_contract": { + "daft_prefix": "0.7.", + "daft_clickhouse": "1.0", + "daft_doris": "1.0", + "lance_ray": "0.5.0", + "pyarrow": "19.0.1", + "pylance": "9.0.0", + "ray": "2.55.1", + "ray_doris": "1.0" } } } diff --git a/tests/integrations/test_e2e_clickhouse.py b/tests/integrations/test_e2e_clickhouse.py index ae27029..5056eb8 100644 --- a/tests/integrations/test_e2e_clickhouse.py +++ b/tests/integrations/test_e2e_clickhouse.py @@ -5,8 +5,9 @@ 运行方式： docker exec ray-head python /opt/tributo/tests/integration/test_e2e_clickhouse.py -前提：Docker 集群已启动，含 Ray / Daft / 已安装本地 wheel 的 daft-clickhouse / -ClickHouse (8123) / MinIO / MLflow (5000)。 +前提：Docker 集群已启动，运行时通过完整镜像或 +`tributo[clickhouse]` / `uv sync --extra clickhouse` 安装了 +`daft-clickhouse==1.0`，并包含 ClickHouse (8123) / MinIO / MLflow (5000)。 """ from __future__ import annotations diff --git a/tests/integrations/test_e2e_multi_class.py b/tests/integrations/test_e2e_multi_class.py index 220cb57..966143d 100644 --- a/tests/integrations/test_e2e_multi_class.py +++ b/tests/integrations/test_e2e_multi_class.py @@ -5,8 +5,9 @@ 运行方式： docker exec ray-head python /opt/tributo/tests/integration/test_e2e_multi_class.py -前提：Docker 集群已启动，含 Ray / Daft / 已安装本地 wheel 的 daft-clickhouse / -ClickHouse (8123) / MLflow (5000)。 +前提：Docker 集群已启动，运行时通过完整镜像或 +`tributo[clickhouse]` / `uv sync --extra clickhouse` 安装了 +`daft-clickhouse==1.0`，并包含 ClickHouse (8123) / MLflow (5000)。 """ from __future__ import annotations diff --git a/tests/tools/test_build_tributo_image.py b/tests/tools/test_build_tributo_image.py new file mode 100644 index 0000000..cb064c3 --- /dev/null +++ b/tests/tools/test_build_tributo_image.py @@ -0,0 +1,338 @@ +"""Unit tests for the reproducible full-runtime image builder.""" + +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from tools.build_tributo_image import ( + BASE_IMAGE, + BASE_IMAGE_MIRROR, + DOCKERFILE, + PLATFORM_AUTO, + RUNTIME_EXTRAS, + UV_IMAGE, + UV_IMAGE_MIRROR, + ImageBuildError, + _manifest_core, + _prepare_pinned_image, + build_command, + build_image, + canonical_json, + detect_host_platform, + load_config, + local_base_image, + local_uv_image, + pip_check_baseline, + prepared_wheelhouse, + sha256_bytes, + wheel_records, +) +from tools.runtime_image_contract import ( + REQUIRED_DISTRIBUTION_VERSIONS, + REQUIRED_DISTRIBUTIONS, + REQUIRED_IMPORTS, +) + +ROOT = Path(__file__).resolve().parents[2] +CONFIG = ROOT / "tools" / "tributo-runtime-full.json" + + +def test_full_runtime_config_is_pinned_and_complete() -> None: + config = load_config(CONFIG, root=ROOT) + + assert config.base_image == BASE_IMAGE + assert config.uv_image == UV_IMAGE + assert config.base_image_mirror == BASE_IMAGE_MIRROR + assert config.uv_image_mirror == UV_IMAGE_MIRROR + assert config.platform == detect_host_platform() + assert config.runtime_extras == RUNTIME_EXTRAS + assert config.external_wheelhouse is None + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("base_image", "rayproject/ray:2.55.1-py312"), + ("uv_image", "ghcr.io/astral-sh/uv:0.11.23"), + ("platform", "linux/ppc64le"), + ], +) +def test_config_rejects_unpinned_or_unsupported_values( + field: str, value: str, tmp_path: Path +) -> None: + payload = json.loads(CONFIG.read_text(encoding="utf-8")) + payload[field] = value + candidate = tmp_path / "tributo-runtime-invalid.json" + candidate.write_text(json.dumps(payload), encoding="utf-8") + with pytest.raises(ImageBuildError): + load_config(candidate, root=ROOT) + + +@pytest.mark.parametrize( + ("machine", "expected"), + [ + ("arm64", "linux/arm64"), + ("aarch64", "linux/arm64"), + ("x86_64", "linux/amd64"), + ], +) +def test_auto_platform_follows_host_architecture( + machine: str, expected: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr( + "tools.build_tributo_image.host_platform.machine", lambda: machine + ) + + assert load_config(CONFIG, root=ROOT).platform == expected + + +def test_explicit_platform_overrides_auto(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "tools.build_tributo_image.host_platform.machine", lambda: "arm64" + ) + + config = load_config(CONFIG, root=ROOT, platform_override="linux/amd64") + + assert config.platform == "linux/amd64" + + +def test_auto_is_the_default_config_value() -> None: + payload = json.loads(CONFIG.read_text(encoding="utf-8")) + + assert payload["platform"] == PLATFORM_AUTO + + +def test_config_rejects_missing_and_unknown_keys(tmp_path: Path) -> None: + payload = json.loads(CONFIG.read_text(encoding="utf-8")) + payload.pop("runtime_extras") + payload["unexpected"] = True + path = tmp_path / "invalid.json" + path.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(ImageBuildError, match="keys differ"): + load_config(path, root=ROOT) + + +def test_wheel_records_are_digest_only_and_deterministic(tmp_path: Path) -> None: + wheel = tmp_path / "example_pkg-1.2.3-py3-none-any.whl" + wheel.write_bytes(b"wheel-content") + + records = wheel_records(tmp_path) + + assert records == [ + { + "filename": wheel.name, + "name": "example-pkg", + "version": "1.2.3", + "size": len(b"wheel-content"), + "sha256": sha256_bytes(b"wheel-content"), + } + ] + assert "path" not in records[0] + + +def test_wheelhouse_context_copies_only_wheels(tmp_path: Path) -> None: + wheel = tmp_path / "example_pkg-1.2.3-py3-none-any.whl" + wheel.write_bytes(b"wheel-content") + (tmp_path / "README.txt").write_text("ignored", encoding="utf-8") + + with prepared_wheelhouse(tmp_path) as context: + assert (context / wheel.name).read_bytes() == b"wheel-content" + assert not (context / "README.txt").exists() + + +def test_build_command_is_shell_free_and_uses_named_context() -> None: + config = load_config(CONFIG, root=ROOT) + command = build_command( + config, + root=ROOT, + wheelhouse_context=Path("/tmp/empty-wheelhouse"), + manifest_sha256="unsealed", + ) + + assert command[:4] == ["docker", "buildx", "build", "--load"] + assert "--platform" in command + assert "--file" in command + assert str(DOCKERFILE) in command + assert "--push" not in command + assert "external-wheelhouse=/tmp/empty-wheelhouse" in command + assert f"BASE_IMAGE={local_base_image(config.platform)}" in command + assert f"UV_IMAGE={local_uv_image(config.platform)}" in command + assert f"TRIBUTO_BASE_IMAGE={BASE_IMAGE}" in command + assert f"TRIBUTO_PLATFORM={config.platform}" in command + assert f"TRIBUTO_RUNTIME_EXTRAS={','.join(RUNTIME_EXTRAS)}" in command + + +def test_runtime_image_contract_is_shared_with_the_gate_job() -> None: + gate_job = ( + ROOT / "tests" / "integrations" / "jobs" / "runtime_image_gate_job.py" + ).read_text(encoding="utf-8") + + from tools import build_tributo_image + + assert build_tributo_image.REQUIRED_IMPORTS is REQUIRED_IMPORTS + assert build_tributo_image.REQUIRED_DISTRIBUTIONS is REQUIRED_DISTRIBUTIONS + assert ( + build_tributo_image.REQUIRED_DISTRIBUTION_VERSIONS + is REQUIRED_DISTRIBUTION_VERSIONS + ) + assert "from tools.runtime_image_contract import" in gate_job + + +def test_runtime_image_dockerfile_seals_and_generates_its_inventory() -> None: + dockerfile = DOCKERFILE.read_text(encoding="utf-8") + + expected_manifest_line = ( + "RUN printf '%s\\n' \"" + + "$" + + '{TRIBUTO_MANIFEST_SHA256}" > /opt/tributo-image/manifest-seal' + ) + assert expected_manifest_line in dockerfile + assert "COPY --chown=ray:users tools/generate_distributions.py" in dockerfile + assert "RUN python /opt/tributo-image/generate_distributions.py" in dockerfile + + +def test_pinned_image_pull_is_platform_scoped_and_digest_checked( + monkeypatch: pytest.MonkeyPatch, +) -> None: + digest = "a" * 64 + mirror = f"docker.m.daocloud.io/example/runtime:1@sha256:{digest}" + calls: list[list[str]] = [] + + def fake_run(args: list[str], **kwargs: object) -> SimpleNamespace: + calls.append(args) + return SimpleNamespace( + returncode=0, + stdout=f"Digest: sha256:{digest}\n", + stderr="", + ) + + monkeypatch.setattr("tools.build_tributo_image._run", fake_run) + monkeypatch.setattr( + "tools.build_tributo_image._inspect_image", + lambda image, **kwargs: {"Os": "linux", "Architecture": "arm64"}, + ) + + _prepare_pinned_image( + mirror=mirror, + canonical=f"example/runtime:1@sha256:{digest}", + local="tributo-example:arm64", + platform="linux/arm64", + ) + + assert calls == [ + [ + "docker", + "pull", + "--platform", + "linux/arm64", + "docker.m.daocloud.io/example/runtime:1", + ], + [ + "docker", + "tag", + "docker.m.daocloud.io/example/runtime:1", + "tributo-example:arm64", + ], + ] + + +def test_docker_environment_removes_all_proxy_spellings( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("HTTP_PROXY", "http://127.0.0.1:10080") + monkeypatch.setenv("http_proxy", "http://127.0.0.1:10080") + + from tools.build_tributo_image import _no_pandafan_environment + + environment = _no_pandafan_environment() + + assert all( + name not in environment + for name in ( + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "http_proxy", + "https_proxy", + "all_proxy", + ) + ) + + +def test_manifest_hash_is_canonical() -> None: + left = {"b": 2, "a": [1, True]} + right = {"a": [1, True], "b": 2} + + assert canonical_json(left) == canonical_json(right) + assert sha256_bytes(canonical_json(left)) == sha256_bytes(canonical_json(right)) + + +def test_build_image_seals_dependency_closure_without_docker( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config = load_config(CONFIG, root=ROOT) + built_manifests: list[str] = [] + + def fake_build_once(*args: object, manifest_sha256: str, **kwargs: object) -> None: + built_manifests.append(manifest_sha256) + + def fake_validate( + *args: object, manifest_sha256: str, **kwargs: object + ) -> dict[str, str]: + return {"Id": "sha256:" + "a" * 64} + + distributions = { + "bayesian-optimization": "1.4.3", + "pip": "25.0", + "ray": "2.55.1", + "tributo": "1.0.0", + } + monkeypatch.setattr("tools.build_tributo_image._build_once", fake_build_once) + monkeypatch.setattr( + "tools.build_tributo_image._prepare_pinned_images", lambda *args, **kwargs: None + ) + monkeypatch.setattr("tools.build_tributo_image._validate_image", fake_validate) + monkeypatch.setattr( + "tools.build_tributo_image._installed_distributions", + lambda *args, **kwargs: distributions, + ) + + manifest, profile = build_image(config, root=ROOT, output_dir=tmp_path / "result") + + assert built_manifests[0] == "unsealed" + assert len(built_manifests[1]) == 64 + assert manifest["manifest_sha256"] == built_manifests[1] + assert profile.image_digest == "a" * 64 + assert (tmp_path / "result" / "image-profile.json").is_file() + + +def test_manifest_core_contains_alpha_and_runtime_closure() -> None: + config = load_config(CONFIG, root=ROOT) + core = _manifest_core( + config, + root=ROOT, + distributions={"ray": "2.55.1"}, + external_wheels=[], + ) + + assert core["platform"] == config.platform + assert core["runtime_extras"] == list(RUNTIME_EXTRAS) + assert core["alpha_capabilities"] == [ + "explainability", + "vector_index", + "kafka_streaming", + "pipeline", + "graph", + "causal", + ] + assert "torch_geometric" in core["required_imports"] + assert "bayesian-optimization" in core["required_distributions"] + assert core["pip_check_baseline"] == list(pip_check_baseline(config.platform)) + assert core["image_sources"]["base_image"]["mirror"] == BASE_IMAGE_MIRROR + assert core["image_sources"]["uv_image"]["mirror"] == UV_IMAGE_MIRROR diff --git a/tests/tools/test_tributo_it.py b/tests/tools/test_tributo_it.py index 28bd6f5..d075b90 100644 --- a/tests/tools/test_tributo_it.py +++ b/tests/tools/test_tributo_it.py @@ -123,6 +123,32 @@ def test_profile_rejects_mutable_minio_image(tmp_path: Path) -> None: tributo_it.load_profile(profile.name, root=tmp_path) +def test_domestic_mirror_reference_maps_supported_registries() -> None: + digest = "sha256:" + "a" * 64 + + assert tributo_it._domestic_mirror_reference(f"minio/minio:latest@{digest}") == ( + f"docker.m.daocloud.io/minio/minio:latest@{digest}" + ) + assert ( + tributo_it._domestic_mirror_reference(f"ghcr.io/astral-sh/uv:0.11.23@{digest}") + == f"ghcr.m.daocloud.io/astral-sh/uv:0.11.23@{digest}" + ) + custom = f"quay.io/example/image:1@{digest}" + assert tributo_it._domestic_mirror_reference(custom) == custom + + +def test_docker_command_environment_removes_pandafan_proxy_variables( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("HTTP_PROXY", "http://127.0.0.1:10080") + monkeypatch.setenv("http_proxy", "http://127.0.0.1:10080") + + assert all( + variable not in tributo_it._docker_environment() + for variable in tributo_it.DOCKER_PROXY_VARIABLES + ) + + @pytest.mark.parametrize( "project", [ diff --git a/tests/training/test_data_loader.py b/tests/training/test_data_loader.py index 851f73d..5616370 100644 --- a/tests/training/test_data_loader.py +++ b/tests/training/test_data_loader.py @@ -105,9 +105,22 @@ def test_s3_unsupported_format_raises(): pass -def test_doris_requires_independent_ray_doris_binding() -> None: +def test_doris_requires_independent_ray_doris_binding( + monkeypatch: pytest.MonkeyPatch, +) -> None: """Tributo never falls back to its former in-process MySQL reader.""" - with pytest.raises(EngineNotAvailableError, match=r"ray-doris\[mysql,flight\]"): + import tributo.data.bindings as builtin_bindings + + installed_version = builtin_bindings._distribution_version + monkeypatch.setattr(builtin_bindings, "_DEFAULT_BINDINGS", None) + monkeypatch.setattr( + builtin_bindings, + "_distribution_version", + lambda name: None if name == "ray-doris" else installed_version(name), + ) + with pytest.raises( + EngineNotAvailableError, match=r"ray-doris==1\.0.*tributo\[mysql\]" + ): load_ray_dataset_from_source( { "type": "sql", diff --git a/tests/vector_index/test_ci_contract.py b/tests/vector_index/test_ci_contract.py index 31ddf6c..87c80e5 100644 --- a/tests/vector_index/test_ci_contract.py +++ b/tests/vector_index/test_ci_contract.py @@ -69,15 +69,21 @@ def test_vector_runtime_is_exactly_versioned_in_the_shared_profile() -> None: assert "vector-index" in profile["extras"] assert profile["version_contract"] == { "daft_prefix": "0.7.", + "daft_clickhouse": "1.0", + "daft_doris": "1.0", "lance_ray": "0.5.0", "pyarrow": "19.0.1", "pylance": "9.0.0", "ray": "2.55.1", + "ray_doris": "1.0", } assert "--extra vector-index" in dockerfile assert "m.version('pylance') == '9.0.0'" in dockerfile assert "m.version('lance-ray') == '0.5.0'" in dockerfile assert "m.version('pyarrow') == '19.0.1'" in dockerfile + assert "m.version('daft-clickhouse') == '1.0'" in dockerfile + assert "m.version('daft-doris') == '1.0'" in dockerfile + assert "m.version('ray-doris') == '1.0'" in dockerfile def test_vector_project_names_are_scoped_and_cli_dispatches(monkeypatch) -> None: diff --git a/tools/build_tributo_image.py b/tools/build_tributo_image.py new file mode 100644 index 0000000..101a2d5 --- /dev/null +++ b/tools/build_tributo_image.py @@ -0,0 +1,1016 @@ +"""Build and attest the pinned Tributo full runtime image. + +The command is intentionally JSON-configured and fail-closed. It builds the +same image twice: the first build discovers the dependency closure, and the +second build seals that closure into an OCI label. The emitted manifest and +``ImageProfile`` are the hand-off consumed by Ray runtime selection and image +validation; this tool does not submit a Ray job or publish an image. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform as host_platform +import re +import shutil +import subprocess +import sys +import tempfile +import tomllib +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterator, Mapping, Sequence + +from packaging.utils import ( + InvalidWheelFilename, + canonicalize_name, + parse_wheel_filename, +) + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from tools.runtime_image_contract import ( # noqa: E402 + REQUIRED_DISTRIBUTION_VERSIONS, + REQUIRED_DISTRIBUTIONS, + REQUIRED_IMPORTS, +) + +try: + from tributo.algorithms.api.artifacts import ImageProfile +except ModuleNotFoundError: # pragma: no cover - direct source checkout fallback + _SOURCE_ROOT = Path(__file__).resolve().parents[1] / "src" + if str(_SOURCE_ROOT) not in sys.path: + sys.path.insert(0, str(_SOURCE_ROOT)) + from tributo.algorithms.api.artifacts import ImageProfile + + +DOCKERFILE = ROOT / "docker" / "tributo-runtime" / "Dockerfile" +BASE_IMAGE = ( + "rayproject/ray:2.55.1-py312@" + "sha256:911245f2478ad2e9f67ac13978dc2a75bcae0498b9f188b10bba703324b78379" +) +UV_IMAGE = ( + "ghcr.io/astral-sh/uv:0.11.23@" + "sha256:d0a0a753ab981624b49c97abc98821c1c09f4ca69d1ef5cee69c501be3d88479" +) +BASE_IMAGE_MIRROR = ( + "docker.m.daocloud.io/rayproject/ray:2.55.1-py312@" + "sha256:911245f2478ad2e9f67ac13978dc2a75bcae0498b9f188b10bba703324b78379" +) +UV_IMAGE_MIRROR = ( + "ghcr.m.daocloud.io/astral-sh/uv:0.11.23@" + "sha256:d0a0a753ab981624b49c97abc98821c1c09f4ca69d1ef5cee69c501be3d88479" +) +LOCAL_BASE_IMAGE = "tributo-ray-base:2.55.1-py312" +LOCAL_UV_IMAGE = "tributo-uv:0.11.23" +PLATFORM_AUTO = "auto" +SUPPORTED_PLATFORMS = ("linux/amd64", "linux/arm64") +RAY_VERSION = "2.55.1" +PYTHON_VERSION = "3.12" +PYTHON_SPEC = ">=3.12,<3.14" +PROFILE_ID = "tributo.runtime.full" +IMAGE_DIGEST = re.compile(r"^[0-9a-f]{64}$") +IMAGE_REFERENCE = re.compile(r"^.+:[^/@]+@sha256:[0-9a-f]{64}$") +PULL_DIGEST = re.compile(r"(?m)^Digest:\s+(sha256:[0-9a-f]{64})\s*$") +RUNTIME_IMAGE = re.compile(r"^[^\s@]+:[^\s@]+$") + +RUNTIME_EXTRAS = ( + "data", + "data-daft", + "vector-index", + "postgresql", + "clickhouse", + "mysql", + "doris-flight", + "s3", + "model-export", + "model-export-torch", + "hf", + "model-export-hf", + "training", + "tune", + "explainability", + "identity", + "streaming", + "grpc", + "registry", + "graph", + "causal", + "streaming-inference", +) + +ALPHA_CAPABILITIES = ( + "explainability", + "vector_index", + "kafka_streaming", + "pipeline", + "graph", + "causal", +) +ARM64_PIP_CHECK_BASELINE = ( + "nvidia-cusparselt-cu13 0.8.1 is not supported on this platform", +) + + +class ImageBuildError(RuntimeError): + """Raised when an image cannot satisfy the reproducible build contract.""" + + +@dataclass(frozen=True) +class RuntimeImageConfig: + """Validated inputs for one full runtime image build.""" + + image: str + base_image: str + uv_image: str + base_image_mirror: str + uv_image_mirror: str + platform: str + runtime_extras: tuple[str, ...] + external_wheelhouse: Path | None + + +def canonical_json(value: object) -> bytes: + """Serialize attestation data without whitespace or nondeterminism.""" + return json.dumps( + value, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + + +def sha256_bytes(value: bytes) -> str: + """Return the lower-case SHA-256 digest for ``value``.""" + return hashlib.sha256(value).hexdigest() + + +def normalize_platform(value: object) -> str: + """Normalize and validate a supported Linux container platform.""" + if not isinstance(value, str): + raise ImageBuildError("platform must be a string") + try: + operating_system, architecture = value.strip().split("/", 1) + except ValueError as exc: + raise ImageBuildError(f"invalid Docker platform {value!r}") from exc + architecture = {"aarch64": "arm64", "x86_64": "amd64"}.get( + architecture, architecture + ) + normalized = f"{operating_system}/{architecture}" + if normalized not in SUPPORTED_PLATFORMS: + raise ImageBuildError(f"unsupported Docker platform {value!r}") + return normalized + + +def detect_host_platform() -> str: + """Return the native Linux target corresponding to the host architecture.""" + machine = host_platform.machine().lower() + architecture = {"aarch64": "arm64", "arm64": "arm64"}.get( + machine, + {"x86_64": "amd64", "amd64": "amd64"}.get(machine), + ) + if architecture is None: + raise ImageBuildError( + "cannot select a native runtime image for host architecture " + f"{machine!r}; pass --platform linux/amd64 or --platform linux/arm64" + ) + return f"linux/{architecture}" + + +def resolve_platform(value: object, *, override: str | None = None) -> str: + """Resolve config ``auto`` or an explicit platform to a concrete target.""" + candidate = override if override is not None else value + if candidate == PLATFORM_AUTO: + return detect_host_platform() + return normalize_platform(candidate) + + +def platform_machine(value: str) -> str: + """Return the Linux machine marker for a concrete Docker platform.""" + normalized = normalize_platform(value) + return "x86_64" if normalized == "linux/amd64" else "aarch64" + + +def platform_wheel_tags(value: str) -> tuple[str, ...]: + """Return the platform-specific CPython 3.12 tags accepted by the image.""" + normalized = normalize_platform(value) + architecture = "x86_64" if normalized == "linux/amd64" else "aarch64" + return ( + f"cp312-cp312-manylinux_2_17_{architecture}", + f"cp312-cp312-manylinux2014_{architecture}", + "py3-none-any", + ) + + +def pip_check_baseline(value: str) -> tuple[str, ...]: + """Return only the documented vendor metadata exception for one target.""" + return ( + ARM64_PIP_CHECK_BASELINE if normalize_platform(value) == "linux/arm64" else () + ) + + +def local_base_image(value: str) -> str: + """Return an architecture-scoped local Ray base-image tag.""" + architecture = normalize_platform(value).rsplit("/", 1)[1] + return f"{LOCAL_BASE_IMAGE}-{architecture}" + + +def local_uv_image(value: str) -> str: + """Return an architecture-scoped local uv image tag.""" + architecture = normalize_platform(value).rsplit("/", 1)[1] + return f"{LOCAL_UV_IMAGE}-{architecture}" + + +def _validate_digest_reference(value: object, field: str) -> str: + rendered = str(value) + if IMAGE_REFERENCE.fullmatch(rendered) is None: + raise ImageBuildError( + f"{field} must use tag@sha256:<64 lower-case hex> form: {rendered!r}" + ) + return rendered + + +def _validate_config_payload( + payload: object, + *, + root: Path, + platform_override: str | None = None, +) -> RuntimeImageConfig: + if not isinstance(payload, dict): + raise ImageBuildError("image configuration must be a JSON object") + expected_keys = { + "schema_version", + "image", + "base_image", + "uv_image", + "base_image_mirror", + "uv_image_mirror", + "platform", + "runtime_extras", + "external_wheelhouse", + } + unknown = sorted(set(payload) - expected_keys) + missing = sorted(expected_keys - set(payload)) + if unknown or missing: + raise ImageBuildError( + f"image configuration keys differ; missing={missing}, unknown={unknown}" + ) + if payload["schema_version"] != 1: + raise ImageBuildError("unsupported image configuration schema") + + image = str(payload["image"]) + if RUNTIME_IMAGE.fullmatch(image) is None: + raise ImageBuildError("image must be a readable tag such as repository:tag") + base_image = _validate_digest_reference(payload["base_image"], "base_image") + uv_image = _validate_digest_reference(payload["uv_image"], "uv_image") + if base_image != BASE_IMAGE: + raise ImageBuildError(f"base_image must be the pinned Ray image {BASE_IMAGE!r}") + if uv_image != UV_IMAGE: + raise ImageBuildError(f"uv_image must be the pinned uv image {UV_IMAGE!r}") + base_image_mirror = _validate_digest_reference( + payload["base_image_mirror"], "base_image_mirror" + ) + uv_image_mirror = _validate_digest_reference( + payload["uv_image_mirror"], "uv_image_mirror" + ) + if base_image_mirror != BASE_IMAGE_MIRROR: + raise ImageBuildError( + "base_image_mirror must be the pinned DaoCloud Ray mirror " + f"{BASE_IMAGE_MIRROR!r}" + ) + if uv_image_mirror != UV_IMAGE_MIRROR: + raise ImageBuildError( + f"uv_image_mirror must be the pinned DaoCloud uv mirror {UV_IMAGE_MIRROR!r}" + ) + configured_platform = payload["platform"] + if configured_platform != PLATFORM_AUTO: + normalize_platform(configured_platform) + platform = resolve_platform(configured_platform, override=platform_override) + + raw_extras = payload["runtime_extras"] + if not isinstance(raw_extras, list) or any( + not isinstance(item, str) for item in raw_extras + ): + raise ImageBuildError("runtime_extras must be a JSON string list") + extras = tuple(raw_extras) + if len(set(extras)) != len(extras): + raise ImageBuildError("runtime_extras must not contain duplicates") + if extras != RUNTIME_EXTRAS: + raise ImageBuildError( + "runtime_extras must exactly match the full CPU runtime closure: " + f"expected={list(RUNTIME_EXTRAS)!r}, actual={list(extras)!r}" + ) + + external = payload["external_wheelhouse"] + external_path: Path | None + if external is None: + external_path = None + else: + if not isinstance(external, str) or not external: + raise ImageBuildError("external_wheelhouse must be null or a path") + candidate = Path(external) + if candidate.is_absolute() or ".." in candidate.parts: + raise ImageBuildError("external_wheelhouse must be a relative path") + external_path = (root / candidate).resolve() + try: + external_path.relative_to(root.resolve()) + except ValueError as exc: + raise ImageBuildError( + "external_wheelhouse must stay under repository root" + ) from exc + if not external_path.is_dir(): + raise ImageBuildError( + f"external_wheelhouse is not a directory: {external_path}" + ) + + return RuntimeImageConfig( + image=image, + base_image=base_image, + uv_image=uv_image, + base_image_mirror=base_image_mirror, + uv_image_mirror=uv_image_mirror, + platform=platform, + runtime_extras=RUNTIME_EXTRAS, + external_wheelhouse=external_path, + ) + + +def load_config( + path: Path, + *, + root: Path = ROOT, + platform_override: str | None = None, +) -> RuntimeImageConfig: + """Load and validate a JSON runtime image configuration.""" + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ImageBuildError(f"cannot read image configuration: {path}") from exc + return _validate_config_payload( + payload, root=root, platform_override=platform_override + ) + + +def _hash_file(path: Path) -> tuple[int, str]: + digest = hashlib.sha256() + size = 0 + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + size += len(chunk) + return size, digest.hexdigest() + + +def wheel_records(path: Path | None) -> list[dict[str, Any]]: + """Return safe, digest-only records for an optional external wheelhouse.""" + if path is None: + return [] + if not path.is_dir(): + raise ImageBuildError(f"external wheelhouse is not a directory: {path}") + records: list[dict[str, Any]] = [] + seen: set[str] = set() + for wheel in sorted(path.iterdir(), key=lambda item: item.name): + if wheel.suffix != ".whl": + continue + if wheel.is_symlink() or not wheel.is_file(): + raise ImageBuildError(f"external wheel must be a regular file: {wheel}") + try: + name, version, _build, _tags = parse_wheel_filename(wheel.name) + except InvalidWheelFilename as exc: + raise ImageBuildError( + f"invalid external wheel filename: {wheel.name!r}" + ) from exc + normalized_name = canonicalize_name(str(name)) + if normalized_name in seen: + raise ImageBuildError( + f"duplicate external wheel distribution: {normalized_name!r}" + ) + seen.add(normalized_name) + size, digest = _hash_file(wheel) + records.append( + { + "filename": wheel.name, + "name": normalized_name, + "version": str(version), + "size": size, + "sha256": digest, + } + ) + return records + + +@contextmanager +def prepared_wheelhouse(path: Path | None) -> Iterator[Path]: + """Copy external wheels into a safe named build context.""" + with tempfile.TemporaryDirectory(prefix="tributo-image-wheelhouse-") as temporary: + destination = Path(temporary) + if path is not None: + for source in sorted(path.iterdir(), key=lambda item: item.name): + if source.suffix != ".whl": + continue + if source.is_symlink() or not source.is_file(): + raise ImageBuildError( + f"external wheel must be a regular file: {source}" + ) + shutil.copy2(source, destination / source.name) + yield destination + + +def build_command( + config: RuntimeImageConfig, + *, + root: Path, + wheelhouse_context: Path, + manifest_sha256: str, + metadata_file: Path | None = None, +) -> list[str]: + """Construct the shell-free Buildx command for one image build.""" + if not IMAGE_DIGEST.fullmatch(manifest_sha256) and manifest_sha256 != "unsealed": + raise ImageBuildError("manifest_sha256 must be a digest or 'unsealed'") + command = [ + "docker", + "buildx", + "build", + "--load", + "--platform", + config.platform, + "--file", + str(root / "docker" / "tributo-runtime" / "Dockerfile"), + "--tag", + config.image, + "--build-arg", + f"BASE_IMAGE={local_base_image(config.platform)}", + "--build-arg", + f"UV_IMAGE={local_uv_image(config.platform)}", + "--build-arg", + f"TRIBUTO_BASE_IMAGE={config.base_image}", + "--build-arg", + f"TRIBUTO_PLATFORM={config.platform}", + "--build-arg", + f"TRIBUTO_MANIFEST_SHA256={manifest_sha256}", + "--build-arg", + f"TRIBUTO_RUNTIME_EXTRAS={','.join(config.runtime_extras)}", + "--build-arg", + f"TRIBUTO_VERSION={project_version(root)}", + "--build-context", + f"external-wheelhouse={wheelhouse_context}", + ] + if metadata_file is not None: + command.extend(("--metadata-file", str(metadata_file))) + command.append(str(root)) + return command + + +def project_version(root: Path = ROOT) -> str: + """Read the package version without importing the project.""" + try: + payload = tomllib.loads((root / "pyproject.toml").read_text(encoding="utf-8")) + return str(payload["project"]["version"]) + except (KeyError, OSError, TypeError, tomllib.TOMLDecodeError) as exc: + raise ImageBuildError( + "cannot read project.version from pyproject.toml" + ) from exc + + +class _CommandLog: + def __init__(self) -> None: + self.lines: list[str] = [] + + def command(self, args: Sequence[str]) -> None: + self.lines.append("$ " + " ".join(str(item) for item in args)) + + def result(self, result: subprocess.CompletedProcess[str]) -> None: + if result.stdout: + self.lines.append(result.stdout.rstrip()) + if result.stderr: + self.lines.append(result.stderr.rstrip()) + + +def _no_pandafan_environment( + base: Mapping[str, str] | None = None, +) -> dict[str, str]: + """Return an environment that cannot route Docker traffic through PandaFan.""" + environment = dict(os.environ if base is None else base) + for name in ( + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "http_proxy", + "https_proxy", + "all_proxy", + ): + environment.pop(name, None) + return environment + + +def _run( + args: Sequence[str], + *, + cwd: Path = ROOT, + capture_output: bool = True, + log: _CommandLog | None = None, + env: Mapping[str, str] | None = None, +) -> subprocess.CompletedProcess[str]: + rendered = [str(item) for item in args] + if log is not None: + log.command(rendered) + print("+ " + " ".join(rendered), flush=True) + command_env = env + if command_env is None and rendered and rendered[0] == "docker": + command_env = _no_pandafan_environment() + result = subprocess.run( + rendered, + cwd=cwd, + text=True, + capture_output=capture_output, + check=False, + env=dict(command_env) if command_env is not None else None, + ) + if log is not None and capture_output: + log.result(result) + if result.returncode != 0: + details = (result.stderr or result.stdout or "").strip() + raise ImageBuildError( + f"command failed with exit code {result.returncode}: {' '.join(rendered)}" + + (f"\n{details}" if details else "") + ) + return result + + +def _prepare_pinned_image( + *, + mirror: str, + canonical: str, + local: str, + platform: str, + log: _CommandLog | None = None, +) -> None: + """Pull a mirror image, verify its digest/platform, and create a local tag.""" + mirror_tag = mirror.rsplit("@", 1)[0] + pull_result = _run( + ["docker", "pull", "--platform", platform, mirror_tag], + log=log, + ) + expected_digest = canonical.rsplit("@", 1)[-1] + pull_output = "\n".join( + part for part in (pull_result.stdout, pull_result.stderr) if part + ) + pull_matches = PULL_DIGEST.findall(pull_output) + if pull_matches != [expected_digest]: + raise ImageBuildError( + f"mirror image digest mismatch for {mirror}: " + f"expected={expected_digest}, pull_output={pull_output!r}" + ) + inspected = _inspect_image(mirror_tag, log=log) + actual_platform = f"{inspected.get('Os', '')}/{inspected.get('Architecture', '')}" + if actual_platform != platform: + raise ImageBuildError( + f"mirror image platform mismatch for {mirror}: " + f"expected={platform}, actual={actual_platform}" + ) + _run(["docker", "tag", mirror_tag, local], log=log) + + +def _prepare_pinned_images( + config: RuntimeImageConfig, + *, + log: _CommandLog | None = None, +) -> None: + """Prepare both digest-pinned image inputs without using the host proxy.""" + _prepare_pinned_image( + mirror=config.base_image_mirror, + canonical=config.base_image, + local=local_base_image(config.platform), + platform=config.platform, + log=log, + ) + _prepare_pinned_image( + mirror=config.uv_image_mirror, + canonical=config.uv_image, + local=local_uv_image(config.platform), + platform=config.platform, + log=log, + ) + + +def _inspect_image(image: str, *, log: _CommandLog | None = None) -> dict[str, Any]: + result = _run(["docker", "image", "inspect", image], log=log) + try: + payload = json.loads(result.stdout) + inspected = payload[0] + except (IndexError, KeyError, TypeError, json.JSONDecodeError) as exc: + raise ImageBuildError( + f"unexpected docker image inspect result for {image}" + ) from exc + if not isinstance(inspected, dict): + raise ImageBuildError(f"invalid docker image inspect result for {image}") + return inspected + + +def _container_python( + image: str, + code: str, + *, + platform: str, + log: _CommandLog | None = None, +) -> str: + result = _run( + [ + "docker", + "run", + "--rm", + "--platform", + platform, + "--pull", + "never", + "--entrypoint", + "python", + image, + "-c", + code, + ], + log=log, + ) + return result.stdout.strip() + + +def _validate_image( + config: RuntimeImageConfig, + *, + root: Path = ROOT, + manifest_sha256: str, + log: _CommandLog | None = None, +) -> dict[str, Any]: + inspected = _inspect_image(config.image, log=log) + labels = inspected.get("Config", {}).get("Labels") or {} + expected_labels = { + "org.tributo.base-image": config.base_image, + "org.tributo.ray-version": RAY_VERSION, + "org.tributo.python-version": PYTHON_VERSION, + "org.tributo.platform": config.platform, + "org.tributo.runtime-extras": ",".join(config.runtime_extras), + "org.tributo.manifest-sha256": manifest_sha256, + } + mismatches = { + key: {"expected": value, "actual": labels.get(key)} + for key, value in expected_labels.items() + if labels.get(key) != value + } + actual_platform = f"{inspected.get('Os', '')}/{inspected.get('Architecture', '')}" + if actual_platform != config.platform: + mismatches["platform"] = { + "expected": config.platform, + "actual": actual_platform, + } + user = str(inspected.get("Config", {}).get("User") or "") + if user in {"", "0", "0:0", "root"}: + mismatches["Config.User"] = { + "expected": "non-root", + "actual": user or "", + } + if mismatches: + raise ImageBuildError(f"runtime image labels/platform invalid: {mismatches}") + + pip_check_code = ( + "import subprocess, sys; " + "result = subprocess.run([sys.executable, '-m', 'pip', 'check'], " + "capture_output=True, text=True); " + "output = (result.stdout + result.stderr).strip(); " + f"baseline = {pip_check_baseline(config.platform)!r}; " + "assert result.returncode == 0 or tuple(output.splitlines()) == baseline, " + "(result.returncode, output)" + ) + _container_python(config.image, pip_check_code, platform=config.platform, log=log) + seal_code = ( + "from pathlib import Path; " + f"assert Path('/opt/tributo-image/manifest-seal').read_text().strip() " + f"== {manifest_sha256!r}" + ) + _container_python(config.image, seal_code, platform=config.platform, log=log) + import_code = ( + "import importlib, importlib.metadata as metadata, sys; " + f"assert sys.version_info[:2] == (3, 12); " + f"assert metadata.version('ray') == {RAY_VERSION!r}; " + f"assert metadata.version('tributo') == {project_version(root)!r}; " + + "; ".join(f"metadata.version({name!r})" for name in REQUIRED_DISTRIBUTIONS) + + "; " + + "; ".join( + f"assert metadata.version({name!r}) == {version!r}" + for name, version in REQUIRED_DISTRIBUTION_VERSIONS.items() + ) + + "; " + + "; ".join(f"importlib.import_module({name!r})" for name in REQUIRED_IMPORTS) + ) + _container_python(config.image, import_code, platform=config.platform, log=log) + _run( + [ + "docker", + "run", + "--rm", + "--platform", + config.platform, + "--pull", + "never", + "--entrypoint", + "tributo", + config.image, + "--help", + ], + log=log, + ) + image_id = str(inspected.get("Id") or "") + if ( + not image_id.startswith("sha256:") + or IMAGE_DIGEST.fullmatch(image_id.removeprefix("sha256:")) is None + ): + raise ImageBuildError(f"runtime image has invalid local ID: {image_id!r}") + return inspected + + +def _installed_distributions( + image: str, *, platform: str, log: _CommandLog | None = None +) -> dict[str, str]: + output = _container_python( + image, + "import json, pathlib; print(pathlib.Path('/opt/tributo-image/installed-distributions.json').read_text())", + platform=platform, + log=log, + ) + try: + raw = json.loads(output) + except json.JSONDecodeError as exc: + raise ImageBuildError( + "image installed-distributions.json is not valid JSON" + ) from exc + if not isinstance(raw, dict) or any( + not isinstance(k, str) or not isinstance(v, str) for k, v in raw.items() + ): + raise ImageBuildError( + "image installed-distributions.json must map names to versions" + ) + return dict( + sorted((canonicalize_name(name), version) for name, version in raw.items()) + ) + + +def _build_once( + config: RuntimeImageConfig, + *, + root: Path, + wheelhouse_context: Path, + manifest_sha256: str, + log: _CommandLog, +) -> None: + with tempfile.NamedTemporaryFile( + prefix="tributo-image-buildx-", suffix=".json" + ) as metadata: + command = build_command( + config, + root=root, + wheelhouse_context=wheelhouse_context, + manifest_sha256=manifest_sha256, + metadata_file=Path(metadata.name), + ) + _run(command, cwd=root, capture_output=False, log=log) + if Path(metadata.name).stat().st_size == 0: + raise ImageBuildError("Buildx did not produce its required metadata file") + + +def _manifest_core( + config: RuntimeImageConfig, + *, + root: Path, + distributions: dict[str, str], + external_wheels: list[dict[str, Any]], +) -> dict[str, Any]: + return { + "schema_version": 1, + "image": config.image, + "base_image": config.base_image, + "uv_image": config.uv_image, + "image_sources": { + "base_image": { + "canonical": config.base_image, + "mirror": config.base_image_mirror, + "local": local_base_image(config.platform), + }, + "uv_image": { + "canonical": config.uv_image, + "mirror": config.uv_image_mirror, + "local": local_uv_image(config.platform), + }, + }, + "platform": config.platform, + "python_version": PYTHON_VERSION, + "python_spec": PYTHON_SPEC, + "ray_version": RAY_VERSION, + "tributo_version": project_version(root), + "runtime_extras": list(config.runtime_extras), + "installed_distributions": distributions, + "external_wheels": external_wheels, + "alpha_capabilities": list(ALPHA_CAPABILITIES), + "required_imports": list(REQUIRED_IMPORTS), + "required_distributions": list(REQUIRED_DISTRIBUTIONS), + "required_distribution_versions": dict(REQUIRED_DISTRIBUTION_VERSIONS), + "pip_check_baseline": list(pip_check_baseline(config.platform)), + } + + +def _profile_manifest( + config: RuntimeImageConfig, + *, + root: Path, + distributions: dict[str, str], + external_wheels: list[dict[str, Any]], + manifest_sha256: str, + inspected: dict[str, Any], +) -> tuple[dict[str, Any], ImageProfile]: + image_id = str(inspected["Id"]) + digest = image_id.removeprefix("sha256:") + image_uri = f"{config.image}@sha256:{digest}" + manifest = { + **_manifest_core( + config, + root=root, + distributions=distributions, + external_wheels=external_wheels, + ), + "manifest_sha256": manifest_sha256, + "image_uri": image_uri, + "image_digest": digest, + "profile_id": PROFILE_ID, + } + profile = ImageProfile( + profile_id=PROFILE_ID, + image_uri=image_uri, + image_digest=digest, + ray_version=RAY_VERSION, + python_spec=PYTHON_SPEC, + python_version=PYTHON_VERSION, + sys_platform="linux", + platform_machine=platform_machine(config.platform), + wheel_tags=platform_wheel_tags(config.platform), + installed_distributions=distributions, + allow_offline_pip=True, + pip_check_baseline=pip_check_baseline(config.platform), + ) + return manifest, profile + + +def write_outputs( + output_dir: Path, + *, + manifest: dict[str, Any], + profile: ImageProfile, + distributions: dict[str, str], + log: _CommandLog, +) -> None: + """Write all attestations, refusing to overwrite an existing result.""" + if output_dir.exists() and any(output_dir.iterdir()): + raise ImageBuildError(f"output directory is not empty: {output_dir}") + output_dir.mkdir(parents=True, exist_ok=True) + files = { + "manifest.json": manifest, + "image-profile.json": profile.model_dump(mode="json"), + "installed-distributions.json": distributions, + "capabilities.json": { + "schema_version": 1, + "profile_id": PROFILE_ID, + "status": "alpha", + "capabilities": [ + {"id": capability, "status": "alpha"} + for capability in ALPHA_CAPABILITIES + ], + }, + } + for filename, payload in files.items(): + (output_dir / filename).write_bytes(canonical_json(payload) + b"\n") + (output_dir / "build.log").write_text("\n".join(log.lines) + "\n", encoding="utf-8") + + +def build_image( + config: RuntimeImageConfig, + *, + root: Path = ROOT, + output_dir: Path, + output_archive: Path | None = None, +) -> tuple[dict[str, Any], ImageProfile]: + """Build, validate, seal, and emit one full runtime image.""" + dockerfile = root / "docker" / "tributo-runtime" / "Dockerfile" + if not dockerfile.is_file(): + raise ImageBuildError(f"runtime Dockerfile is missing: {dockerfile}") + external_wheels = wheel_records(config.external_wheelhouse) + log = _CommandLog() + _prepare_pinned_images(config, log=log) + with prepared_wheelhouse(config.external_wheelhouse) as wheelhouse_context: + _build_once( + config, + root=root, + wheelhouse_context=wheelhouse_context, + manifest_sha256="unsealed", + log=log, + ) + _validate_image(config, root=root, manifest_sha256="unsealed", log=log) + distributions = _installed_distributions( + config.image, platform=config.platform, log=log + ) + core = _manifest_core( + config, + root=root, + distributions=distributions, + external_wheels=external_wheels, + ) + manifest_sha256 = sha256_bytes(canonical_json(core)) + _build_once( + config, + root=root, + wheelhouse_context=wheelhouse_context, + manifest_sha256=manifest_sha256, + log=log, + ) + inspected = _validate_image( + config, root=root, manifest_sha256=manifest_sha256, log=log + ) + final_distributions = _installed_distributions( + config.image, platform=config.platform, log=log + ) + if final_distributions != distributions: + raise ImageBuildError( + "sealed image dependency closure differs from discovery build" + ) + manifest, profile = _profile_manifest( + config, + root=root, + distributions=final_distributions, + external_wheels=external_wheels, + manifest_sha256=manifest_sha256, + inspected=inspected, + ) + write_outputs( + output_dir, + manifest=manifest, + profile=profile, + distributions=final_distributions, + log=log, + ) + if output_archive is not None: + output_archive.parent.mkdir(parents=True, exist_ok=True) + _run( + ["docker", "save", "--output", str(output_archive), config.image], + log=log, + ) + return manifest, profile + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--config", type=Path, default=ROOT / "tools" / "tributo-runtime-full.json" + ) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--output-archive", type=Path) + parser.add_argument("--repo-root", type=Path, default=ROOT) + parser.add_argument( + "--platform", + dest="platform_override", + help="target platform; defaults to the native host architecture", + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + root = args.repo_root.resolve() + config = load_config( + args.config.resolve(), + root=root, + platform_override=args.platform_override, + ) + manifest, profile = build_image( + config, + root=root, + output_dir=args.output_dir.resolve(), + output_archive=args.output_archive.resolve() if args.output_archive else None, + ) + print( + json.dumps( + { + "image_uri": profile.image_uri, + "image_digest": profile.image_digest, + "manifest_sha256": manifest["manifest_sha256"], + "output_dir": str(args.output_dir.resolve()), + }, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/generate_distributions.py b/tools/generate_distributions.py new file mode 100644 index 0000000..69167cd --- /dev/null +++ b/tools/generate_distributions.py @@ -0,0 +1,26 @@ +"""Write the normalized installed-distribution inventory inside the image.""" + +from __future__ import annotations + +import importlib.metadata +import json +from pathlib import Path + +from packaging.utils import canonicalize_name + +OUTPUT = Path("/opt/tributo-image/installed-distributions.json") + + +def main() -> None: + distributions = { + canonicalize_name(metadata.metadata["Name"]): metadata.version + for metadata in importlib.metadata.distributions() + if metadata.metadata.get("Name") + } + OUTPUT.write_text( + json.dumps(dict(sorted(distributions.items())), indent=2, sort_keys=True) + "\n" + ) + + +if __name__ == "__main__": + main() diff --git a/tools/runtime_image_contract.py b/tools/runtime_image_contract.py new file mode 100644 index 0000000..0215b3a --- /dev/null +++ b/tools/runtime_image_contract.py @@ -0,0 +1,69 @@ +"""Shared import and distribution contract for the Tributo runtime image.""" + +from __future__ import annotations + +REQUIRED_IMPORTS = ( + "tributo.data", + "tributo.algorithms", + "tributo.training", + "tributo.exporting", + "tributo.inference", + "tributo.serving", + "tributo.streaming", + "tributo.vector_index", + "tributo.explainability", + "tributo.registry", + "tributo.pipeline", + "daft", + "lance_ray", + "lance", + "pyiceberg", + "torch", + "xgboost", + "onnx", + "onnxruntime", + "transformers", + "shap", + "mlflow", + "grpc", + "confluent_kafka", + "torch_geometric", + "daft_clickhouse", + "daft_doris", + "ray_doris", +) + +REQUIRED_DISTRIBUTIONS = ( + "ray", + "tributo", + "daft", + "lance-ray", + "pylance", + "pyiceberg", + "torch", + "xgboost", + "onnx", + "onnxruntime", + "transformers", + "shap", + "mlflow", + "grpcio", + "confluent-kafka", + "torch-geometric", + "dowhy", + "econml", + "bayesian-optimization", + "daft-clickhouse", + "daft-doris", + "ray-doris", + "clickhouse-connect", + "PyMySQL", + "adbc-driver-manager", + "adbc-driver-flightsql", +) + +REQUIRED_DISTRIBUTION_VERSIONS = { + "daft-clickhouse": "1.0", + "daft-doris": "1.0", + "ray-doris": "1.0", +} diff --git a/tools/tributo-runtime-full.json b/tools/tributo-runtime-full.json new file mode 100644 index 0000000..2535f5e --- /dev/null +++ b/tools/tributo-runtime-full.json @@ -0,0 +1,34 @@ +{ + "schema_version": 1, + "image": "tributo-runtime-full:local", + "base_image": "rayproject/ray:2.55.1-py312@sha256:911245f2478ad2e9f67ac13978dc2a75bcae0498b9f188b10bba703324b78379", + "uv_image": "ghcr.io/astral-sh/uv:0.11.23@sha256:d0a0a753ab981624b49c97abc98821c1c09f4ca69d1ef5cee69c501be3d88479", + "base_image_mirror": "docker.m.daocloud.io/rayproject/ray:2.55.1-py312@sha256:911245f2478ad2e9f67ac13978dc2a75bcae0498b9f188b10bba703324b78379", + "uv_image_mirror": "ghcr.m.daocloud.io/astral-sh/uv:0.11.23@sha256:d0a0a753ab981624b49c97abc98821c1c09f4ca69d1ef5cee69c501be3d88479", + "platform": "auto", + "runtime_extras": [ + "data", + "data-daft", + "vector-index", + "postgresql", + "clickhouse", + "mysql", + "doris-flight", + "s3", + "model-export", + "model-export-torch", + "hf", + "model-export-hf", + "training", + "tune", + "explainability", + "identity", + "streaming", + "grpc", + "registry", + "graph", + "causal", + "streaming-inference" + ], + "external_wheelhouse": null +} diff --git a/tools/tributo_it.py b/tools/tributo_it.py index 1fb69b0..9166f53 100644 --- a/tools/tributo_it.py +++ b/tools/tributo_it.py @@ -70,6 +70,16 @@ EXCLUDED_FILE_NAMES = {".coverage", ".pypirc", "id_rsa", "id_ed25519"} PROJECT_PREFIXES = frozenset({"tributo-ingestion", "tributo-lance-vector"}) DIGEST_REFERENCE_PATTERN = re.compile(r"^.+:[^/@]+@sha256:[0-9a-f]{64}$") +LOCAL_RUNTIME_BASE_IMAGE = "tributo-ray-base:2.55.1-py312" +LOCAL_RUNTIME_UV_IMAGE = "tributo-uv:0.11.23" +DOCKER_PROXY_VARIABLES = ( + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "http_proxy", + "https_proxy", + "all_proxy", +) class TributoITError(RuntimeError): @@ -100,6 +110,16 @@ def base_image(self) -> str: def uv_image(self) -> str: return str(self.definition["uv_image"]) + @property + def base_image_mirror(self) -> str | None: + value = self.definition.get("base_image_mirror") + return str(value) if value is not None else None + + @property + def uv_image_mirror(self) -> str | None: + value = self.definition.get("uv_image_mirror") + return str(value) if value is not None else None + @property def tool_image(self) -> str: return str(self.definition["tool_image"]) @@ -153,10 +173,11 @@ def _run( capture_output: bool = True, ) -> subprocess.CompletedProcess[str]: print(f"+ {shlex.join(str(part) for part in args)}", flush=True) + command_env = _docker_environment(env) if args and args[0] == "docker" else env result = subprocess.run( [str(part) for part in args], cwd=cwd, - env=env, + env=command_env, text=True, capture_output=capture_output, check=False, @@ -171,6 +192,14 @@ def _run( return result +def _docker_environment(env: dict[str, str] | None = None) -> dict[str, str]: + """Return a Docker environment without PandaFan proxy variables.""" + result = dict(os.environ if env is None else env) + for variable in DOCKER_PROXY_VARIABLES: + result.pop(variable, None) + return result + + def _canonical_json(value: object) -> bytes: return json.dumps( value, @@ -228,6 +257,16 @@ def load_profile( str(definition["tool_image"]), str(definition["uv_image"]), ] + mirror_fields = [ + definition.get("base_image_mirror"), + definition.get("uv_image_mirror"), + ] + if any(value is not None for value in mirror_fields): + if any(value is None for value in mirror_fields): + raise TributoITError( + f"runtime profile {name!r} must define both base/uv image mirrors" + ) + image_fields.extend(str(value) for value in mirror_fields) invalid_images = [ reference for reference in image_fields @@ -308,6 +347,9 @@ def runtime_identity(profile: RuntimeProfile, platform: str) -> RuntimeIdentity: "version_contract", ) } + for key in ("base_image_mirror", "uv_image_mirror"): + if key in profile.definition: + runtime_definition[key] = profile.definition[key] parts = [ ("schema", b"tributo-it-runtime-v2"), ("profile", _canonical_json(runtime_definition)), @@ -368,6 +410,9 @@ def _version_check_code(identity: RuntimeIdentity) -> str: ("pylance", "pylance"), ("lance_ray", "lance-ray"), ("pyarrow", "pyarrow"), + ("daft_clickhouse", "daft-clickhouse"), + ("daft_doris", "daft-doris"), + ("ray_doris", "ray-doris"), ): if key in contract: checks.append( @@ -415,6 +460,8 @@ def validate_runtime_image( "docker", "run", "--rm", + "--platform", + identity.platform, "--pull", "never", "--entrypoint", @@ -430,6 +477,25 @@ def validate_runtime_image( return image_id +def _domestic_mirror_reference(reference: str) -> str: + """Map Docker Hub/GHCR references to the configured domestic mirrors.""" + image, digest = reference.split("@", 1) + parts = image.split("/", 1) + if len(parts) == 2 and ( + "." in parts[0] or ":" in parts[0] or parts[0] == "localhost" + ): + registry, remainder = parts + else: + registry, remainder = "docker.io", image + mirror_registry = { + "docker.io": "docker.m.daocloud.io", + "ghcr.io": "ghcr.m.daocloud.io", + }.get(registry) + if mirror_registry is None: + return reference + return f"{mirror_registry}/{remainder}@{digest}" + + def _docker_daemon_identity() -> str: result = _run( ["docker", "info", "--format", "{{.ID}}|{{.Name}}|{{.DockerRootDir}}"] @@ -643,14 +709,77 @@ def _pull_registry_runtime( time.sleep(min(10.0, max(0.0, remaining))) +def _prepare_mirrored_image( + *, + mirror: str, + canonical: str, + local: str, + platform: str, +) -> None: + """Pull, verify, and locally tag one digest-pinned domestic mirror image.""" + _run(["docker", "pull", "--platform", platform, mirror], capture_output=False) + inspected = _image_inspect(mirror) + if inspected is None: + raise TributoITError(f"mirror image is unavailable after pull: {mirror}") + actual_platform = f"{inspected.get('Os', '')}/{inspected.get('Architecture', '')}" + if normalize_platform(actual_platform) != normalize_platform(platform): + raise TributoITError( + f"mirror image platform mismatch for {mirror}: " + f"expected={platform}, actual={actual_platform}" + ) + expected_digest = canonical.rsplit("@", 1)[-1] + repo_digests = [str(value) for value in inspected.get("RepoDigests") or []] + if not any(value.rsplit("@", 1)[-1] == expected_digest for value in repo_digests): + raise TributoITError( + f"mirror image digest mismatch for {mirror}: " + f"expected={expected_digest}, repo_digests={repo_digests}" + ) + _run(["docker", "tag", mirror, local]) + + +def _prepare_runtime_build_inputs( + identity: RuntimeIdentity, +) -> tuple[str, str]: + """Prepare domestic mirror inputs when a runtime profile declares them.""" + profile = identity.profile + if profile.base_image_mirror is None and profile.uv_image_mirror is None: + return profile.base_image, profile.uv_image + if profile.base_image_mirror is None or profile.uv_image_mirror is None: + raise TributoITError( + f"runtime profile {profile.name!r} must define both base/uv image mirrors" + ) + _prepare_mirrored_image( + mirror=profile.base_image_mirror, + canonical=profile.base_image, + local=LOCAL_RUNTIME_BASE_IMAGE, + platform=identity.platform, + ) + _prepare_mirrored_image( + mirror=profile.uv_image_mirror, + canonical=profile.uv_image, + local=LOCAL_RUNTIME_UV_IMAGE, + platform=identity.platform, + ) + return LOCAL_RUNTIME_BASE_IMAGE, LOCAL_RUNTIME_UV_IMAGE + + def _build_runtime(identity: RuntimeIdentity) -> str: labels = _expected_labels(identity) + base_image, uv_image = _prepare_runtime_build_inputs(identity) descriptor, metadata_name = tempfile.mkstemp( prefix="tributo-buildx-", suffix=".json" ) os.close(descriptor) metadata_file = Path(metadata_name) try: + # The full runtime Dockerfile declares this named context even when no + # external connector wheels are requested. Supplying an empty context + # here keeps the generic IT helper compatible with that Dockerfile and + # does not alter the source snapshot or image contents. + wheelhouse_context = tempfile.TemporaryDirectory( + prefix="tributo-it-wheelhouse-" + ) + wheelhouse = wheelhouse_context.name command = [ "docker", "buildx", @@ -665,38 +794,50 @@ def _build_runtime(identity: RuntimeIdentity) -> str: "--metadata-file", str(metadata_file), "--build-arg", - f"BASE_IMAGE={identity.profile.base_image}", + f"BASE_IMAGE={base_image}", "--build-arg", - f"UV_IMAGE={identity.profile.uv_image}", + f"UV_IMAGE={uv_image}", + "--build-context", + f"external-wheelhouse={wheelhouse}", ] - for key, value in sorted(labels.items()): - command.extend(("--label", f"{key}={value}")) - for cache_from in filter( - None, os.environ.get("TRIBUTO_IT_BUILDX_CACHE_FROM", "").splitlines() - ): - command.extend(("--cache-from", cache_from)) - for cache_to in filter( - None, os.environ.get("TRIBUTO_IT_BUILDX_CACHE_TO", "").splitlines() - ): - command.extend(("--cache-to", cache_to)) - command.append(str(identity.profile.root)) - _run(command, capture_output=False) - if metadata_file.stat().st_size == 0: - raise TributoITError("Buildx did not produce its required metadata file") - metadata = json.loads(metadata_file.read_text(encoding="utf-8")) - print( - "Buildx metadata: " - + json.dumps( - { - key: metadata[key] - for key in sorted(metadata) - if key in {"containerimage.digest", "containerimage.config.digest"} - }, - sort_keys=True, - ), - flush=True, - ) - return validate_runtime_image(identity) + if identity.profile.base_image_mirror is not None: + command.extend( + ("--build-arg", f"TRIBUTO_BASE_IMAGE={identity.profile.base_image}") + ) + try: + for key, value in sorted(labels.items()): + command.extend(("--label", f"{key}={value}")) + for cache_from in filter( + None, os.environ.get("TRIBUTO_IT_BUILDX_CACHE_FROM", "").splitlines() + ): + command.extend(("--cache-from", cache_from)) + for cache_to in filter( + None, os.environ.get("TRIBUTO_IT_BUILDX_CACHE_TO", "").splitlines() + ): + command.extend(("--cache-to", cache_to)) + command.append(str(identity.profile.root)) + _run(command, capture_output=False) + if metadata_file.stat().st_size == 0: + raise TributoITError( + "Buildx did not produce its required metadata file" + ) + metadata = json.loads(metadata_file.read_text(encoding="utf-8")) + print( + "Buildx metadata: " + + json.dumps( + { + key: metadata[key] + for key in sorted(metadata) + if key + in {"containerimage.digest", "containerimage.config.digest"} + }, + sort_keys=True, + ), + flush=True, + ) + return validate_runtime_image(identity) + finally: + wheelhouse_context.cleanup() finally: metadata_file.unlink(missing_ok=True) @@ -768,6 +909,7 @@ def ensure_digest_image(reference: str) -> str: ) expected_digest = reference.split("@", 1)[1] readable_tag = reference.split("@", 1)[0] + mirror_reference = _domestic_mirror_reference(reference) tagged = _image_inspect(readable_tag) if tagged is not None: tagged_repo_digests = [str(value) for value in tagged.get("RepoDigests") or []] @@ -779,10 +921,13 @@ def ensure_digest_image(reference: str) -> str: f"refusing to overwrite {readable_tag}" ) - inspected = _image_inspect(reference) or tagged + inspected = _image_inspect(reference) or _image_inspect(mirror_reference) or tagged if inspected is None: - _run(["docker", "pull", reference], capture_output=False) - inspected = _image_inspect(reference) + _run( + ["docker", "pull", "--platform", docker_platform(), mirror_reference], + capture_output=False, + ) + inspected = _image_inspect(mirror_reference) if inspected is None: raise TributoITError( f"third-party image is unavailable after pull: {reference}" @@ -805,6 +950,21 @@ def ensure_digest_image(reference: str) -> str: return image_id +def _local_digest_reference(reference: str) -> str: + """Return the locally resolvable pinned reference after mirror preparation.""" + expected_digest = reference.rsplit("@", 1)[-1] + for candidate in (reference, _domestic_mirror_reference(reference)): + inspected = _image_inspect(candidate) + if inspected is None: + continue + repo_digests = [str(value) for value in inspected.get("RepoDigests") or []] + if any(value.rsplit("@", 1)[-1] == expected_digest for value in repo_digests): + return candidate + raise TributoITError( + f"no locally resolvable pinned reference after image preparation: {reference}" + ) + + def _excluded(relative: Path, *, is_directory: bool) -> bool: if any(part in EXCLUDED_DIRECTORY_NAMES for part in relative.parts): return True @@ -1048,6 +1208,9 @@ def _compose_environment( project: str, runtime: PreparedRuntime, profile: RuntimeProfile, + *, + tool_image: str, + minio_image: str, ) -> dict[str, str]: env = os.environ.copy() env.update( @@ -1055,8 +1218,8 @@ def _compose_environment( "COMPOSE_PROJECT_NAME": project, "TRIBUTO_IT_RUNTIME_IMAGE": runtime.identity.local_tag, "TRIBUTO_IT_SOURCE_ROOT": str(ROOT), - "TRIBUTO_IT_TOOL_IMAGE": profile.tool_image, - "TRIBUTO_IT_MINIO_IMAGE": profile.minio_image, + "TRIBUTO_IT_TOOL_IMAGE": tool_image, + "TRIBUTO_IT_MINIO_IMAGE": minio_image, } ) return env @@ -1085,6 +1248,9 @@ def validate_compose_contract( config: dict[str, Any], runtime: PreparedRuntime, profile: RuntimeProfile, + *, + tool_image: str | None = None, + minio_image: str | None = None, ) -> None: """Validate the resolved config rather than trusting YAML inheritance.""" services = config.get("services") @@ -1116,16 +1282,18 @@ def validate_compose_contract( source_init = services["source-init"] workspace_init = services["workspace-init"] - if source_init.get("image") != profile.tool_image: + expected_tool_image = tool_image or profile.tool_image + expected_minio_image = minio_image or profile.minio_image + if source_init.get("image") != expected_tool_image: raise TributoITError("source-init must use the pinned tool image") - if workspace_init.get("image") != profile.tool_image: + if workspace_init.get("image") != expected_tool_image: raise TributoITError("workspace-init must use the pinned tool image") if source_init.get("restart") != "no": raise TributoITError('source-init must set restart: "no"') source_input = _volume_mount(source_init, "/host-source") if source_input is None or not source_input.get("read_only"): raise TributoITError("source-init checkout input must be read-only") - if services["minio"].get("image") != profile.minio_image: + if services["minio"].get("image") != expected_minio_image: raise TributoITError("MinIO must use its pinned readable tag@digest") runtime_users = { name @@ -1455,7 +1623,7 @@ def _run_streamed(command: Sequence[str], env: dict[str, str], log_path: Path) - process = subprocess.Popen( list(command), cwd=ROOT, - env=env, + env=_docker_environment(env) if command and command[0] == "docker" else env, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -1560,9 +1728,23 @@ def _interrupt(signum: int, _frame: object) -> None: ) ensure_digest_image(profile.tool_image) ensure_digest_image(profile.minio_image) - env = _compose_environment(project, prepared, profile) + tool_image = _local_digest_reference(profile.tool_image) + minio_image = _local_digest_reference(profile.minio_image) + env = _compose_environment( + project, + prepared, + profile, + tool_image=tool_image, + minio_image=minio_image, + ) config = resolved_compose_config(env) - validate_compose_contract(config, prepared, profile) + validate_compose_contract( + config, + prepared, + profile, + tool_image=tool_image, + minio_image=minio_image, + ) _run( _compose_args( "up", diff --git a/uv.lock b/uv.lock index ae5592e..38116e9 100644 --- a/uv.lock +++ b/uv.lock @@ -308,6 +308,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ef/c3/f30a7a63e664acc7c2545ca0491b6ce8264536e0e5cad3965f1d1b91e960/aws_xray_sdk-2.15.0-py2.py3-none-any.whl", hash = "sha256:422d62ad7d52e373eebb90b642eb1bb24657afe03b22a8df4a8b2e5108e278a3", size = 103228, upload-time = "2025-10-29T21:00:24.12Z" }, ] +[[package]] +name = "bayesian-optimization" +version = "1.4.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'x86_64' and sys_platform == 'darwin'" }, + { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine != 'x86_64' or sys_platform != 'darwin'" }, + { name = "scikit-learn" }, + { name = "scipy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9a/37/e34b70e97b83393d98bb53cdfcf0213af29ac5932d6752bbf2abbda5b998/bayesian-optimization-1.4.3.tar.gz", hash = "sha256:f9a448e1b52d961301cbc953ce4199709f6c26b1c88994c9b4dadb7752a64550", size = 27803, upload-time = "2023-04-25T21:58:35.41Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/cf/3016b660afca02c6ecca3c1cc6d8df3b8f1a6ff4878103204d0aa6b4c769/bayesian_optimization-1.4.3-py3-none-any.whl", hash = "sha256:2719272d5825f1ba7d7609f3b1c1fdca13eba1b7ad52a5ca2e62f34154ecae28", size = 18346, upload-time = "2023-04-25T21:58:33.498Z" }, +] + [[package]] name = "blinker" version = "1.9.0" @@ -846,6 +862,47 @@ ray = [ { name = "ray", extra = ["client", "data"] }, ] +[[package]] +name = "daft-clickhouse" +version = "1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "daft" }, + { name = "pyarrow" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b6/0b/4857e4746fdbe8f1cd911854e4e4821d1650b7f2c55efb558fcbad2707aa/daft_clickhouse-1.0.tar.gz", hash = "sha256:053ee0b740eb68246b48c3cf7bcce8eb8e9766e6c9530968a80f3b0fa08ce77a", size = 169506, upload-time = "2026-08-17T14:44:58.758Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/b2/c32e73be7521cbd0e6eb5fbbe00114a5dcb637265a7db1e5f7a94bf6af15/daft_clickhouse-1.0-py3-none-any.whl", hash = "sha256:6b3ce6d2dc30fb0c282cfa99f14faa41887e139f31424f16ebc8b4d090ee1c7f", size = 51936, upload-time = "2026-08-17T14:44:57.553Z" }, +] + +[package.optional-dependencies] +clickhouse = [ + { name = "clickhouse-connect", extra = ["arrow", "async"] }, +] + +[[package]] +name = "daft-doris" +version = "1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "daft" }, + { name = "pyarrow" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3e/cd/b8f2453d39f5af496c66c16d23baa59be82eb1b815726696db7f39495086/daft_doris-1.0.tar.gz", hash = "sha256:43450c9088c5f25ac25934ac896b4c4c8a1c7df15e6680559b299d668fd3cf4a", size = 88745, upload-time = "2026-08-17T15:05:54.317Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bd/2b/78c76b5fd313051841b4cc800f7495e01a4c123963f9eeb2c2d0dc57e474/daft_doris-1.0-py3-none-any.whl", hash = "sha256:72a19e2f300a955ac7a12a263ae9a0a332019126c9efa7b04800575e96b51593", size = 66653, upload-time = "2026-08-17T15:05:52.826Z" }, +] + +[package.optional-dependencies] +doris = [ + { name = "pymysql" }, +] +doris-flight = [ + { name = "adbc-driver-flightsql" }, + { name = "adbc-driver-manager" }, + { name = "pymysql" }, +] + [[package]] name = "daft-lance" version = "0.4.0" @@ -3758,6 +3815,24 @@ tune = [ { name = "tensorboardx" }, ] +[[package]] +name = "ray-doris" +version = "1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pymysql" }, + { name = "ray", extra = ["data"] }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/9e/e254d027970f261075d2687181381bebd379d7f12cbd34d90549a4dda402/ray_doris-1.0.tar.gz", hash = "sha256:898bed3d511089a1166f0f793fde7e4c12450fe303ce2732bb722312f22de3bd", size = 127207, upload-time = "2026-08-17T14:44:45.409Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/59/ca/bb18462b40f9a42f8c2c57b65062c346225c7695ba45c1838673c6089f7b/ray_doris-1.0-py3-none-any.whl", hash = "sha256:055c8e08fdd7b2903d5e88ccd0e78660e3f74095ce6b88ae46b38bac3330df5b", size = 51323, upload-time = "2026-08-17T14:44:44.319Z" }, +] + +[package.optional-dependencies] +flight = [ + { name = "adbc-driver-flightsql" }, +] + [[package]] name = "redis" version = "8.1.0" @@ -4484,6 +4559,7 @@ causal = [ ] clickhouse = [ { name = "clickhouse-connect", extra = ["arrow", "async"] }, + { name = "daft-clickhouse", extra = ["clickhouse"] }, ] data = [ { name = "lance-ray" }, @@ -4508,7 +4584,9 @@ dev = [ doris-flight = [ { name = "adbc-driver-flightsql" }, { name = "adbc-driver-manager" }, + { name = "daft-doris", extra = ["doris-flight"] }, { name = "pymysql" }, + { name = "ray-doris", extra = ["flight"] }, ] explainability = [ { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'x86_64' and sys_platform == 'darwin'" }, @@ -4577,7 +4655,9 @@ model-export-torch = [ { name = "torch" }, ] mysql = [ + { name = "daft-doris", extra = ["doris"] }, { name = "pymysql" }, + { name = "ray-doris" }, ] postgresql = [ { name = "psycopg", extra = ["binary"] }, @@ -4604,6 +4684,8 @@ test-integration = [ { name = "boto3" }, { name = "clickhouse-connect", extra = ["arrow", "async"] }, { name = "daft", extra = ["lance", "ray"] }, + { name = "daft-clickhouse", extra = ["clickhouse"] }, + { name = "daft-doris", extra = ["doris"] }, { name = "lance-ray" }, { name = "ml-dtypes" }, { name = "mlflow" }, @@ -4616,6 +4698,7 @@ test-integration = [ { name = "pylance" }, { name = "pymysql" }, { name = "pytest" }, + { name = "ray-doris" }, { name = "requests" }, { name = "skl2onnx" }, { name = "sqlalchemy" }, @@ -4626,6 +4709,8 @@ training = [ { name = "boto3" }, { name = "clickhouse-connect", extra = ["arrow", "async"] }, { name = "daft", extra = ["lance", "ray"] }, + { name = "daft-clickhouse", extra = ["clickhouse"] }, + { name = "daft-doris", extra = ["doris"] }, { name = "lance-ray" }, { name = "ml-dtypes" }, { name = "onnx", version = "1.19.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, @@ -4636,11 +4721,15 @@ training = [ { name = "pyiceberg" }, { name = "pylance" }, { name = "pymysql" }, + { name = "ray-doris" }, { name = "skl2onnx" }, { name = "sqlalchemy" }, { name = "sqlglot" }, { name = "xgboost" }, ] +tune = [ + { name = "bayesian-optimization" }, +] vector-index = [ { name = "boto3" }, { name = "lance-ray" }, @@ -4673,11 +4762,15 @@ requires-dist = [ { name = "adbc-driver-flightsql", marker = "extra == 'doris-flight'", specifier = ">=1.6,<2" }, { name = "adbc-driver-manager", marker = "extra == 'doris-flight'", specifier = ">=1.6,<2" }, { name = "aiohttp", specifier = ">=3.13.4" }, + { name = "bayesian-optimization", marker = "extra == 'tune'", specifier = "==1.4.3" }, { name = "boto3", marker = "extra == 's3'", specifier = ">=1.42.91" }, { name = "click", specifier = ">=8.0.0" }, { name = "clickhouse-connect", extras = ["arrow", "async"], marker = "extra == 'clickhouse'", specifier = ">=1.5,<1.6" }, { name = "confluent-kafka", marker = "extra == 'streaming-inference'", specifier = ">=2.3.0" }, { name = "daft", extras = ["lance", "ray"], marker = "extra == 'data-daft'", specifier = ">=0.7.23,<0.7.24" }, + { name = "daft-clickhouse", extras = ["clickhouse"], marker = "extra == 'clickhouse'", specifier = "==1.0" }, + { name = "daft-doris", extras = ["doris"], marker = "extra == 'mysql'", specifier = "==1.0" }, + { name = "daft-doris", extras = ["doris-flight"], marker = "extra == 'doris-flight'", specifier = "==1.0" }, { name = "dowhy", marker = "extra == 'causal'", specifier = ">=0.11.0" }, { name = "econml", marker = "extra == 'causal'", specifier = ">=0.15.0" }, { name = "grpcio", marker = "extra == 'grpc'", specifier = ">=1.60.0" }, @@ -4712,6 +4805,8 @@ requires-dist = [ { name = "pytest-timeout", marker = "extra == 'dev'", specifier = ">=2.1.0" }, { name = "ray", extras = ["default", "serve", "tune"], specifier = "==2.55.1" }, { name = "ray", extras = ["serve"], marker = "extra == 'streaming-inference'" }, + { name = "ray-doris", marker = "extra == 'mysql'", specifier = "==1.0" }, + { name = "ray-doris", extras = ["flight"], marker = "extra == 'doris-flight'", specifier = "==1.0" }, { name = "requests", specifier = ">=2.33.0" }, { name = "requests", marker = "extra == 'test-integration'", specifier = ">=2.25.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.6.0,<1.0" }, @@ -4739,7 +4834,7 @@ requires-dist = [ { name = "types-pyyaml", marker = "extra == 'dev'", specifier = ">=6.0.12.20250516" }, { name = "xgboost", marker = "extra == 'training'", specifier = ">=2.1.0" }, ] -provides-extras = ["dev", "data", "data-daft", "vector-index", "postgresql", "clickhouse", "mysql", "doris-flight", "s3", "model-export", "model-export-torch", "hf", "model-export-hf", "training", "explainability", "identity", "streaming", "grpc", "registry", "graph", "causal", "streaming-inference", "test-integration"] +provides-extras = ["dev", "data", "data-daft", "vector-index", "postgresql", "clickhouse", "mysql", "doris-flight", "s3", "model-export", "model-export-torch", "hf", "model-export-hf", "training", "tune", "explainability", "identity", "streaming", "grpc", "registry", "graph", "causal", "streaming-inference", "test-integration"] [package.metadata.requires-dev] dev = [