updated 1 parameter in dockerfile

.devcontainer/Dockerfile

@@ -64,7 +64,7 @@ RUN curl -fsSL https://github.com/mikefarah/yq/releases/download/v4.44.1/yq_linu
 
 ARG SPARK_VERSION=4.1.1
 ARG HADOOP_FLAVOR=hadoop3
-RUN wget -q https://downloads.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-${HADOOP_FLAVOR}.tgz \
+RUN wget -q https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-${HADOOP_FLAVOR}.tgz \
     && tar -xzf spark-${SPARK_VERSION}-bin-${HADOOP_FLAVOR}.tgz \
     && mv spark-${SPARK_VERSION}-bin-${HADOOP_FLAVOR} /opt/spark \
     && rm spark-${SPARK_VERSION}-bin-${HADOOP_FLAVOR}.tgz \

.gitignore

@@ -11,3 +11,4 @@ derby.log
 spark-warehouse/
 metastore_db/
 .env
+.vscode

CLAUDE.md

@@ -28,7 +28,11 @@ Do not propose alternatives to these.
 
 **Pipeline library type is `file:`, not `notebook:`** — `.py` source files in pipeline resources use `libraries: - file: path: ...`. `notebook:` expects an `.ipynb` or Databricks notebook format and will reject plain Python files.
 
-**`environment.dependencies: - --editable ${workspace.file_path}`** on every pipeline resource — makes the `data_product` package importable in the DLT runtime. DABs uploads the source tree to `${workspace.file_path}` on deploy; the editable install reads the package from there. No wheel build required. Requires `[build-system]` (hatchling) in `pyproject.toml` and `[tool.hatch.build.targets.wheel] packages = ["src/data_product"]`.
+**`data_product` reaches the DLT runtime as a built wheel, not an editable install.** Each pipeline resource declares `environment.dependencies: - ../dist/*.whl` (relative path resolves to the bundle-root `dist/`). A top-level `artifacts.data_product_wheel` block (`type: whl`, `build: uv build --wheel`, `path: .`) builds the wheel on `bundle deploy`; DABs uploads it and rewrites the dependency to the workspace artifact path. Requires `[build-system]` (hatchling) in `pyproject.toml` and `[tool.hatch.build.targets.wheel] packages = ["src/data_product"]` (maps `src/data_product` → import name `data_product`).
+
+`[project].dependencies` must stay **empty** so the wheel carries no `Requires-Dist`: serverless DLT provides pyspark, and the notebook tasks get pyspark/databricks-sdk from their job compute. pyspark/databricks-sdk/databricks-bundles live in `[dependency-groups].dev` for local dev, tests, and the deploy-time mutators only.
+
+History: the original pattern was `--editable ${workspace.file_path}`. It silently stopped making `data_product` importable on serverless DLT (config correct, source uploaded, but the runtime editable build/`.pth` was not effective). Switched to the wheel artifact 2026-06-15 — a plain wheel lands `data_product` directly in site-packages, no `/Workspace` mount or runtime editable build, and no third-party dependency closure resolved at pipeline runtime.
 
 **`@dp.table` stubs use a rate stream filtered to zero rows** — `createDataFrame()` returns a batch relation which DLT rejects for streaming tables. Pattern:
 ```python

databricks.yml

@@ -1,6 +1,11 @@
 bundle:
   name: databricks-dataops-template
   databricks_cli_version: ">= 0.240.0"
+  # Pin the deploy engine. CLI v1.3.0 made `direct` the default; this bundle was
+  # only ever validated on `terraform`. Declared explicitly so deploy behaviour
+  # is deterministic across CLI versions rather than riding a floating default.
+  # Migrating to the direct engine is a tracked follow-up (needs a live test).
+  engine: terraform
 
 variables:
   catalog:
@@ -34,6 +39,12 @@ presets:
     target: ${bundle.target}
     git_branch: ${bundle.git.branch}
 
+artifacts:
+  data_product_wheel:
+    type: whl
+    build: uv build --wheel
+    path: .
+
 include:
   - resources/*.yml
 

pyproject.toml

@@ -2,14 +2,19 @@
 name = "databricks-dataops-template"
 version = "0.1.0"
 requires-python = ">=3.11"
-dependencies = [
-    "databricks-sdk>=0.117.0",
-    "pyspark[pipelines]>=4.1.2",
-    "databricks-bundles>=1.3.0",
-]
+# No runtime dependencies: the data_product wheel is installed into the
+# serverless DLT pipeline environment, which already provides pyspark. The
+# notebook tasks (validate_*, finalize_ops_status) get pyspark/databricks-sdk
+# from their job compute. databricks-bundles is a deploy-time tool only.
+# Keeping these out of [project] means the wheel carries no Requires-Dist, so
+# the pipeline env installs nothing beyond data_product itself.
+dependencies = []
 
 [dependency-groups]
 dev = [
+    "databricks-sdk>=0.109.0",
+    "databricks-bundles>=0.299.2",
+    "pyspark[pipelines]>=4.1.1",
     "pandas>=3.0.3",
     "pytest>=9.0.3",
     "ruff>=0.15.15",

resources/customers_pipeline.yml

@@ -18,4 +18,4 @@ resources:
 
       environment:
         dependencies:
-          - --editable ${workspace.file_path}
+          - ../dist/*.whl

resources/data_product_operational_job.yml

@@ -19,14 +19,12 @@ resources:
 
       tasks:
         - task_key: run_customers
-          max_retries: 1
-          min_retry_interval_millis: 60000
+          max_retries: 0
           pipeline_task:
             pipeline_id: ${resources.pipelines.customers_pipeline.id}
 
         - task_key: run_orders
-          max_retries: 1
-          min_retry_interval_millis: 60000
+          max_retries: 0
           pipeline_task:
             pipeline_id: ${resources.pipelines.orders_pipeline.id}
 
@@ -47,8 +45,7 @@ resources:
         - task_key: run_gold
           depends_on:
             - task_key: validate_silver_readiness
-          max_retries: 1
-          min_retry_interval_millis: 60000
+          max_retries: 0
           pipeline_task:
             pipeline_id: ${resources.pipelines.gold_pipeline.id}
 

resources/gold_pipeline.yml

@@ -17,4 +17,4 @@ resources:
 
       environment:
         dependencies:
-          - --editable ${workspace.file_path}
+          - ../dist/*.whl

resources/orders_pipeline.yml

@@ -18,4 +18,4 @@ resources:
 
       environment:
         dependencies:
-          - --editable ${workspace.file_path}
+          - ../dist/*.whl