From dab746a2f09ff8485a868277fa62860e0c73095c Mon Sep 17 00:00:00 2001 From: Sigurd Lislegaard Date: Fri, 12 Jun 2026 11:34:53 +0200 Subject: [PATCH 1/3] updated 1 parameter in dockerfile --- .devcontainer/Dockerfile | 2 +- .gitignore | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 4aa0ee1..3e502e8 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -64,7 +64,7 @@ RUN curl -fsSL https://github.com/mikefarah/yq/releases/download/v4.44.1/yq_linu ARG SPARK_VERSION=4.1.1 ARG HADOOP_FLAVOR=hadoop3 -RUN wget -q https://downloads.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-${HADOOP_FLAVOR}.tgz \ +RUN wget -q https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-${HADOOP_FLAVOR}.tgz \ && tar -xzf spark-${SPARK_VERSION}-bin-${HADOOP_FLAVOR}.tgz \ && mv spark-${SPARK_VERSION}-bin-${HADOOP_FLAVOR} /opt/spark \ && rm spark-${SPARK_VERSION}-bin-${HADOOP_FLAVOR}.tgz \ diff --git a/.gitignore b/.gitignore index 8a36f15..3395832 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ derby.log spark-warehouse/ metastore_db/ .env +.vscode \ No newline at end of file From 6577f76a471617cdac70117029b6c8c7ea71440e Mon Sep 17 00:00:00 2001 From: Sigurd Lislegaard Date: Mon, 15 Jun 2026 06:19:25 +0000 Subject: [PATCH 2/3] added cli-version to databricks.yml --- databricks.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/databricks.yml b/databricks.yml index 67a6868..b733674 100644 --- a/databricks.yml +++ b/databricks.yml @@ -1,6 +1,12 @@ bundle: name: databricks-dataops-template databricks_cli_version: ">= 0.240.0" + # Pin the deploy engine. CLI v1.3.0 flipped the default from terraform to + # `direct`, which breaks the --editable ${workspace.file_path} source upload + # the pipelines import data_product from. Declared explicitly so deploy + # behaviour is deterministic across CLI versions. Direct-engine migration + # is a tracked follow-up. + engine: terraform variables: catalog: From 58e8c4c76c852391d549863b979cd9746ae807e4 Mon Sep 17 00:00:00 2001 From: Sigurd Lislegaard Date: Mon, 15 Jun 2026 07:28:49 +0000 Subject: [PATCH 3/3] The serverless DLT runtime stopped honouring the dependency: config was correct and the source tree was uploaded, but was not importable at pipeline runtime (PYTHON.MODULE_NOT_FOUND_ERROR). All three pipelines failed on import, and prod-mode DLT auto-retried the failed updates, producing a retry loop. --- CLAUDE.md | 6 +++++- databricks.yml | 15 ++++++++++----- pyproject.toml | 15 ++++++++++----- resources/customers_pipeline.yml | 2 +- resources/data_product_operational_job.yml | 9 +++------ resources/gold_pipeline.yml | 2 +- resources/orders_pipeline.yml | 2 +- uv.lock | 16 ++++++---------- 8 files changed, 37 insertions(+), 30 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 2db47c4..a8fc741 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -28,7 +28,11 @@ Do not propose alternatives to these. **Pipeline library type is `file:`, not `notebook:`** — `.py` source files in pipeline resources use `libraries: - file: path: ...`. `notebook:` expects an `.ipynb` or Databricks notebook format and will reject plain Python files. -**`environment.dependencies: - --editable ${workspace.file_path}`** on every pipeline resource — makes the `data_product` package importable in the DLT runtime. DABs uploads the source tree to `${workspace.file_path}` on deploy; the editable install reads the package from there. No wheel build required. Requires `[build-system]` (hatchling) in `pyproject.toml` and `[tool.hatch.build.targets.wheel] packages = ["src/data_product"]`. +**`data_product` reaches the DLT runtime as a built wheel, not an editable install.** Each pipeline resource declares `environment.dependencies: - ../dist/*.whl` (relative path resolves to the bundle-root `dist/`). A top-level `artifacts.data_product_wheel` block (`type: whl`, `build: uv build --wheel`, `path: .`) builds the wheel on `bundle deploy`; DABs uploads it and rewrites the dependency to the workspace artifact path. Requires `[build-system]` (hatchling) in `pyproject.toml` and `[tool.hatch.build.targets.wheel] packages = ["src/data_product"]` (maps `src/data_product` → import name `data_product`). + +`[project].dependencies` must stay **empty** so the wheel carries no `Requires-Dist`: serverless DLT provides pyspark, and the notebook tasks get pyspark/databricks-sdk from their job compute. pyspark/databricks-sdk/databricks-bundles live in `[dependency-groups].dev` for local dev, tests, and the deploy-time mutators only. + +History: the original pattern was `--editable ${workspace.file_path}`. It silently stopped making `data_product` importable on serverless DLT (config correct, source uploaded, but the runtime editable build/`.pth` was not effective). Switched to the wheel artifact 2026-06-15 — a plain wheel lands `data_product` directly in site-packages, no `/Workspace` mount or runtime editable build, and no third-party dependency closure resolved at pipeline runtime. **`@dp.table` stubs use a rate stream filtered to zero rows** — `createDataFrame()` returns a batch relation which DLT rejects for streaming tables. Pattern: ```python diff --git a/databricks.yml b/databricks.yml index b733674..92276a3 100644 --- a/databricks.yml +++ b/databricks.yml @@ -1,11 +1,10 @@ bundle: name: databricks-dataops-template databricks_cli_version: ">= 0.240.0" - # Pin the deploy engine. CLI v1.3.0 flipped the default from terraform to - # `direct`, which breaks the --editable ${workspace.file_path} source upload - # the pipelines import data_product from. Declared explicitly so deploy - # behaviour is deterministic across CLI versions. Direct-engine migration - # is a tracked follow-up. + # Pin the deploy engine. CLI v1.3.0 made `direct` the default; this bundle was + # only ever validated on `terraform`. Declared explicitly so deploy behaviour + # is deterministic across CLI versions rather than riding a floating default. + # Migrating to the direct engine is a tracked follow-up (needs a live test). engine: terraform variables: @@ -40,6 +39,12 @@ presets: target: ${bundle.target} git_branch: ${bundle.git.branch} +artifacts: + data_product_wheel: + type: whl + build: uv build --wheel + path: . + include: - resources/*.yml diff --git a/pyproject.toml b/pyproject.toml index f21c527..0ed53b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,14 +2,19 @@ name = "databricks-dataops-template" version = "0.1.0" requires-python = ">=3.11" -dependencies = [ - "databricks-sdk>=0.109.0", - "pyspark[pipelines]>=4.1.1", - "databricks-bundles>=0.299.2", -] +# No runtime dependencies: the data_product wheel is installed into the +# serverless DLT pipeline environment, which already provides pyspark. The +# notebook tasks (validate_*, finalize_ops_status) get pyspark/databricks-sdk +# from their job compute. databricks-bundles is a deploy-time tool only. +# Keeping these out of [project] means the wheel carries no Requires-Dist, so +# the pipeline env installs nothing beyond data_product itself. +dependencies = [] [dependency-groups] dev = [ + "databricks-sdk>=0.109.0", + "databricks-bundles>=0.299.2", + "pyspark[pipelines]>=4.1.1", "pandas>=3.0.3", "pytest>=9.0.3", "ruff>=0.15.13", diff --git a/resources/customers_pipeline.yml b/resources/customers_pipeline.yml index a46aa0c..6cbef20 100644 --- a/resources/customers_pipeline.yml +++ b/resources/customers_pipeline.yml @@ -18,4 +18,4 @@ resources: environment: dependencies: - - --editable ${workspace.file_path} + - ../dist/*.whl diff --git a/resources/data_product_operational_job.yml b/resources/data_product_operational_job.yml index 08119ed..8071ee7 100644 --- a/resources/data_product_operational_job.yml +++ b/resources/data_product_operational_job.yml @@ -19,14 +19,12 @@ resources: tasks: - task_key: run_customers - max_retries: 1 - min_retry_interval_millis: 60000 + max_retries: 0 pipeline_task: pipeline_id: ${resources.pipelines.customers_pipeline.id} - task_key: run_orders - max_retries: 1 - min_retry_interval_millis: 60000 + max_retries: 0 pipeline_task: pipeline_id: ${resources.pipelines.orders_pipeline.id} @@ -47,8 +45,7 @@ resources: - task_key: run_gold depends_on: - task_key: validate_silver_readiness - max_retries: 1 - min_retry_interval_millis: 60000 + max_retries: 0 pipeline_task: pipeline_id: ${resources.pipelines.gold_pipeline.id} diff --git a/resources/gold_pipeline.yml b/resources/gold_pipeline.yml index 3fdf7ff..ef80eee 100644 --- a/resources/gold_pipeline.yml +++ b/resources/gold_pipeline.yml @@ -17,4 +17,4 @@ resources: environment: dependencies: - - --editable ${workspace.file_path} + - ../dist/*.whl diff --git a/resources/orders_pipeline.yml b/resources/orders_pipeline.yml index 3c7b835..a9ee96d 100644 --- a/resources/orders_pipeline.yml +++ b/resources/orders_pipeline.yml @@ -18,4 +18,4 @@ resources: environment: dependencies: - - --editable ${workspace.file_path} + - ../dist/*.whl diff --git a/uv.lock b/uv.lock index 748b030..8eea1d3 100644 --- a/uv.lock +++ b/uv.lock @@ -259,29 +259,25 @@ wheels = [ name = "databricks-dataops-template" version = "0.1.0" source = { editable = "." } -dependencies = [ - { name = "databricks-bundles" }, - { name = "databricks-sdk" }, - { name = "pyspark", extra = ["pipelines"] }, -] [package.dev-dependencies] dev = [ + { name = "databricks-bundles" }, + { name = "databricks-sdk" }, { name = "pandas" }, + { name = "pyspark", extra = ["pipelines"] }, { name = "pytest" }, { name = "ruff" }, ] [package.metadata] -requires-dist = [ - { name = "databricks-bundles", specifier = ">=0.299.2" }, - { name = "databricks-sdk", specifier = ">=0.109.0" }, - { name = "pyspark", extras = ["pipelines"], specifier = ">=4.1.1" }, -] [package.metadata.requires-dev] dev = [ + { name = "databricks-bundles", specifier = ">=0.299.2" }, + { name = "databricks-sdk", specifier = ">=0.109.0" }, { name = "pandas", specifier = ">=3.0.3" }, + { name = "pyspark", extras = ["pipelines"], specifier = ">=4.1.1" }, { name = "pytest", specifier = ">=9.0.3" }, { name = "ruff", specifier = ">=0.15.13" }, ]