diff --git a/Dockerfile b/Dockerfile index 3321c0b..20232cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -91,6 +91,8 @@ ENV SNOWFLAKE_ADBC_DRIVER_PATH=/usr/local/lib/libadbc_driver_snowflake.so # persistent-file-log feature in the deployed image; a local/dev invocation # of this same binary outside the image still gets it by default. ENV DATAMK_LOG=off +# The cell mounts read-only; eject artifacts go to the scratch emptyDir. +ENV DATAMK_MATERIALIZE_DIR=/tmp/materialize # Bake every DuckDB extension the engine may INSTALL at first use (~200 MB: # ducklake, httpfs, json, postgres, sqlite, community bigquery) into the # image, so a pod needs no registry egress and no exec-capable scratch diff --git a/docs/guides/incremental.md b/docs/guides/incremental.md index c139670..61c8b95 100644 --- a/docs/guides/incremental.md +++ b/docs/guides/incremental.md @@ -85,6 +85,8 @@ Other rules: inherits nothing. - The composed statements are written to `.cell/materialize/.sql` every run (audit and portability; overwritten; not a valid `transforms:` target). + `DATAMK_MATERIALIZE_DIR` redirects them; the container image sets it to + `/tmp/materialize` because the cell mounts read-only there. - One-off backfills and manual corrections: `datamk attach -f cell.yaml -p prod` and run SQL against the lake directly. diff --git a/src/engine/mod.rs b/src/engine/mod.rs index 14c95d6..e643a72 100644 --- a/src/engine/mod.rs +++ b/src/engine/mod.rs @@ -1250,13 +1250,26 @@ fn execute_materialize( /// only for `replace` (schema-validated — `append`/`upsert` always carry a /// non-empty `key:`), which never ran any guard to lose, so its header says /// that plainly instead of naming guards that don't apply to it. +/// Where the eject artifacts land: `DATAMK_MATERIALIZE_DIR` if set, else +/// `.cell/materialize` under the cell directory. The override exists for +/// the same reason `DATAMK_LOG=off` does (`logging.rs`): a deployed Builder +/// mounts the cell read-only from a ConfigMap, so the default path is +/// unwritable there and the image points this at the scratch emptyDir. +/// Env-only, deliberately not a CLI flag — a laptop run never needs it. +fn materialize_dir(dir: &Path) -> PathBuf { + match std::env::var_os("DATAMK_MATERIALIZE_DIR") { + Some(d) if !d.is_empty() => PathBuf::from(d), + _ => dir.join(".cell").join("materialize"), + } +} + fn write_eject_artifact( dir: &Path, table: &str, key: &[String], statements: &[String], ) -> Result { - let materialize_dir = dir.join(".cell").join("materialize"); + let materialize_dir = materialize_dir(dir); std::fs::create_dir_all(&materialize_dir) .with_context(|| format!("creating {}", materialize_dir.display()))?; let path = materialize_dir.join(format!("{table}.sql")); @@ -6158,6 +6171,20 @@ mod tests { assert!(artifact.exists()); } + #[test] + fn materialize_dir_honours_the_env_override_and_ignores_an_empty_one() { + // Nothing else in this binary reads `DATAMK_MATERIALIZE_DIR`, so + // mutating it here is safe (same discipline as the DATAMK_LOG test). + let dir = Path::new("/some/cell"); + std::env::remove_var("DATAMK_MATERIALIZE_DIR"); + assert_eq!(materialize_dir(dir), dir.join(".cell").join("materialize")); + std::env::set_var("DATAMK_MATERIALIZE_DIR", ""); + assert_eq!(materialize_dir(dir), dir.join(".cell").join("materialize")); + std::env::set_var("DATAMK_MATERIALIZE_DIR", "/tmp/materialize"); + assert_eq!(materialize_dir(dir), PathBuf::from("/tmp/materialize")); + std::env::remove_var("DATAMK_MATERIALIZE_DIR"); + } + // --- ADR 0008 §3 (founder-ratified): `materialize: replace` ------------ #[test] diff --git a/test/integrations/kind_e2e/manifests/minio.yaml b/test/integrations/kind_e2e/manifests/minio.yaml index 7e8e2c2..b82bfb9 100644 --- a/test/integrations/kind_e2e/manifests/minio.yaml +++ b/test/integrations/kind_e2e/manifests/minio.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: minio - image: minio/minio:latest + image: quay.io/minio/minio:RELEASE.2025-07-23T15-54-02Z args: ["server", "/data", "--console-address", ":9001"] env: - name: MINIO_ROOT_USER diff --git a/test/integrations/kind_e2e/run.sh b/test/integrations/kind_e2e/run.sh index 4dfeaf8..53755ed 100755 --- a/test/integrations/kind_e2e/run.sh +++ b/test/integrations/kind_e2e/run.sh @@ -168,7 +168,7 @@ spec: restartPolicy: Never containers: - name: mc - image: minio/mc:latest + image: quay.io/minio/mc:RELEASE.2025-07-21T05-28-08Z command: - /bin/sh - -c @@ -399,7 +399,9 @@ phase_validate() { log "validate: GET /openapi.json, discovering the export route" local openapi route openapi="$(curl -fsS "$base/openapi.json")" || die "GET /openapi.json failed" - route="$(echo "$openapi" | jq -r '.paths | keys[0]')" + # Export routes carry '@'; the spec also documents /, /context, + # /context/{route}, and /openapi.json, which sort ahead of them. + route="$(echo "$openapi" | jq -r '[.paths | keys[] | select(contains("@"))][0]')" [ -n "$route" ] && [ "$route" != "null" ] || die "openapi.json has no routes" log "discovered route: $route"