From 1d10b90ca65921397a6642ceec7894272c0be632 Mon Sep 17 00:00:00 2001 From: Alex Porter <102990994+am-porter@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:59:08 -0800 Subject: [PATCH 1/6] Allow DiskGraph consumer state outside the base index --- .../diskgraph/diskgraph-index/src/graph.rs | 4 +- .../diskgraph/runtime_batch_iterator.py | 23 ++++-- .../python/tests/test_cutoff_col_stats.py | 10 +++ .../tests/test_runtime_batch_iterator_e2e.py | 76 +++++++++++++++++++ .../diskgraph-pybind/src/cutoff_col_stats.rs | 18 +++-- .../diskgraph/diskgraph-runtime/src/lib.rs | 22 +++++- 6 files changed, 135 insertions(+), 18 deletions(-) diff --git a/packages/diskgraph/diskgraph-index/src/graph.rs b/packages/diskgraph/diskgraph-index/src/graph.rs index dc5187982..c2380a01e 100644 --- a/packages/diskgraph/diskgraph-index/src/graph.rs +++ b/packages/diskgraph/diskgraph-index/src/graph.rs @@ -1227,8 +1227,8 @@ impl GraphIndexReader { } /// Generalization of [`open_with_tt`] to an extension at an arbitrary - /// relative `subdir` under the parent (e.g. `pred_tables/job1` for the - /// batch-prediction path, or `train_tables/` for training). The + /// relative `subdir` under the parent, or an absolute path outside it. + /// External paths allow consumers to keep a shared parent read-only. The /// extension's tables are merged under the `/` /// keys and its cross-boundary edges are decoded against the parent's /// node types — the same recipe `open_with_tt` uses. diff --git a/packages/diskgraph/diskgraph-pybind/python/diskgraph/runtime_batch_iterator.py b/packages/diskgraph/diskgraph-pybind/python/diskgraph/runtime_batch_iterator.py index 136e1f299..d5df98e40 100644 --- a/packages/diskgraph/diskgraph-pybind/python/diskgraph/runtime_batch_iterator.py +++ b/packages/diskgraph/diskgraph-pybind/python/diskgraph/runtime_batch_iterator.py @@ -19,6 +19,7 @@ import json import os +from pathlib import Path from typing import Any, Mapping, Optional from ._exceptions import DiskgraphConfigError @@ -113,10 +114,15 @@ def _extension_ref_from_stage_path(parent_root: str, stage_index_path: str) -> s """Recover the seed-table extension reference from a stage ``index_path``. The stage path itself is the source of truth: return a relative - ``train_tables/`` or ``pred_tables/`` ref under the parent. + ``train_tables/`` or ``pred_tables/`` ref under the parent, + or an absolute external extension path for a read-only base index. External + paths require finalized index metadata; the native reader validates it. """ abs_parent = os.path.abspath(parent_root.rstrip("/")) abs_stage = os.path.abspath(stage_index_path.rstrip("/")) + parent, stage = Path(abs_parent).resolve(), Path(abs_stage).resolve() + if stage == parent: + raise DiskgraphConfigError("stage index_path must identify an extension, not the base") for extension_dir in ("train_tables", "pred_tables"): expected_prefix = os.path.join(abs_parent, extension_dir) + os.sep if not abs_stage.startswith(expected_prefix): @@ -128,11 +134,15 @@ def _extension_ref_from_stage_path(parent_root: str, stage_index_path: str) -> s f"{stage_index_path}" ) return f"{extension_dir}/{extension_id}" - raise DiskgraphConfigError( - "stage index_path must live under /train_tables/ " - "or /pred_tables/: " - f"parent={abs_parent}, stage={abs_stage}" - ) + if stage.is_relative_to(parent): + raise DiskgraphConfigError( + "an index-local stage index_path must live under train_tables/ or pred_tables/" + ) + if not (stage / "index_metadata.json").is_file(): + raise DiskgraphConfigError( + f"external stage index_path has no finalized index_metadata.json: {abs_stage}" + ) + return abs_stage def _sampling_plan_json(sampling_plan: Any) -> str: @@ -459,4 +469,3 @@ def iter( ) return iterator, iterator.num_samples() - diff --git a/packages/diskgraph/diskgraph-pybind/python/tests/test_cutoff_col_stats.py b/packages/diskgraph/diskgraph-pybind/python/tests/test_cutoff_col_stats.py index 1d4a03542..5aedc8039 100644 --- a/packages/diskgraph/diskgraph-pybind/python/tests/test_cutoff_col_stats.py +++ b/packages/diskgraph/diskgraph-pybind/python/tests/test_cutoff_col_stats.py @@ -143,6 +143,16 @@ def test_recompute_from_the_index_matches_ingest(data, ingested, tmp_path): assert _stats(str(index)) == _stats(ingested) +def test_recompute_into_external_directory(data, ingested, tmp_path): + index = tmp_path / "index" + shutil.copytree(ingested, index, ignore=shutil.ignore_patterns("col_stats")) + output = tmp_path / "cache" + diskgraph.cutoff_col_stats(str(index), json.dumps(REQUEST), output_dir=str(output), + num_threads=2) + assert _stats(str(output)) == _stats(ingested) + assert not (index / "col_stats").exists() + + def test_rejects_unknown_columns_and_exclusive_options(data, ingested, tmp_path): with pytest.raises(Exception, match="has no column missing"): diskgraph.cutoff_col_stats(ingested, json.dumps( diff --git a/packages/diskgraph/diskgraph-pybind/python/tests/test_runtime_batch_iterator_e2e.py b/packages/diskgraph/diskgraph-pybind/python/tests/test_runtime_batch_iterator_e2e.py index c0afc3bdd..bf703164c 100644 --- a/packages/diskgraph/diskgraph-pybind/python/tests/test_runtime_batch_iterator_e2e.py +++ b/packages/diskgraph/diskgraph-pybind/python/tests/test_runtime_batch_iterator_e2e.py @@ -25,6 +25,7 @@ import inspect import json import pathlib +import shutil import struct from typing import Any @@ -328,6 +329,81 @@ def test_iterator_yields_dict_matching_contract(seed_table_fixture): ) +def test_external_extension_runs_without_an_index_local_copy(seed_table_fixture, tmp_path): + parent = tmp_path / "parent" + shutil.copytree(seed_table_fixture["parent_root"], parent) + external = tmp_path / "consumer" / seed_table_fixture["tt_id"] + external.parent.mkdir() + shutil.move(str(parent / "train_tables" / seed_table_fixture["tt_id"]), external) + before = {str(p.relative_to(parent)): p.read_bytes() + for p in parent.rglob("*") if p.is_file()} + manifest = json.loads(pathlib.Path(seed_table_fixture["manifest_path"]).read_text()) + manifest["graph"]["index_path"] = str(parent) + manifest["stages"]["train"]["index_path"] = str(external) + path = tmp_path / "manifest.json" + path.write_text(json.dumps(manifest)) + iterator, count = make_runtime_batch_iterator( + str(path), stage="train", entity_node_type="users", sampling_plan=_sampling_plan(), + num_workers=1, worker_id=0, shuffle=False, rnd_seed=17, epoch=0, + batch_size=3, drop_last=False, + ) + assert count == seed_table_fixture["n_seeds"] + batches = list(iterator) + assert len(batches) == 2 + for batch in batches: + _assert_runtime_batch_dict_contract(batch, 3, seed_table_fixture["n_users"]) + assert {str(p.relative_to(parent)): p.read_bytes() + for p in parent.rglob("*") if p.is_file()} == before + + +@pytest.mark.parametrize("directory", ["train_tables", "pred_tables"]) +def test_extension_reference_preserves_legacy_layout(tmp_path, directory): + from diskgraph.runtime_batch_iterator import _extension_ref_from_stage_path + + parent = tmp_path / "base" + assert _extension_ref_from_stage_path( + str(parent), str(parent / directory / "task") + ) == f"{directory}/task" + + +@pytest.mark.parametrize("kind", ["missing", "empty", "base", "base_alias", "index_local"]) +def test_extension_reference_refuses_non_extension_paths(tmp_path, kind): + from diskgraph.runtime_batch_iterator import _extension_ref_from_stage_path + + parent = tmp_path / "base" + parent.mkdir() + (parent / "index_metadata.json").write_text("{}") + stage = tmp_path / "stage" + if kind == "empty": + stage.mkdir() + elif kind == "base": + stage = parent + elif kind == "base_alias": + stage.symlink_to(parent, target_is_directory=True) + elif kind == "index_local": + stage = parent / "arbitrary-directory" + stage.mkdir() + (stage / "index_metadata.json").write_text("{}") + with pytest.raises(diskgraph.DiskgraphConfigError): + _extension_ref_from_stage_path(str(parent), str(stage)) + + +def test_external_extension_metadata_is_validated_by_native_reader(seed_table_fixture, tmp_path): + external = tmp_path / "not-an-index" + external.mkdir() + (external / "index_metadata.json").write_text("{}") + manifest = json.loads(pathlib.Path(seed_table_fixture["manifest_path"]).read_text()) + manifest["stages"]["train"]["index_path"] = str(external) + path = tmp_path / "manifest.json" + path.write_text(json.dumps(manifest)) + with pytest.raises(diskgraph.DiskgraphSystemError, match="extension"): + make_runtime_batch_iterator( + str(path), stage="train", entity_node_type="users", sampling_plan=_sampling_plan(), + num_workers=1, worker_id=0, shuffle=False, rnd_seed=17, epoch=0, + batch_size=3, drop_last=False, num_threads=1, + ) + + def test_static_task_manifest_suppresses_seed_timestamps( seed_table_fixture, tmp_path): """Static task datasets may carry pseudo TIMESTAMP columns, but those diff --git a/packages/diskgraph/diskgraph-pybind/src/cutoff_col_stats.rs b/packages/diskgraph/diskgraph-pybind/src/cutoff_col_stats.rs index ba306752c..457ba1a4f 100644 --- a/packages/diskgraph/diskgraph-pybind/src/cutoff_col_stats.rs +++ b/packages/diskgraph/diskgraph-pybind/src/cutoff_col_stats.rs @@ -43,7 +43,7 @@ const TOKENIZER_TRAINER: &str = "tokenizers 0.23.2 onig"; type WordCounts = AHashMap; /// Recomputes `request` from an index's stored rows. -fn from_index(index_dir: &Path, request: &CutoffColStatsRequest) -> Result<()> { +fn from_index(index_dir: &Path, output_dir: &Path, request: &CutoffColStatsRequest) -> Result<()> { let config = read_persisted_config(index_dir)?; request.validate(&config.tables)?; let names: HashSet = request.columns.keys().cloned().collect(); @@ -107,17 +107,19 @@ fn from_index(index_dir: &Path, request: &CutoffColStatsRequest) -> Result<()> { } tables.push(table); } - request.publish(index_dir, clock, tables) + request.publish(output_dir, clock, tables) } -/// Recomputes column statistics at cutoffs from an index's stored rows, into `col_stats//`. +/// Recomputes statistics from stored rows into `output_dir/col_stats//`. +/// The output defaults to the index directory for existing callers. #[pyfunction] -#[pyo3(signature = (index_dir, request_json, num_threads=None))] +#[pyo3(signature = (index_dir, request_json, num_threads=None, *, output_dir=None))] pub fn cutoff_col_stats( py: Python<'_>, index_dir: &str, request_json: &str, num_threads: Option, + output_dir: Option<&str>, ) -> PyResult<()> { let request: CutoffColStatsRequest = serde_json::from_str(request_json) .map_err(|e| DiskgraphConfigError::new_err(format!("invalid request_json: {e}")))?; @@ -125,7 +127,13 @@ pub fn cutoff_col_stats( let pool = rayon::ThreadPoolBuilder::new() .num_threads(num_threads.unwrap_or(0)) .build()?; - pool.install(|| from_index(Path::new(index_dir), &request)) + pool.install(|| { + from_index( + Path::new(index_dir), + Path::new(output_dir.unwrap_or(index_dir)), + &request, + ) + }) }) .map_err(map_anyhow_err) } diff --git a/packages/diskgraph/diskgraph-runtime/src/lib.rs b/packages/diskgraph/diskgraph-runtime/src/lib.rs index 0a9316637..b212816ca 100644 --- a/packages/diskgraph/diskgraph-runtime/src/lib.rs +++ b/packages/diskgraph/diskgraph-runtime/src/lib.rs @@ -1253,7 +1253,8 @@ pub struct RuntimeIndex { namespace: String, } impl RuntimeIndex { - /// Open the graph and training-table extension; propagate typed index errors. + /// Open the graph and a seed extension, by legacy reference or absolute path. + /// Absolute paths allow task storage outside a read-only graph index. pub fn new( parent_root: String, tt_id: String, @@ -1261,9 +1262,22 @@ impl RuntimeIndex { num_threads: Option, ) -> Result { let parent_path = std::path::Path::new(&parent_root); - let reader = diskgraph_index::graph::GraphIndexReader::open_with_tt(parent_path, &tt_id) - .map_err(|e| RuntimeError::system(e.to_string()))?; - let namespace = tt_id.rsplit('/').next().unwrap_or(&tt_id).to_string(); + let extension = std::path::Path::new(&tt_id); + let namespace = extension + .file_name() + .and_then(|n| n.to_str()) + .ok_or_else(|| RuntimeError::system("invalid extension path".to_string()))? + .to_string(); + let reader = if extension.is_absolute() { + diskgraph_index::graph::GraphIndexReader::open_with_extension_subdir( + parent_path, + &tt_id, + &namespace, + ) + } else { + diskgraph_index::graph::GraphIndexReader::open_with_tt(parent_path, &tt_id) + } + .map_err(|e| RuntimeError::system(e.to_string()))?; let reader = Arc::new(reader); let engine = Arc::new( DiskGraphEngine::from_reader(reader.clone(), rnd_seed, num_threads) From 6b2e66d0186d3feed90c9ec465fd0d36d0acb5a7 Mon Sep 17 00:00:00 2001 From: Alex Porter <102990994+am-porter@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:25:04 -0800 Subject: [PATCH 2/6] Clarify native extension path validation responsibility --- packages/diskgraph/diskgraph-index/src/graph.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/diskgraph/diskgraph-index/src/graph.rs b/packages/diskgraph/diskgraph-index/src/graph.rs index c2380a01e..7390b965c 100644 --- a/packages/diskgraph/diskgraph-index/src/graph.rs +++ b/packages/diskgraph/diskgraph-index/src/graph.rs @@ -1227,8 +1227,9 @@ impl GraphIndexReader { } /// Generalization of [`open_with_tt`] to an extension at an arbitrary - /// relative `subdir` under the parent, or an absolute path outside it. - /// External paths allow consumers to keep a shared parent read-only. The + /// relative `subdir` under the parent, or an absolute path. This low-level + /// reader does not enforce path placement; callers validate it. External + /// extensions allow consumers to keep a shared parent read-only. The /// extension's tables are merged under the `/` /// keys and its cross-boundary edges are decoded against the parent's /// node types — the same recipe `open_with_tt` uses. From 504dd1163411af57ab0376e0bf57df009578345c Mon Sep 17 00:00:00 2001 From: Alex Porter <102990994+am-porter@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:37:12 -0800 Subject: [PATCH 3/6] Document RuntimeIndex caller-owned path placement checks --- packages/diskgraph/diskgraph-runtime/src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/diskgraph/diskgraph-runtime/src/lib.rs b/packages/diskgraph/diskgraph-runtime/src/lib.rs index b212816ca..b3a46cd31 100644 --- a/packages/diskgraph/diskgraph-runtime/src/lib.rs +++ b/packages/diskgraph/diskgraph-runtime/src/lib.rs @@ -1254,7 +1254,8 @@ pub struct RuntimeIndex { } impl RuntimeIndex { /// Open the graph and a seed extension, by legacy reference or absolute path. - /// Absolute paths allow task storage outside a read-only graph index. + /// Absolute paths are passed to the index reader without placement checks; + /// callers validate their location, including any read-only base policy. pub fn new( parent_root: String, tt_id: String, From 8cdf9650a4479784230018c3ee44d6ceebc06224 Mon Sep 17 00:00:00 2001 From: Alex Porter <102990994+am-porter@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:50:25 -0800 Subject: [PATCH 4/6] Resolve original extension paths before lexical normalization --- .../diskgraph/runtime_batch_iterator.py | 7 +++-- .../tests/test_runtime_batch_iterator_e2e.py | 28 +++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/packages/diskgraph/diskgraph-pybind/python/diskgraph/runtime_batch_iterator.py b/packages/diskgraph/diskgraph-pybind/python/diskgraph/runtime_batch_iterator.py index d5df98e40..76dfbfc92 100644 --- a/packages/diskgraph/diskgraph-pybind/python/diskgraph/runtime_batch_iterator.py +++ b/packages/diskgraph/diskgraph-pybind/python/diskgraph/runtime_batch_iterator.py @@ -118,9 +118,10 @@ def _extension_ref_from_stage_path(parent_root: str, stage_index_path: str) -> s or an absolute external extension path for a read-only base index. External paths require finalized index metadata; the native reader validates it. """ - abs_parent = os.path.abspath(parent_root.rstrip("/")) - abs_stage = os.path.abspath(stage_index_path.rstrip("/")) - parent, stage = Path(abs_parent).resolve(), Path(abs_stage).resolve() + # Resolve symlinks before collapsing '..', so validation and native opening + # both select the directory identified by the original filesystem path. + parent, stage = Path(parent_root).resolve(), Path(stage_index_path).resolve() + abs_parent, abs_stage = str(parent), str(stage) if stage == parent: raise DiskgraphConfigError("stage index_path must identify an extension, not the base") for extension_dir in ("train_tables", "pred_tables"): diff --git a/packages/diskgraph/diskgraph-pybind/python/tests/test_runtime_batch_iterator_e2e.py b/packages/diskgraph/diskgraph-pybind/python/tests/test_runtime_batch_iterator_e2e.py index bf703164c..92bfefd8d 100644 --- a/packages/diskgraph/diskgraph-pybind/python/tests/test_runtime_batch_iterator_e2e.py +++ b/packages/diskgraph/diskgraph-pybind/python/tests/test_runtime_batch_iterator_e2e.py @@ -366,6 +366,34 @@ def test_extension_reference_preserves_legacy_layout(tmp_path, directory): ) == f"{directory}/task" +@pytest.mark.parametrize("path_input", ["stage", "parent"]) +def test_extension_reference_resolves_symlink_before_parent_segment(tmp_path, path_input): + from diskgraph.runtime_batch_iterator import _extension_ref_from_stage_path + + runs = tmp_path / "runs" + (runs / "run-1").mkdir(parents=True) + consumer = tmp_path / "consumer" + consumer.mkdir() + alias = consumer / "current" + alias.symlink_to(runs / "run-1", target_is_directory=True) + if path_input == "stage": + # Both locations pass the metadata-presence gate; lexical collapse + # would silently select consumer/task instead of runs/task. + for task in (runs / "task", consumer / "task"): + task.mkdir() + (task / "index_metadata.json").write_text("{}") + parent = tmp_path / "base" + stage = alias / ".." / "task" + expected = str((runs / "task").resolve()) + else: + parent = alias / ".." / "base" + stage = runs / "base" / "train_tables" / "task" + stage.mkdir(parents=True) + (stage / "index_metadata.json").write_text("{}") + expected = "train_tables/task" + assert _extension_ref_from_stage_path(str(parent), str(stage)) == expected + + @pytest.mark.parametrize("kind", ["missing", "empty", "base", "base_alias", "index_local"]) def test_extension_reference_refuses_non_extension_paths(tmp_path, kind): from diskgraph.runtime_batch_iterator import _extension_ref_from_stage_path From 3153cf5d7af4a18ad0e4fd048286723a50ab0ee3 Mon Sep 17 00:00:00 2001 From: Alex Porter <102990994+am-porter@users.noreply.github.com> Date: Wed, 18 Nov 2026 16:22:25 -0800 Subject: [PATCH 5/6] Share extension path validation in the native index reader --- .../diskgraph/diskgraph-index/src/graph.rs | 53 ++++++++++++++++--- .../diskgraph/runtime_batch_iterator.py | 30 +++-------- .../tests/test_runtime_batch_iterator_e2e.py | 36 +++++++++---- .../diskgraph/diskgraph-runtime/src/lib.rs | 4 +- 4 files changed, 80 insertions(+), 43 deletions(-) diff --git a/packages/diskgraph/diskgraph-index/src/graph.rs b/packages/diskgraph/diskgraph-index/src/graph.rs index 7390b965c..982d7d782 100644 --- a/packages/diskgraph/diskgraph-index/src/graph.rs +++ b/packages/diskgraph/diskgraph-index/src/graph.rs @@ -1226,10 +1226,10 @@ impl GraphIndexReader { Self::open_with_extension_subdir(parent_root, &subdir_rel, &namespace) } - /// Generalization of [`open_with_tt`] to an extension at an arbitrary - /// relative `subdir` under the parent, or an absolute path. This low-level - /// reader does not enforce path placement; callers validate it. External - /// extensions allow consumers to keep a shared parent read-only. The + /// Generalization of [`open_with_tt`] to a relative or absolute extension + /// path. Resolved paths must differ from the parent and, if inside it, + /// identify `train_tables/` or `pred_tables/`. External extensions + /// allow consumers to keep a shared parent read-only. The /// extension's tables are merged under the `/` /// keys and its cross-boundary edges are decoded against the parent's /// node types — the same recipe `open_with_tt` uses. @@ -1238,9 +1238,24 @@ impl GraphIndexReader { subdir_rel: &str, namespace: &str, ) -> Result { - let parent_meta_path = parent_root.join("index_metadata.json"); - let mut reader = Self::open(&parent_meta_path, parent_root)?; - let subdir = parent_root.join(subdir_rel); + // All opening paths share this validation, including direct native + // callers and legacy relative references that traverse symlinks. + let parent = fs::canonicalize(parent_root).context("resolve parent index path")?; + let subdir = + fs::canonicalize(parent_root.join(subdir_rel)).context("resolve extension index path")?; + ensure!( + subdir != parent, + "extension path must not identify the base index" + ); + if let Ok(relative) = subdir.strip_prefix(&parent) { + let parts: Vec<_> = relative.iter().collect(); + ensure!( + parts.len() == 2 && (parts[0] == "train_tables" || parts[0] == "pred_tables"), + "an index-local extension must identify train_tables/ or pred_tables/" + ); + } + let parent_meta_path = parent.join("index_metadata.json"); + let mut reader = Self::open(&parent_meta_path, &parent)?; let ext_meta_path = subdir.join("index_metadata.json"); if !ext_meta_path.is_file() { return Err(anyhow!( @@ -1974,6 +1989,30 @@ fn load_pk_hash_table( Ok(PkHashTable::new(key_type, shards)) } +#[cfg(test)] +mod extension_path_tests { + use super::*; + + #[test] + fn rejects_base_before_reading_metadata() { + let base = tempfile::tempdir().unwrap(); + for extension in [".", base.path().to_str().unwrap()] { + let result = GraphIndexReader::open_with_extension_subdir(base.path(), extension, "task"); + assert!(result.err().unwrap().to_string().contains("base index")); + } + } + + #[cfg(unix)] + #[test] + fn legacy_reference_cannot_alias_base() { + let base = tempfile::tempdir().unwrap(); + fs::create_dir(base.path().join("train_tables")).unwrap(); + std::os::unix::fs::symlink(base.path(), base.path().join("train_tables/alias")).unwrap(); + let result = GraphIndexReader::open_with_tt(base.path(), "alias"); + assert!(result.err().unwrap().to_string().contains("base index")); + } +} + #[cfg(test)] mod scratch_cleanup_tests { use super::*; diff --git a/packages/diskgraph/diskgraph-pybind/python/diskgraph/runtime_batch_iterator.py b/packages/diskgraph/diskgraph-pybind/python/diskgraph/runtime_batch_iterator.py index 76dfbfc92..f29186777 100644 --- a/packages/diskgraph/diskgraph-pybind/python/diskgraph/runtime_batch_iterator.py +++ b/packages/diskgraph/diskgraph-pybind/python/diskgraph/runtime_batch_iterator.py @@ -118,32 +118,14 @@ def _extension_ref_from_stage_path(parent_root: str, stage_index_path: str) -> s or an absolute external extension path for a read-only base index. External paths require finalized index metadata; the native reader validates it. """ - # Resolve symlinks before collapsing '..', so validation and native opening - # both select the directory identified by the original filesystem path. + # Preserve filesystem semantics (symlinks before '..') and legacy references. + # Placement and finalized-metadata validation belong to the native reader. parent, stage = Path(parent_root).resolve(), Path(stage_index_path).resolve() - abs_parent, abs_stage = str(parent), str(stage) - if stage == parent: - raise DiskgraphConfigError("stage index_path must identify an extension, not the base") - for extension_dir in ("train_tables", "pred_tables"): - expected_prefix = os.path.join(abs_parent, extension_dir) + os.sep - if not abs_stage.startswith(expected_prefix): - continue - extension_id = abs_stage[len(expected_prefix):].split(os.sep, 1)[0] - if not extension_id: - raise DiskgraphConfigError( - "could not extract extension id from stage index_path " - f"{stage_index_path}" - ) - return f"{extension_dir}/{extension_id}" if stage.is_relative_to(parent): - raise DiskgraphConfigError( - "an index-local stage index_path must live under train_tables/ or pred_tables/" - ) - if not (stage / "index_metadata.json").is_file(): - raise DiskgraphConfigError( - f"external stage index_path has no finalized index_metadata.json: {abs_stage}" - ) - return abs_stage + parts = stage.relative_to(parent).parts + if len(parts) == 2 and parts[0] in ("train_tables", "pred_tables"): + return "/".join(parts) + return str(stage) def _sampling_plan_json(sampling_plan: Any) -> str: diff --git a/packages/diskgraph/diskgraph-pybind/python/tests/test_runtime_batch_iterator_e2e.py b/packages/diskgraph/diskgraph-pybind/python/tests/test_runtime_batch_iterator_e2e.py index 92bfefd8d..a65da050c 100644 --- a/packages/diskgraph/diskgraph-pybind/python/tests/test_runtime_batch_iterator_e2e.py +++ b/packages/diskgraph/diskgraph-pybind/python/tests/test_runtime_batch_iterator_e2e.py @@ -394,13 +394,11 @@ def test_extension_reference_resolves_symlink_before_parent_segment(tmp_path, pa assert _extension_ref_from_stage_path(str(parent), str(stage)) == expected -@pytest.mark.parametrize("kind", ["missing", "empty", "base", "base_alias", "index_local"]) -def test_extension_reference_refuses_non_extension_paths(tmp_path, kind): - from diskgraph.runtime_batch_iterator import _extension_ref_from_stage_path - - parent = tmp_path / "base" - parent.mkdir() - (parent / "index_metadata.json").write_text("{}") +@pytest.mark.parametrize("entrypoint", ["native", "manifest"]) +@pytest.mark.parametrize("kind", ["missing", "empty", "base", "base_alias", "index_local", + "legacy_alias"]) +def test_runtime_rejects_non_extension_paths(seed_table_fixture, tmp_path, kind, entrypoint): + parent = pathlib.Path(seed_table_fixture["parent_root"]) stage = tmp_path / "stage" if kind == "empty": stage.mkdir() @@ -410,10 +408,28 @@ def test_extension_reference_refuses_non_extension_paths(tmp_path, kind): stage.symlink_to(parent, target_is_directory=True) elif kind == "index_local": stage = parent / "arbitrary-directory" - stage.mkdir() + stage.mkdir(exist_ok=True) (stage / "index_metadata.json").write_text("{}") - with pytest.raises(diskgraph.DiskgraphConfigError): - _extension_ref_from_stage_path(str(parent), str(stage)) + elif kind == "legacy_alias": + stage = parent / "train_tables" / ("base-alias-" + entrypoint) + stage.symlink_to(parent, target_is_directory=True) + match = "base index" if kind in ("base", "base_alias", "legacy_alias") else "extension" + with pytest.raises(diskgraph.DiskgraphSystemError, match=match): + if entrypoint == "native": + from diskgraph._c import RuntimeIndex as NativeRuntimeIndex + + ref = str(stage.relative_to(parent)) if kind == "legacy_alias" else str(stage) + NativeRuntimeIndex(str(parent), ref, 17, 1) + else: + manifest = json.loads(pathlib.Path(seed_table_fixture["manifest_path"]).read_text()) + manifest["stages"]["train"]["index_path"] = str(stage) + path = tmp_path / "manifest.json" + path.write_text(json.dumps(manifest)) + make_runtime_batch_iterator( + str(path), stage="train", entity_node_type="users", sampling_plan=_sampling_plan(), + num_workers=1, worker_id=0, shuffle=False, rnd_seed=17, epoch=0, + batch_size=3, drop_last=False, num_threads=1, + ) def test_external_extension_metadata_is_validated_by_native_reader(seed_table_fixture, tmp_path): diff --git a/packages/diskgraph/diskgraph-runtime/src/lib.rs b/packages/diskgraph/diskgraph-runtime/src/lib.rs index b3a46cd31..ce63047c7 100644 --- a/packages/diskgraph/diskgraph-runtime/src/lib.rs +++ b/packages/diskgraph/diskgraph-runtime/src/lib.rs @@ -1254,8 +1254,8 @@ pub struct RuntimeIndex { } impl RuntimeIndex { /// Open the graph and a seed extension, by legacy reference or absolute path. - /// Absolute paths are passed to the index reader without placement checks; - /// callers validate their location, including any read-only base policy. + /// The shared index reader validates relative and absolute extension paths, + /// including rejecting the parent itself and aliases of it. pub fn new( parent_root: String, tt_id: String, From 3b92023ebe1646c1911a5347651184c11144b82e Mon Sep 17 00:00:00 2001 From: Alex Porter <102990994+am-porter@users.noreply.github.com> Date: Wed, 18 Nov 2026 16:36:29 -0800 Subject: [PATCH 6/6] Preserve extension identity in path resolution errors --- packages/diskgraph/diskgraph-index/src/graph.rs | 4 ++-- .../python/tests/test_runtime_batch_iterator_e2e.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/diskgraph/diskgraph-index/src/graph.rs b/packages/diskgraph/diskgraph-index/src/graph.rs index 982d7d782..85d75878d 100644 --- a/packages/diskgraph/diskgraph-index/src/graph.rs +++ b/packages/diskgraph/diskgraph-index/src/graph.rs @@ -1241,8 +1241,8 @@ impl GraphIndexReader { // All opening paths share this validation, including direct native // callers and legacy relative references that traverse symlinks. let parent = fs::canonicalize(parent_root).context("resolve parent index path")?; - let subdir = - fs::canonicalize(parent_root.join(subdir_rel)).context("resolve extension index path")?; + let subdir = fs::canonicalize(parent_root.join(subdir_rel)) + .with_context(|| format!("resolve extension `{namespace}` index path: {subdir_rel}"))?; ensure!( subdir != parent, "extension path must not identify the base index" diff --git a/packages/diskgraph/diskgraph-pybind/python/tests/test_runtime_batch_iterator_e2e.py b/packages/diskgraph/diskgraph-pybind/python/tests/test_runtime_batch_iterator_e2e.py index a65da050c..47bbb8aaa 100644 --- a/packages/diskgraph/diskgraph-pybind/python/tests/test_runtime_batch_iterator_e2e.py +++ b/packages/diskgraph/diskgraph-pybind/python/tests/test_runtime_batch_iterator_e2e.py @@ -414,6 +414,8 @@ def test_runtime_rejects_non_extension_paths(seed_table_fixture, tmp_path, kind, stage = parent / "train_tables" / ("base-alias-" + entrypoint) stage.symlink_to(parent, target_is_directory=True) match = "base index" if kind in ("base", "base_alias", "legacy_alias") else "extension" + if kind == "missing": + match = "extension `stage`" with pytest.raises(diskgraph.DiskgraphSystemError, match=match): if entrypoint == "native": from diskgraph._c import RuntimeIndex as NativeRuntimeIndex