diff --git a/packages/diskgraph/diskgraph-index/src/graph.rs b/packages/diskgraph/diskgraph-index/src/graph.rs index dc5187982..85d75878d 100644 --- a/packages/diskgraph/diskgraph-index/src/graph.rs +++ b/packages/diskgraph/diskgraph-index/src/graph.rs @@ -1226,9 +1226,10 @@ impl GraphIndexReader { Self::open_with_extension_subdir(parent_root, &subdir_rel, &namespace) } - /// Generalization of [`open_with_tt`] to an extension at an arbitrary - /// relative `subdir` under the parent (e.g. `pred_tables/job1` for the - /// batch-prediction path, or `train_tables/` for training). The + /// Generalization of [`open_with_tt`] to a relative or absolute extension + /// path. Resolved paths must differ from the parent and, if inside it, + /// identify `train_tables/` or `pred_tables/`. External extensions + /// allow consumers to keep a shared parent read-only. The /// extension's tables are merged under the `/` /// keys and its cross-boundary edges are decoded against the parent's /// node types — the same recipe `open_with_tt` uses. @@ -1237,9 +1238,24 @@ impl GraphIndexReader { subdir_rel: &str, namespace: &str, ) -> Result { - let parent_meta_path = parent_root.join("index_metadata.json"); - let mut reader = Self::open(&parent_meta_path, parent_root)?; - let subdir = parent_root.join(subdir_rel); + // All opening paths share this validation, including direct native + // callers and legacy relative references that traverse symlinks. + let parent = fs::canonicalize(parent_root).context("resolve parent index path")?; + let subdir = fs::canonicalize(parent_root.join(subdir_rel)) + .with_context(|| format!("resolve extension `{namespace}` index path: {subdir_rel}"))?; + ensure!( + subdir != parent, + "extension path must not identify the base index" + ); + if let Ok(relative) = subdir.strip_prefix(&parent) { + let parts: Vec<_> = relative.iter().collect(); + ensure!( + parts.len() == 2 && (parts[0] == "train_tables" || parts[0] == "pred_tables"), + "an index-local extension must identify train_tables/ or pred_tables/" + ); + } + let parent_meta_path = parent.join("index_metadata.json"); + let mut reader = Self::open(&parent_meta_path, &parent)?; let ext_meta_path = subdir.join("index_metadata.json"); if !ext_meta_path.is_file() { return Err(anyhow!( @@ -1973,6 +1989,30 @@ fn load_pk_hash_table( Ok(PkHashTable::new(key_type, shards)) } +#[cfg(test)] +mod extension_path_tests { + use super::*; + + #[test] + fn rejects_base_before_reading_metadata() { + let base = tempfile::tempdir().unwrap(); + for extension in [".", base.path().to_str().unwrap()] { + let result = GraphIndexReader::open_with_extension_subdir(base.path(), extension, "task"); + assert!(result.err().unwrap().to_string().contains("base index")); + } + } + + #[cfg(unix)] + #[test] + fn legacy_reference_cannot_alias_base() { + let base = tempfile::tempdir().unwrap(); + fs::create_dir(base.path().join("train_tables")).unwrap(); + std::os::unix::fs::symlink(base.path(), base.path().join("train_tables/alias")).unwrap(); + let result = GraphIndexReader::open_with_tt(base.path(), "alias"); + assert!(result.err().unwrap().to_string().contains("base index")); + } +} + #[cfg(test)] mod scratch_cleanup_tests { use super::*; diff --git a/packages/diskgraph/diskgraph-pybind/python/diskgraph/runtime_batch_iterator.py b/packages/diskgraph/diskgraph-pybind/python/diskgraph/runtime_batch_iterator.py index 136e1f299..f29186777 100644 --- a/packages/diskgraph/diskgraph-pybind/python/diskgraph/runtime_batch_iterator.py +++ b/packages/diskgraph/diskgraph-pybind/python/diskgraph/runtime_batch_iterator.py @@ -19,6 +19,7 @@ import json import os +from pathlib import Path from typing import Any, Mapping, Optional from ._exceptions import DiskgraphConfigError @@ -113,26 +114,18 @@ def _extension_ref_from_stage_path(parent_root: str, stage_index_path: str) -> s """Recover the seed-table extension reference from a stage ``index_path``. The stage path itself is the source of truth: return a relative - ``train_tables/`` or ``pred_tables/`` ref under the parent. + ``train_tables/`` or ``pred_tables/`` ref under the parent, + or an absolute external extension path for a read-only base index. External + paths require finalized index metadata; the native reader validates it. """ - abs_parent = os.path.abspath(parent_root.rstrip("/")) - abs_stage = os.path.abspath(stage_index_path.rstrip("/")) - for extension_dir in ("train_tables", "pred_tables"): - expected_prefix = os.path.join(abs_parent, extension_dir) + os.sep - if not abs_stage.startswith(expected_prefix): - continue - extension_id = abs_stage[len(expected_prefix):].split(os.sep, 1)[0] - if not extension_id: - raise DiskgraphConfigError( - "could not extract extension id from stage index_path " - f"{stage_index_path}" - ) - return f"{extension_dir}/{extension_id}" - raise DiskgraphConfigError( - "stage index_path must live under /train_tables/ " - "or /pred_tables/: " - f"parent={abs_parent}, stage={abs_stage}" - ) + # Preserve filesystem semantics (symlinks before '..') and legacy references. + # Placement and finalized-metadata validation belong to the native reader. + parent, stage = Path(parent_root).resolve(), Path(stage_index_path).resolve() + if stage.is_relative_to(parent): + parts = stage.relative_to(parent).parts + if len(parts) == 2 and parts[0] in ("train_tables", "pred_tables"): + return "/".join(parts) + return str(stage) def _sampling_plan_json(sampling_plan: Any) -> str: @@ -459,4 +452,3 @@ def iter( ) return iterator, iterator.num_samples() - diff --git a/packages/diskgraph/diskgraph-pybind/python/tests/test_cutoff_col_stats.py b/packages/diskgraph/diskgraph-pybind/python/tests/test_cutoff_col_stats.py index 1d4a03542..5aedc8039 100644 --- a/packages/diskgraph/diskgraph-pybind/python/tests/test_cutoff_col_stats.py +++ b/packages/diskgraph/diskgraph-pybind/python/tests/test_cutoff_col_stats.py @@ -143,6 +143,16 @@ def test_recompute_from_the_index_matches_ingest(data, ingested, tmp_path): assert _stats(str(index)) == _stats(ingested) +def test_recompute_into_external_directory(data, ingested, tmp_path): + index = tmp_path / "index" + shutil.copytree(ingested, index, ignore=shutil.ignore_patterns("col_stats")) + output = tmp_path / "cache" + diskgraph.cutoff_col_stats(str(index), json.dumps(REQUEST), output_dir=str(output), + num_threads=2) + assert _stats(str(output)) == _stats(ingested) + assert not (index / "col_stats").exists() + + def test_rejects_unknown_columns_and_exclusive_options(data, ingested, tmp_path): with pytest.raises(Exception, match="has no column missing"): diskgraph.cutoff_col_stats(ingested, json.dumps( diff --git a/packages/diskgraph/diskgraph-pybind/python/tests/test_runtime_batch_iterator_e2e.py b/packages/diskgraph/diskgraph-pybind/python/tests/test_runtime_batch_iterator_e2e.py index c0afc3bdd..47bbb8aaa 100644 --- a/packages/diskgraph/diskgraph-pybind/python/tests/test_runtime_batch_iterator_e2e.py +++ b/packages/diskgraph/diskgraph-pybind/python/tests/test_runtime_batch_iterator_e2e.py @@ -25,6 +25,7 @@ import inspect import json import pathlib +import shutil import struct from typing import Any @@ -328,6 +329,127 @@ def test_iterator_yields_dict_matching_contract(seed_table_fixture): ) +def test_external_extension_runs_without_an_index_local_copy(seed_table_fixture, tmp_path): + parent = tmp_path / "parent" + shutil.copytree(seed_table_fixture["parent_root"], parent) + external = tmp_path / "consumer" / seed_table_fixture["tt_id"] + external.parent.mkdir() + shutil.move(str(parent / "train_tables" / seed_table_fixture["tt_id"]), external) + before = {str(p.relative_to(parent)): p.read_bytes() + for p in parent.rglob("*") if p.is_file()} + manifest = json.loads(pathlib.Path(seed_table_fixture["manifest_path"]).read_text()) + manifest["graph"]["index_path"] = str(parent) + manifest["stages"]["train"]["index_path"] = str(external) + path = tmp_path / "manifest.json" + path.write_text(json.dumps(manifest)) + iterator, count = make_runtime_batch_iterator( + str(path), stage="train", entity_node_type="users", sampling_plan=_sampling_plan(), + num_workers=1, worker_id=0, shuffle=False, rnd_seed=17, epoch=0, + batch_size=3, drop_last=False, + ) + assert count == seed_table_fixture["n_seeds"] + batches = list(iterator) + assert len(batches) == 2 + for batch in batches: + _assert_runtime_batch_dict_contract(batch, 3, seed_table_fixture["n_users"]) + assert {str(p.relative_to(parent)): p.read_bytes() + for p in parent.rglob("*") if p.is_file()} == before + + +@pytest.mark.parametrize("directory", ["train_tables", "pred_tables"]) +def test_extension_reference_preserves_legacy_layout(tmp_path, directory): + from diskgraph.runtime_batch_iterator import _extension_ref_from_stage_path + + parent = tmp_path / "base" + assert _extension_ref_from_stage_path( + str(parent), str(parent / directory / "task") + ) == f"{directory}/task" + + +@pytest.mark.parametrize("path_input", ["stage", "parent"]) +def test_extension_reference_resolves_symlink_before_parent_segment(tmp_path, path_input): + from diskgraph.runtime_batch_iterator import _extension_ref_from_stage_path + + runs = tmp_path / "runs" + (runs / "run-1").mkdir(parents=True) + consumer = tmp_path / "consumer" + consumer.mkdir() + alias = consumer / "current" + alias.symlink_to(runs / "run-1", target_is_directory=True) + if path_input == "stage": + # Both locations pass the metadata-presence gate; lexical collapse + # would silently select consumer/task instead of runs/task. + for task in (runs / "task", consumer / "task"): + task.mkdir() + (task / "index_metadata.json").write_text("{}") + parent = tmp_path / "base" + stage = alias / ".." / "task" + expected = str((runs / "task").resolve()) + else: + parent = alias / ".." / "base" + stage = runs / "base" / "train_tables" / "task" + stage.mkdir(parents=True) + (stage / "index_metadata.json").write_text("{}") + expected = "train_tables/task" + assert _extension_ref_from_stage_path(str(parent), str(stage)) == expected + + +@pytest.mark.parametrize("entrypoint", ["native", "manifest"]) +@pytest.mark.parametrize("kind", ["missing", "empty", "base", "base_alias", "index_local", + "legacy_alias"]) +def test_runtime_rejects_non_extension_paths(seed_table_fixture, tmp_path, kind, entrypoint): + parent = pathlib.Path(seed_table_fixture["parent_root"]) + stage = tmp_path / "stage" + if kind == "empty": + stage.mkdir() + elif kind == "base": + stage = parent + elif kind == "base_alias": + stage.symlink_to(parent, target_is_directory=True) + elif kind == "index_local": + stage = parent / "arbitrary-directory" + stage.mkdir(exist_ok=True) + (stage / "index_metadata.json").write_text("{}") + elif kind == "legacy_alias": + stage = parent / "train_tables" / ("base-alias-" + entrypoint) + stage.symlink_to(parent, target_is_directory=True) + match = "base index" if kind in ("base", "base_alias", "legacy_alias") else "extension" + if kind == "missing": + match = "extension `stage`" + with pytest.raises(diskgraph.DiskgraphSystemError, match=match): + if entrypoint == "native": + from diskgraph._c import RuntimeIndex as NativeRuntimeIndex + + ref = str(stage.relative_to(parent)) if kind == "legacy_alias" else str(stage) + NativeRuntimeIndex(str(parent), ref, 17, 1) + else: + manifest = json.loads(pathlib.Path(seed_table_fixture["manifest_path"]).read_text()) + manifest["stages"]["train"]["index_path"] = str(stage) + path = tmp_path / "manifest.json" + path.write_text(json.dumps(manifest)) + make_runtime_batch_iterator( + str(path), stage="train", entity_node_type="users", sampling_plan=_sampling_plan(), + num_workers=1, worker_id=0, shuffle=False, rnd_seed=17, epoch=0, + batch_size=3, drop_last=False, num_threads=1, + ) + + +def test_external_extension_metadata_is_validated_by_native_reader(seed_table_fixture, tmp_path): + external = tmp_path / "not-an-index" + external.mkdir() + (external / "index_metadata.json").write_text("{}") + manifest = json.loads(pathlib.Path(seed_table_fixture["manifest_path"]).read_text()) + manifest["stages"]["train"]["index_path"] = str(external) + path = tmp_path / "manifest.json" + path.write_text(json.dumps(manifest)) + with pytest.raises(diskgraph.DiskgraphSystemError, match="extension"): + make_runtime_batch_iterator( + str(path), stage="train", entity_node_type="users", sampling_plan=_sampling_plan(), + num_workers=1, worker_id=0, shuffle=False, rnd_seed=17, epoch=0, + batch_size=3, drop_last=False, num_threads=1, + ) + + def test_static_task_manifest_suppresses_seed_timestamps( seed_table_fixture, tmp_path): """Static task datasets may carry pseudo TIMESTAMP columns, but those diff --git a/packages/diskgraph/diskgraph-pybind/src/cutoff_col_stats.rs b/packages/diskgraph/diskgraph-pybind/src/cutoff_col_stats.rs index ba306752c..457ba1a4f 100644 --- a/packages/diskgraph/diskgraph-pybind/src/cutoff_col_stats.rs +++ b/packages/diskgraph/diskgraph-pybind/src/cutoff_col_stats.rs @@ -43,7 +43,7 @@ const TOKENIZER_TRAINER: &str = "tokenizers 0.23.2 onig"; type WordCounts = AHashMap; /// Recomputes `request` from an index's stored rows. -fn from_index(index_dir: &Path, request: &CutoffColStatsRequest) -> Result<()> { +fn from_index(index_dir: &Path, output_dir: &Path, request: &CutoffColStatsRequest) -> Result<()> { let config = read_persisted_config(index_dir)?; request.validate(&config.tables)?; let names: HashSet = request.columns.keys().cloned().collect(); @@ -107,17 +107,19 @@ fn from_index(index_dir: &Path, request: &CutoffColStatsRequest) -> Result<()> { } tables.push(table); } - request.publish(index_dir, clock, tables) + request.publish(output_dir, clock, tables) } -/// Recomputes column statistics at cutoffs from an index's stored rows, into `col_stats//`. +/// Recomputes statistics from stored rows into `output_dir/col_stats//`. +/// The output defaults to the index directory for existing callers. #[pyfunction] -#[pyo3(signature = (index_dir, request_json, num_threads=None))] +#[pyo3(signature = (index_dir, request_json, num_threads=None, *, output_dir=None))] pub fn cutoff_col_stats( py: Python<'_>, index_dir: &str, request_json: &str, num_threads: Option, + output_dir: Option<&str>, ) -> PyResult<()> { let request: CutoffColStatsRequest = serde_json::from_str(request_json) .map_err(|e| DiskgraphConfigError::new_err(format!("invalid request_json: {e}")))?; @@ -125,7 +127,13 @@ pub fn cutoff_col_stats( let pool = rayon::ThreadPoolBuilder::new() .num_threads(num_threads.unwrap_or(0)) .build()?; - pool.install(|| from_index(Path::new(index_dir), &request)) + pool.install(|| { + from_index( + Path::new(index_dir), + Path::new(output_dir.unwrap_or(index_dir)), + &request, + ) + }) }) .map_err(map_anyhow_err) } diff --git a/packages/diskgraph/diskgraph-runtime/src/lib.rs b/packages/diskgraph/diskgraph-runtime/src/lib.rs index 0a9316637..ce63047c7 100644 --- a/packages/diskgraph/diskgraph-runtime/src/lib.rs +++ b/packages/diskgraph/diskgraph-runtime/src/lib.rs @@ -1253,7 +1253,9 @@ pub struct RuntimeIndex { namespace: String, } impl RuntimeIndex { - /// Open the graph and training-table extension; propagate typed index errors. + /// Open the graph and a seed extension, by legacy reference or absolute path. + /// The shared index reader validates relative and absolute extension paths, + /// including rejecting the parent itself and aliases of it. pub fn new( parent_root: String, tt_id: String, @@ -1261,9 +1263,22 @@ impl RuntimeIndex { num_threads: Option, ) -> Result { let parent_path = std::path::Path::new(&parent_root); - let reader = diskgraph_index::graph::GraphIndexReader::open_with_tt(parent_path, &tt_id) - .map_err(|e| RuntimeError::system(e.to_string()))?; - let namespace = tt_id.rsplit('/').next().unwrap_or(&tt_id).to_string(); + let extension = std::path::Path::new(&tt_id); + let namespace = extension + .file_name() + .and_then(|n| n.to_str()) + .ok_or_else(|| RuntimeError::system("invalid extension path".to_string()))? + .to_string(); + let reader = if extension.is_absolute() { + diskgraph_index::graph::GraphIndexReader::open_with_extension_subdir( + parent_path, + &tt_id, + &namespace, + ) + } else { + diskgraph_index::graph::GraphIndexReader::open_with_tt(parent_path, &tt_id) + } + .map_err(|e| RuntimeError::system(e.to_string()))?; let reader = Arc::new(reader); let engine = Arc::new( DiskGraphEngine::from_reader(reader.clone(), rnd_seed, num_threads)