Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 46 additions & 6 deletions packages/diskgraph/diskgraph-index/src/graph.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1226,9 +1226,10 @@ impl GraphIndexReader {
Self::open_with_extension_subdir(parent_root, &subdir_rel, &namespace)
}

/// Generalization of [`open_with_tt`] to an extension at an arbitrary
/// relative `subdir` under the parent (e.g. `pred_tables/job1` for the
/// batch-prediction path, or `train_tables/<tt_id>` for training). The
/// Generalization of [`open_with_tt`] to a relative or absolute extension
/// path. Resolved paths must differ from the parent and, if inside it,
/// identify `train_tables/<id>` or `pred_tables/<id>`. External extensions
/// allow consumers to keep a shared parent read-only. The
/// extension's tables are merged under the `<namespace>/<table_name>`
/// keys and its cross-boundary edges are decoded against the parent's
/// node types — the same recipe `open_with_tt` uses.
Expand All @@ -1237,9 +1238,24 @@ impl GraphIndexReader {
subdir_rel: &str,
namespace: &str,
) -> Result<Self> {
let parent_meta_path = parent_root.join("index_metadata.json");
let mut reader = Self::open(&parent_meta_path, parent_root)?;
let subdir = parent_root.join(subdir_rel);
// All opening paths share this validation, including direct native
// callers and legacy relative references that traverse symlinks.
let parent = fs::canonicalize(parent_root).context("resolve parent index path")?;
let subdir = fs::canonicalize(parent_root.join(subdir_rel))
.with_context(|| format!("resolve extension `{namespace}` index path: {subdir_rel}"))?;
ensure!(
subdir != parent,
"extension path must not identify the base index"
);
if let Ok(relative) = subdir.strip_prefix(&parent) {
let parts: Vec<_> = relative.iter().collect();
ensure!(
parts.len() == 2 && (parts[0] == "train_tables" || parts[0] == "pred_tables"),
"an index-local extension must identify train_tables/<id> or pred_tables/<id>"
);
}
let parent_meta_path = parent.join("index_metadata.json");
let mut reader = Self::open(&parent_meta_path, &parent)?;
let ext_meta_path = subdir.join("index_metadata.json");
if !ext_meta_path.is_file() {
return Err(anyhow!(
Expand Down Expand Up @@ -1973,6 +1989,30 @@ fn load_pk_hash_table(
Ok(PkHashTable::new(key_type, shards))
}

#[cfg(test)]
mod extension_path_tests {
use super::*;

#[test]
fn rejects_base_before_reading_metadata() {
let base = tempfile::tempdir().unwrap();
for extension in [".", base.path().to_str().unwrap()] {
let result = GraphIndexReader::open_with_extension_subdir(base.path(), extension, "task");
assert!(result.err().unwrap().to_string().contains("base index"));
}
}

#[cfg(unix)]
#[test]
fn legacy_reference_cannot_alias_base() {
let base = tempfile::tempdir().unwrap();
fs::create_dir(base.path().join("train_tables")).unwrap();
std::os::unix::fs::symlink(base.path(), base.path().join("train_tables/alias")).unwrap();
let result = GraphIndexReader::open_with_tt(base.path(), "alias");
assert!(result.err().unwrap().to_string().contains("base index"));
}
}

#[cfg(test)]
mod scratch_cleanup_tests {
use super::*;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

import json
import os
from pathlib import Path
from typing import Any, Mapping, Optional

from ._exceptions import DiskgraphConfigError
Expand Down Expand Up @@ -113,26 +114,18 @@ def _extension_ref_from_stage_path(parent_root: str, stage_index_path: str) -> s
"""Recover the seed-table extension reference from a stage ``index_path``.

The stage path itself is the source of truth: return a relative
``train_tables/<tt_id>`` or ``pred_tables/<job_id>`` ref under the parent.
``train_tables/<tt_id>`` or ``pred_tables/<job_id>`` ref under the parent,
or an absolute external extension path for a read-only base index. External
paths require finalized index metadata; the native reader validates it.
"""
abs_parent = os.path.abspath(parent_root.rstrip("/"))
abs_stage = os.path.abspath(stage_index_path.rstrip("/"))
for extension_dir in ("train_tables", "pred_tables"):
expected_prefix = os.path.join(abs_parent, extension_dir) + os.sep
if not abs_stage.startswith(expected_prefix):
continue
extension_id = abs_stage[len(expected_prefix):].split(os.sep, 1)[0]
if not extension_id:
raise DiskgraphConfigError(
"could not extract extension id from stage index_path "
f"{stage_index_path}"
)
return f"{extension_dir}/{extension_id}"
raise DiskgraphConfigError(
"stage index_path must live under <graph_index_path>/train_tables/ "
"or <graph_index_path>/pred_tables/: "
f"parent={abs_parent}, stage={abs_stage}"
)
# Preserve filesystem semantics (symlinks before '..') and legacy references.
# Placement and finalized-metadata validation belong to the native reader.
parent, stage = Path(parent_root).resolve(), Path(stage_index_path).resolve()
if stage.is_relative_to(parent):
parts = stage.relative_to(parent).parts
if len(parts) == 2 and parts[0] in ("train_tables", "pred_tables"):
return "/".join(parts)
return str(stage)


def _sampling_plan_json(sampling_plan: Any) -> str:
Expand Down Expand Up @@ -459,4 +452,3 @@ def iter(
)
return iterator, iterator.num_samples()


Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,16 @@ def test_recompute_from_the_index_matches_ingest(data, ingested, tmp_path):
assert _stats(str(index)) == _stats(ingested)


def test_recompute_into_external_directory(data, ingested, tmp_path):
index = tmp_path / "index"
shutil.copytree(ingested, index, ignore=shutil.ignore_patterns("col_stats"))
output = tmp_path / "cache"
diskgraph.cutoff_col_stats(str(index), json.dumps(REQUEST), output_dir=str(output),
num_threads=2)
assert _stats(str(output)) == _stats(ingested)
assert not (index / "col_stats").exists()


def test_rejects_unknown_columns_and_exclusive_options(data, ingested, tmp_path):
with pytest.raises(Exception, match="has no column missing"):
diskgraph.cutoff_col_stats(ingested, json.dumps(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import inspect
import json
import pathlib
import shutil
import struct
from typing import Any

Expand Down Expand Up @@ -328,6 +329,127 @@ def test_iterator_yields_dict_matching_contract(seed_table_fixture):
)


def test_external_extension_runs_without_an_index_local_copy(seed_table_fixture, tmp_path):
parent = tmp_path / "parent"
shutil.copytree(seed_table_fixture["parent_root"], parent)
external = tmp_path / "consumer" / seed_table_fixture["tt_id"]
external.parent.mkdir()
shutil.move(str(parent / "train_tables" / seed_table_fixture["tt_id"]), external)
before = {str(p.relative_to(parent)): p.read_bytes()
for p in parent.rglob("*") if p.is_file()}
manifest = json.loads(pathlib.Path(seed_table_fixture["manifest_path"]).read_text())
manifest["graph"]["index_path"] = str(parent)
manifest["stages"]["train"]["index_path"] = str(external)
path = tmp_path / "manifest.json"
path.write_text(json.dumps(manifest))
iterator, count = make_runtime_batch_iterator(
str(path), stage="train", entity_node_type="users", sampling_plan=_sampling_plan(),
num_workers=1, worker_id=0, shuffle=False, rnd_seed=17, epoch=0,
batch_size=3, drop_last=False,
)
assert count == seed_table_fixture["n_seeds"]
batches = list(iterator)
assert len(batches) == 2
for batch in batches:
_assert_runtime_batch_dict_contract(batch, 3, seed_table_fixture["n_users"])
assert {str(p.relative_to(parent)): p.read_bytes()
for p in parent.rglob("*") if p.is_file()} == before


@pytest.mark.parametrize("directory", ["train_tables", "pred_tables"])
def test_extension_reference_preserves_legacy_layout(tmp_path, directory):
from diskgraph.runtime_batch_iterator import _extension_ref_from_stage_path

parent = tmp_path / "base"
assert _extension_ref_from_stage_path(
str(parent), str(parent / directory / "task")
) == f"{directory}/task"


@pytest.mark.parametrize("path_input", ["stage", "parent"])
def test_extension_reference_resolves_symlink_before_parent_segment(tmp_path, path_input):
from diskgraph.runtime_batch_iterator import _extension_ref_from_stage_path

runs = tmp_path / "runs"
(runs / "run-1").mkdir(parents=True)
consumer = tmp_path / "consumer"
consumer.mkdir()
alias = consumer / "current"
alias.symlink_to(runs / "run-1", target_is_directory=True)
if path_input == "stage":
# Both locations pass the metadata-presence gate; lexical collapse
# would silently select consumer/task instead of runs/task.
for task in (runs / "task", consumer / "task"):
task.mkdir()
(task / "index_metadata.json").write_text("{}")
parent = tmp_path / "base"
stage = alias / ".." / "task"
expected = str((runs / "task").resolve())
else:
parent = alias / ".." / "base"
stage = runs / "base" / "train_tables" / "task"
stage.mkdir(parents=True)
(stage / "index_metadata.json").write_text("{}")
expected = "train_tables/task"
assert _extension_ref_from_stage_path(str(parent), str(stage)) == expected


@pytest.mark.parametrize("entrypoint", ["native", "manifest"])
@pytest.mark.parametrize("kind", ["missing", "empty", "base", "base_alias", "index_local",
"legacy_alias"])
def test_runtime_rejects_non_extension_paths(seed_table_fixture, tmp_path, kind, entrypoint):
parent = pathlib.Path(seed_table_fixture["parent_root"])
stage = tmp_path / "stage"
if kind == "empty":
stage.mkdir()
elif kind == "base":
stage = parent
elif kind == "base_alias":
stage.symlink_to(parent, target_is_directory=True)
elif kind == "index_local":
stage = parent / "arbitrary-directory"
stage.mkdir(exist_ok=True)
(stage / "index_metadata.json").write_text("{}")
elif kind == "legacy_alias":
stage = parent / "train_tables" / ("base-alias-" + entrypoint)
stage.symlink_to(parent, target_is_directory=True)
match = "base index" if kind in ("base", "base_alias", "legacy_alias") else "extension"
if kind == "missing":
match = "extension `stage`"
with pytest.raises(diskgraph.DiskgraphSystemError, match=match):
if entrypoint == "native":
from diskgraph._c import RuntimeIndex as NativeRuntimeIndex

ref = str(stage.relative_to(parent)) if kind == "legacy_alias" else str(stage)
NativeRuntimeIndex(str(parent), ref, 17, 1)
else:
manifest = json.loads(pathlib.Path(seed_table_fixture["manifest_path"]).read_text())
manifest["stages"]["train"]["index_path"] = str(stage)
path = tmp_path / "manifest.json"
path.write_text(json.dumps(manifest))
make_runtime_batch_iterator(
str(path), stage="train", entity_node_type="users", sampling_plan=_sampling_plan(),
num_workers=1, worker_id=0, shuffle=False, rnd_seed=17, epoch=0,
batch_size=3, drop_last=False, num_threads=1,
)


def test_external_extension_metadata_is_validated_by_native_reader(seed_table_fixture, tmp_path):
external = tmp_path / "not-an-index"
external.mkdir()
(external / "index_metadata.json").write_text("{}")
manifest = json.loads(pathlib.Path(seed_table_fixture["manifest_path"]).read_text())
manifest["stages"]["train"]["index_path"] = str(external)
path = tmp_path / "manifest.json"
path.write_text(json.dumps(manifest))
with pytest.raises(diskgraph.DiskgraphSystemError, match="extension"):
make_runtime_batch_iterator(
str(path), stage="train", entity_node_type="users", sampling_plan=_sampling_plan(),
num_workers=1, worker_id=0, shuffle=False, rnd_seed=17, epoch=0,
batch_size=3, drop_last=False, num_threads=1,
)


def test_static_task_manifest_suppresses_seed_timestamps(
seed_table_fixture, tmp_path):
"""Static task datasets may carry pseudo TIMESTAMP columns, but those
Expand Down
18 changes: 13 additions & 5 deletions packages/diskgraph/diskgraph-pybind/src/cutoff_col_stats.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ const TOKENIZER_TRAINER: &str = "tokenizers 0.23.2 onig";
type WordCounts = AHashMap<CompactString, u64>;

/// Recomputes `request` from an index's stored rows.
fn from_index(index_dir: &Path, request: &CutoffColStatsRequest) -> Result<()> {
fn from_index(index_dir: &Path, output_dir: &Path, request: &CutoffColStatsRequest) -> Result<()> {
let config = read_persisted_config(index_dir)?;
request.validate(&config.tables)?;
let names: HashSet<String> = request.columns.keys().cloned().collect();
Expand Down Expand Up @@ -107,25 +107,33 @@ fn from_index(index_dir: &Path, request: &CutoffColStatsRequest) -> Result<()> {
}
tables.push(table);
}
request.publish(index_dir, clock, tables)
request.publish(output_dir, clock, tables)
}

/// Recomputes column statistics at cutoffs from an index's stored rows, into `col_stats/<dir>/`.
/// Recomputes statistics from stored rows into `output_dir/col_stats/<dir>/`.
/// The output defaults to the index directory for existing callers.
#[pyfunction]
#[pyo3(signature = (index_dir, request_json, num_threads=None))]
#[pyo3(signature = (index_dir, request_json, num_threads=None, *, output_dir=None))]
pub fn cutoff_col_stats(
py: Python<'_>,
index_dir: &str,
request_json: &str,
num_threads: Option<usize>,
output_dir: Option<&str>,
) -> PyResult<()> {
let request: CutoffColStatsRequest = serde_json::from_str(request_json)
.map_err(|e| DiskgraphConfigError::new_err(format!("invalid request_json: {e}")))?;
py.detach(|| {
let pool = rayon::ThreadPoolBuilder::new()
.num_threads(num_threads.unwrap_or(0))
.build()?;
pool.install(|| from_index(Path::new(index_dir), &request))
pool.install(|| {
from_index(
Path::new(index_dir),
Path::new(output_dir.unwrap_or(index_dir)),
&request,
)
})
})
.map_err(map_anyhow_err)
}
Expand Down
23 changes: 19 additions & 4 deletions packages/diskgraph/diskgraph-runtime/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1253,17 +1253,32 @@ pub struct RuntimeIndex {
namespace: String,
}
impl RuntimeIndex {
/// Open the graph and training-table extension; propagate typed index errors.
/// Open the graph and a seed extension, by legacy reference or absolute path.
/// The shared index reader validates relative and absolute extension paths,
/// including rejecting the parent itself and aliases of it.
pub fn new(
parent_root: String,
tt_id: String,
rnd_seed: Option<u64>,
num_threads: Option<usize>,
) -> Result<Self> {
let parent_path = std::path::Path::new(&parent_root);
let reader = diskgraph_index::graph::GraphIndexReader::open_with_tt(parent_path, &tt_id)
.map_err(|e| RuntimeError::system(e.to_string()))?;
let namespace = tt_id.rsplit('/').next().unwrap_or(&tt_id).to_string();
let extension = std::path::Path::new(&tt_id);
let namespace = extension
.file_name()
.and_then(|n| n.to_str())
.ok_or_else(|| RuntimeError::system("invalid extension path".to_string()))?
.to_string();
let reader = if extension.is_absolute() {
diskgraph_index::graph::GraphIndexReader::open_with_extension_subdir(
parent_path,
&tt_id,
&namespace,
)
} else {
diskgraph_index::graph::GraphIndexReader::open_with_tt(parent_path, &tt_id)
}
.map_err(|e| RuntimeError::system(e.to_string()))?;
let reader = Arc::new(reader);
let engine = Arc::new(
DiskGraphEngine::from_reader(reader.clone(), rnd_seed, num_threads)
Expand Down
Loading