From 5f7a8e5c61de17034774fbe851c6e6102f51d257 Mon Sep 17 00:00:00 2001 From: Yzabelle Go Date: Tue, 10 Nov 2026 15:37:42 -0500 Subject: [PATCH 1/3] feat(store): configured namespaces with stores of their own An @/ id in a namespace configured under [namespaces.] (document and object tables) keeps to that namespace's stores: its entities by id, what a run of its tasks records, and every entity and object that run writes, objects under a / key prefix. Reads no id places see the configured store and the running task's namespace, plus only the namespaces reading_namespaces() opens, so a service decides who reads which. A shared entity may not name a namespace's ids, a run never writes a new version of a shared entity, and a run's objects never go to the shared store. Routing is inert with no [namespaces] table. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/config/describe.py | 3 +- src/agent_env/config/runtime.py | 128 +++++++- src/agent_env/env/snapshot_store.py | 5 +- src/agent_env/store/ids.py | 57 +++- src/agent_env/store/routing.py | 307 ++++++++++++++++++- tst/unit/config/test_describe.py | 10 +- tst/unit/store/configured_namespace_test.py | 321 ++++++++++++++++++++ tst/unit/store/ids_test.py | 8 +- 8 files changed, 797 insertions(+), 42 deletions(-) create mode 100644 tst/unit/store/configured_namespace_test.py diff --git a/src/agent_env/config/describe.py b/src/agent_env/config/describe.py index e8e291d7f..88c3a42fc 100644 --- a/src/agent_env/config/describe.py +++ b/src/agent_env/config/describe.py @@ -269,7 +269,7 @@ class FileSection: name: str toml_path: tuple[str, ...] owner: str - keys: Mapping[str, Shape] + keys: Shape check: Optional[Callable[[Mapping[str, Any]], Any]] = None @@ -283,6 +283,7 @@ class FileSection: FileSection("envs", ("envs",), "env.registry", {"impls": None, **_resolved_keys("envs")}), FileSection("artifacts", ("artifacts",), "artifact.registry", {"impls": None, "type_aliases": None}), FileSection("task_steps", ("task_steps",), "task_step.registry", {"impls": None}), + FileSection("namespaces", ("namespaces",), "config.runtime", Named({"document": _SEAM, "object": _SEAM})), FileSection("explorer", ("explorer",), "explorer.app", {**dict.fromkeys(("port", "cors_origins", "allowed_hosts", "static_dir")), "plugins": {"impls": None}}), diff --git a/src/agent_env/config/runtime.py b/src/agent_env/config/runtime.py index a98b17005..060f2a233 100644 --- a/src/agent_env/config/runtime.py +++ b/src/agent_env/config/runtime.py @@ -254,6 +254,8 @@ class Config: _routed_stores: dict[str, Any] = field(default_factory=dict, repr=False, compare=False) _separate_stores: dict[str, tuple[Any, Any]] = field(default_factory=dict, repr=False, compare=False) _routing_lock: Any = field(default_factory=threading.RLock, repr=False, compare=False) + _namespace_stores_built: dict[str, tuple[Any, Any]] = field(default_factory=dict, repr=False, compare=False) + _namespaced: dict[str, tuple[Any, Any]] = field(default_factory=dict, repr=False, compare=False) _secret_store: Optional[SecretStore] = field(default=None, repr=False, compare=False) _secret_store_lock: Any = field(default_factory=threading.Lock, repr=False, compare=False) _model_cfg: Optional[model_config.ModelConfig] = field(default=None, repr=False, compare=False) @@ -501,13 +503,83 @@ def get_document_store(self) -> "DocumentStore": """The control-plane document store (cached): an explicitly-set backend, else the one from ``AGENT_ENV_DOCUMENT_STORE`` / config.toml (``local`` → SQLite default). With namespace routing on, a local namespace's documents and its runs' records go to that namespace's - own local store (``agent_env.store.routing``).""" + own local store (``agent_env.store.routing``). A configured namespace's documents always go to its own.""" from agent_env.store import routing configured = self._configured_document_store() + shared = self._with_namespaces(configured) if not routing.namespace_routing_enabled(): + return shared + return self._routed("document", configured, shared) + + def configured_namespaces(self) -> tuple[str, ...]: + """The namespaces ``[namespaces.]`` configures, each with document and object stores of its own.""" + from agent_env.store.ids import validate_namespace_name + + table = self.section("namespaces") + for name, entry in table.items(): + try: + validate_namespace_name(name) + except ValueError as e: + raise ConfigError(f"[namespaces.{name}]: {e}") from e + if not isinstance(entry, Mapping) or not all(isinstance(entry.get(kind), Mapping) for kind in ("document", "object")): + raise ConfigError(f"[namespaces.{name}] needs a document and an object table, each with an impl") + return tuple(table) + + def namespace_document_store(self, namespace: str) -> "DocumentStore": + """The configured ``namespace``'s own document store, unrouted.""" + return self._namespace_store(namespace)[0] + + def namespace_object_store(self, namespace: str) -> ObjectStore: + """The configured ``namespace``'s own object store: its table's, with every key under ``/``.""" + return self._namespace_store(namespace)[1] + + def namespace_of(self, entity_id: str) -> Optional[str]: + """The configured namespace ``entity_id`` is in, or None.""" + from agent_env.store.ids import parse_namespace + + namespace = parse_namespace(entity_id) + return namespace if namespace is not None and namespace in self.configured_namespaces() else None + + def _namespace_store(self, namespace: str) -> tuple[Any, Any]: + from agent_env.store.document_store import DocumentStore + from agent_env.store.object_store import ObjectStore + from agent_env.store.routing import PrefixedObjectStore + + if namespace not in self.configured_namespaces(): + raise ConfigError(f"no [namespaces.{namespace}] is configured") + with self._routing_lock: + if namespace not in self._namespace_stores_built: + entry = self.section("namespaces", namespace) + documents = config_loader.build_store(dict(entry["document"]), DocumentStore, + secret_resolver=self._resolve_secret) + objects = config_loader.build_store(dict(entry["object"]), ObjectStore, secret_resolver=self._resolve_secret) + self._namespace_stores_built[namespace] = (documents, PrefixedObjectStore(objects, f"{namespace}/")) + return self._namespace_stores_built[namespace] + + def _with_namespaces(self, configured: "DocumentStore") -> "DocumentStore": + """``configured`` routed beside every configured namespace's document store, or itself when there are none.""" + from agent_env.store import routing + + names = self.configured_namespaces() + if not names: return configured - return self._routed("document", configured) + with self._routing_lock: + held = self._namespaced.get("document") + if held is None or held[0] is not configured: + held = (configured, routing.NamespaceDocumentStore( + configured, {name: self.namespace_document_store(name) for name in names})) + self._namespaced["document"] = held + return held[1] + + def _run_namespace(self) -> Optional[str]: + from agent_env.store import routing + + namespace = routing.run_namespace() + return namespace if namespace is not None and namespace in self.configured_namespaces() else None + + def _namespace_owning(self, object_url: str) -> Optional[str]: + return next((name for name in self.configured_namespaces() if self.namespace_object_store(name).owns(object_url)), None) def _configured_document_store(self) -> "DocumentStore": if self._document_store is None: @@ -670,20 +742,38 @@ def config_file(self) -> dict: def get_object_store(self) -> ObjectStore: """The object (blob) store (cached): an explicitly-set backend, else the one from ``AGENT_ENV_OBJECT_STORE`` / config.toml (``local`` → filesystem default). While an ``@local`` - task runs under namespace routing, its view of it keeps writes local.""" + task runs under namespace routing, its view of it keeps writes local; while a configured namespace's task + runs, its view writes to that namespace's store.""" from agent_env.store import routing configured = self._configured_object_store() + if (namespace := self._run_namespace()) is not None: + return self._namespace_run_object_store(namespace, configured) if not routing.in_local_run(): return configured return self._routed("object", configured) + def _namespace_run_object_store(self, namespace: str, configured: ObjectStore) -> ObjectStore: + from agent_env.store import routing + + with self._routing_lock: + held = self._namespaced.get(f"object:{namespace}") + if held is None or held[0] is not configured: + others = {name: self.namespace_object_store(name) for name in self.configured_namespaces() if name != namespace} + held = (configured, routing.NamespaceRunObjectStore( + configured, self.namespace_object_store(namespace), namespace, others)) + self._namespaced[f"object:{namespace}"] = held + return held[1] + def get_object_store_for(self, entity_id: str) -> ObjectStore: - """The object store ``entity_id``'s objects are written to: under namespace routing, the - per-user local one for an ``@local`` id; otherwise the configured one.""" + """The object store ``entity_id``'s objects are written to: a configured namespace's own for an id in it, or + for any entity a run in that namespace writes; under namespace routing, the per-user local one for an + ``@local`` id; otherwise the configured one.""" from agent_env.store import routing from agent_env.store.ids import is_local_id + if (namespace := self.namespace_of(entity_id) or self._run_namespace()) is not None: + return self.namespace_object_store(namespace) configured, local = self._namespace_stores("object") if local is None: return configured @@ -694,9 +784,11 @@ def get_object_store_for(self, entity_id: str) -> ObjectStore: def get_object_store_at(self, object_url: str) -> ObjectStore: """The object store ``object_url`` addresses: under namespace routing a url the local store - owns is the local store's, unless the configured store owns it too.""" + owns is the local store's, unless the configured store owns it too; a configured namespace's url is its store's.""" from agent_env.store import routing + if (namespace := self._namespace_owning(object_url)) is not None: + return self.namespace_object_store(namespace) configured, local = self._namespace_stores("object") return configured if local is None else routing.object_store_holding(object_url, configured, local) @@ -708,10 +800,19 @@ def get_object_store_to_write(self, object_url: str, entity_id: str) -> ObjectSt def check_object_url(self, entity_id: str, object_url: str) -> None: """Refuse recording ``object_url`` as ``entity_id``'s object across namespaces: under namespace routing an ``@local`` entity's objects live in the local store, a bare one's in the configured - store, and an ``@local`` task run writes only locally.""" + store, and an ``@local`` task run writes only locally. A configured namespace's entity, or any a run in it + writes, keeps its objects in that namespace's store, and nothing else records one there.""" from agent_env.store import routing from agent_env.store.ids import is_local_id + namespace = self.namespace_of(entity_id) or self._run_namespace() + owner = self._namespace_owning(object_url) + if namespace != owner and namespace is not None: + raise ValueError(f"{entity_id!r} keeps its objects in the @{namespace} namespace's store, not {object_url!r}") + if namespace != owner: + raise ValueError(f"{entity_id!r} is in no configured namespace, so it can't record {object_url!r}, an @{owner} object") + if namespace is not None: + return configured, local = self._namespace_stores("object") if local is None: return @@ -903,23 +1004,24 @@ def _compare_local_store(self, kind: str, configured: Any) -> Any: ) return None if same else local - def _routed(self, kind: str, configured: Any) -> Any: - """``configured`` wrapped by namespace routing, or ``configured`` itself when it already is - the local namespaces' store.""" + def _routed(self, kind: str, configured: Any, shown: Any = None) -> Any: + """``shown`` (``configured`` unless given) wrapped by namespace routing, or ``shown`` itself when + ``configured`` already is the local namespaces' store.""" from agent_env.store import routing + shown = configured if shown is None else shown local = self._separate_local_store(kind, configured) if local is None: if kind == "document": namespace, kept = next((namespace, store) for namespace, store in self._namespace_document_stores().items() if store.path.resolve() == configured.path.resolve()) raise ConfigError(f"{kept.path} is kept for @{namespace} documents; point [stores.document] somewhere else") - return configured + return shown with self._routing_lock: routed = self._routed_stores.get(kind) - if routed is None or routed.configured is not configured: + if routed is None or routed.configured is not shown: if kind == "document": - routed = routing.RoutingDocumentStore(configured, self._namespace_document_stores()) + routed = routing.RoutingDocumentStore(shown, self._namespace_document_stores()) elif kind == "object": routed = routing.LocalRunObjectStore(configured, local) else: diff --git a/src/agent_env/env/snapshot_store.py b/src/agent_env/env/snapshot_store.py index 279abdd76..ac4198ba6 100644 --- a/src/agent_env/env/snapshot_store.py +++ b/src/agent_env/env/snapshot_store.py @@ -112,10 +112,11 @@ async def create(cls, instance_id: str, on_progress: ProgressCallback | None = N raise ValueError(f"Instance '{instance_id}' has no service_universe loaded") universe_id = environment_universe["id"] universe_version = environment_universe["version"] - if is_local_id(env_id) != is_local_id(universe_id): + if is_local_id(env_id) != is_local_id(universe_id) or ( + get_config().namespace_of(env_id) != get_config().namespace_of(universe_id)): raise ValueError( f"env {env_id!r} and universe {universe_id!r} are in different namespaces; a snapshot is recorded " - "under both, so both have to be @local ids or neither" + "under both, so both have to be in the same one" ) log("reconnect", f"Reconnecting to sandbox {deployed_env.sandbox_id}...", 10) multi_env = await MultiEnv.from_deployed_env(deployed_env) diff --git a/src/agent_env/store/ids.py b/src/agent_env/store/ids.py index fdf721f54..2391f17da 100644 --- a/src/agent_env/store/ids.py +++ b/src/agent_env/store/ids.py @@ -1,10 +1,11 @@ """Namespaced entity ids, and the object keys, image repositories and filenames derived from them. An id opening ``@/`` names the store it lives in. The local namespaces, ``@local/…`` and ``@hf/…`` -(a bundle ``agent-env run hf://`` downloaded from the Hugging Face Hub), are this machine's local stores; no other -exists today. Documents keep an id verbatim. Everything derived from a local id uses one bounded -segment, ``local/-``; an object key or image repository takes a bare id over -200 bytes as ``-``; every other id passes through byte-identical. +(a bundle ``agent-env run hf://`` downloaded from the Hugging Face Hub), are this machine's local stores; a +configured namespace (``[namespaces.]``) has stores of its own beside the configured ones. Documents keep an +id verbatim. Everything derived from a local id uses one bounded segment, ``local/-``, and +from another namespace's id ``/-``; an object key or image repository takes a +bare id over 200 bytes as ``-``; every other id passes through byte-identical. The encoders never raise: they expect ids that ``validate_local_id`` accepts, and still return a safe segment for any other input. An id may contain spaces, ``&`` and parentheses, so quote one before it reaches a shell or a URL, and derive object keys, image repositories and filenames with @@ -33,6 +34,7 @@ _ENCODED_LOCAL_ID = re.compile( rf"{LOCAL_NAMESPACE}(?:/|-(?:[a-z0-9]+(?:-[a-z0-9]+)*-)?[0-9a-f]{{{_HASH_LENGTH}}}(?:/|$))" ) +_NAMESPACE_NAME = re.compile(r"[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?") _ID_PUNCTUATION = frozenset(" ._-~/@()+,&") _ID_CATEGORIES = frozenset("LNM") _INVISIBLE = re.compile( @@ -73,23 +75,43 @@ def validate_local_id(entity_id: str) -> None: namespace = local_namespace(entity_id) if namespace is None: raise ValueError(f"local id {entity_id!r} must start with {LOCAL_PREFIX!r} or {HF_PREFIX!r}") + _validate_in(entity_id, namespace, "local id") + + +def validate_namespace_name(namespace: str) -> None: + """Raise ValueError unless ``namespace`` can name a configured namespace: 1 to 64 lowercase letters, digits and + inner ``-``, and not a local namespace.""" + if namespace in LOCAL_NAMESPACES: + raise ValueError(f"@{namespace} is a local namespace, so it can't be configured") + if not isinstance(namespace, str) or not _NAMESPACE_NAME.fullmatch(namespace): + raise ValueError(f"namespace {namespace!r} must be 1 to 64 lowercase letters, digits and inner '-'") + + +def validate_namespaced_id(entity_id: str, namespace: str) -> None: + """``validate_local_id``'s rules for an id in the configured ``namespace``.""" + if not entity_id.startswith(namespace_prefix(namespace)): + raise ValueError(f"id {entity_id!r} must start with {namespace_prefix(namespace)!r}") + _validate_in(entity_id, namespace, f"@{namespace} id") + + +def _validate_in(entity_id: str, namespace: str, what: str) -> None: prefix = namespace_prefix(namespace) size = len(entity_id.encode("utf-8", "surrogatepass")) if size > MAX_LOCAL_ID_BYTES: - raise ValueError(f"local id is {size} bytes; the limit is {MAX_LOCAL_ID_BYTES}: {entity_id[:64]!r}…") + raise ValueError(f"{what} is {size} bytes; the limit is {MAX_LOCAL_ID_BYTES}: {entity_id[:64]!r}…") disallowed = sorted({c for c in entity_id[len(prefix):] if not _allowed_in_id(c)}) if disallowed: raise ValueError( - f"local id {entity_id!r} contains {ascii(''.join(disallowed))}; " - "a local id may contain only letters, digits, spaces and . _ - ~ / @ ( ) + , &" + f"{what} {entity_id!r} contains {ascii(''.join(disallowed))}; " + f"a {what} may contain only letters, digits, spaces and . _ - ~ / @ ( ) + , &" ) segments = entity_id[len(prefix):].split("/") if any(segment in ("", ".", "..") for segment in segments): - raise ValueError(f"local id {entity_id!r} has an empty, '.' or '..' path segment") + raise ValueError(f"{what} {entity_id!r} has an empty, '.' or '..' path segment") if any(segment != segment.strip(" ") for segment in segments): - raise ValueError(f"local id {entity_id!r} has a path segment that opens or closes with a space") + raise ValueError(f"{what} {entity_id!r} has a path segment that opens or closes with a space") if any(_detached_mark(segment) for segment in segments): - raise ValueError(f"local id {entity_id!r} has a combining mark that does not follow a letter or digit") + raise ValueError(f"{what} {entity_id!r} has a combining mark that does not follow a letter or digit") def _allowed_in_id(character: str) -> bool: @@ -103,12 +125,12 @@ def _detached_mark(segment: str) -> bool: return any(this == "M" and before not in _ID_CATEGORIES for before, this in zip(["Z", *categories], categories)) -def _local_segment(entity_id: str) -> str: +def _local_segment(entity_id: str, namespace: str = LOCAL_NAMESPACE) -> str: digest = hashlib.sha256(entity_id.encode("utf-8", "surrogatepass")).hexdigest()[:_HASH_LENGTH] - decomposed = unicodedata.normalize("NFKD", entity_id.removeprefix(LOCAL_PREFIX)) + decomposed = unicodedata.normalize("NFKD", entity_id.removeprefix(namespace_prefix(namespace))) folded = "".join(c for c in decomposed if not unicodedata.combining(c)).casefold().replace("~", "") slug = _NON_SLUG_RUN.sub("-", folded).strip("-")[:_SLUG_LENGTH].rstrip("-") - return f"{LOCAL_NAMESPACE}/{slug}-{digest}" if slug else f"{LOCAL_NAMESPACE}/{digest}" + return f"{namespace}/{slug}-{digest}" if slug else f"{namespace}/{digest}" def aliases_local_encoding(entity_id: str) -> bool: @@ -117,10 +139,19 @@ def aliases_local_encoding(entity_id: str) -> bool: return _ENCODED_LOCAL_ID.match(entity_id) is not None +def _namespaced(entity_id: str) -> bool: + """Whether ``entity_id`` is ``@/…`` for a name a namespace could be configured under.""" + namespace = parse_namespace(entity_id) + return (namespace is not None and namespace not in LOCAL_NAMESPACES and "/" in entity_id + and _NAMESPACE_NAME.fullmatch(namespace) is not None) + + def key_segment(entity_id: str) -> str: """The object-key segment for ``entity_id`` (``artifacts////…``).""" if is_local_id(entity_id): return _local_segment(entity_id) + if _namespaced(entity_id): + return _local_segment(entity_id, parse_namespace(entity_id)) if len(entity_id.encode("utf-8", "surrogatepass")) > MAX_BARE_SEGMENT_BYTES: return _local_segment(entity_id).removeprefix(f"{LOCAL_NAMESPACE}/") return entity_id diff --git a/src/agent_env/store/routing.py b/src/agent_env/store/routing.py index 4f917e504..f9195d522 100644 --- a/src/agent_env/store/routing.py +++ b/src/agent_env/store/routing.py @@ -14,6 +14,11 @@ else sees the registry's first. While a local task runs, a write that would reach a configured store is refused (the write fence), and a bare entity may never name a local id (references point outward only). + +A configured namespace (``[namespaces.]``) is routed whether or not local routing is on: its ids, what a run +of its tasks records and every entity and object that run writes go to that namespace's own document and object +stores. A read no id places sees the configured store and the running task's namespace, plus only the namespaces +``reading_namespaces`` opens, so a service decides who reads which. Nothing outside a namespace may name its ids. """ from __future__ import annotations @@ -48,6 +53,7 @@ namespace_prefix, parse_namespace, validate_local_id, + validate_namespaced_id, ) from agent_env.store.image_store import ImageStore, OciRegistryImageStore from agent_env.store.image_store.oci_registry_credentials import ( @@ -66,6 +72,8 @@ # The fields a run-time document names its owner by: one holding a local id places it in that id's namespace. _OWNER_FIELDS = ("id", "instance_id", "task_id", "task_instance_id", "env_id", "agent_id") +# A record about a universe, a snapshot's say, is a namespace's when its universe is. +_NAMESPACE_OWNER_FIELDS = (*_OWNER_FIELDS, "service_universe_id", "environment_universe_id", "universe_id") _LOCAL_REPOSITORY_PREFIX = f"{LOCAL_NAMESPACE}/" _TIMESTAMP = re.compile(r"\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}") _T = TypeVar("_T") @@ -74,6 +82,7 @@ _scopes = 0 _scopes_lock = threading.Lock() _run_namespace: ContextVar[Optional[str]] = ContextVar("agent_env_run_namespace", default=None) +_readable: ContextVar[frozenset[str]] = ContextVar("agent_env_readable_namespaces", default=frozenset()) class LocalRunWriteError(ValueError): @@ -84,6 +93,10 @@ class OutwardReferenceError(ValueError): """A write that would point outside the local namespaces' stores at a local id.""" +class NamespaceWriteError(ValueError): + """A write that would carry a configured namespace's data outside its stores.""" + + def enable_namespace_routing() -> None: """Route local ids to the per-user local stores in this process. The CLI turns this on; a service never does, so its stores stay exactly the configured ones.""" @@ -117,10 +130,10 @@ def namespace_routing() -> Iterator[None]: @contextmanager def run_scope(owner_id: str) -> Iterator[None]: """The run in progress, a task's or a deployment the CLI makes: what it records follows its owner id's - namespace. A run started inside a local run stays in that run's namespace unless its own id is local, so a - nested registry task can't reopen the configured stores.""" + namespace. A run started inside a run in a namespace stays in that run's namespace unless its own id is local, + so a nested registry task can't reopen the configured stores.""" outer = _run_namespace.get() - inherited = outer if outer in LOCAL_NAMESPACES else parse_namespace(owner_id) or "" + inherited = outer or parse_namespace(owner_id) or "" token = _run_namespace.set(local_namespace(owner_id) or inherited) try: yield @@ -138,6 +151,26 @@ def in_local_run() -> bool: return local_run_namespace() is not None +def run_namespace() -> Optional[str]: + """The namespace of the task running, local or configured; None outside a run or in a run of a bare id.""" + return _run_namespace.get() or None + + +@contextmanager +def reading_namespaces(*namespaces: str) -> Iterator[None]: + """Let reads no id places also see these configured namespaces' stores for the length of the block, as a service + does for a caller it has checked may read them. A run always sees its own namespace's.""" + token = _readable.set(_readable.get() | frozenset(namespaces)) + try: + yield + finally: + _readable.reset(token) + + +def readable_namespaces() -> frozenset[str]: + return _readable.get() + + def refuse_in_local_run(what: str) -> None: if (namespace := local_run_namespace()) is not None: raise LocalRunWriteError( @@ -148,23 +181,24 @@ def refuse_in_local_run(what: str) -> None: def refuse_local_references(entity_id: object, value: Any) -> None: """Refuse a bare entity naming a local id anywhere in ``value``: a shared store must never point into one person's local stores.""" - path = _local_reference(value, "") + path = _reference(value, "", is_local_id) if path is not None: raise OutwardReferenceError( f"{entity_id!r} is not a local id, so it can't reference one: {path or 'its value'} names a local id" ) -def _local_reference(value: Any, path: str) -> Optional[str]: +def _reference(value: Any, path: str, matches: Callable[[str], bool]) -> Optional[str]: + """The path in ``value`` of the first string ``matches`` accepts, ``""`` for ``value`` itself; None when none does.""" if isinstance(value, str): - return path if is_local_id(value) else None + return path if matches(value) else None if isinstance(value, Mapping): items = ((f"{path}.{key}" if path else str(key), item) for key, item in value.items()) elif isinstance(value, (list, tuple, set)): items = ((f"{path}[{index}]", item) for index, item in enumerate(value)) else: return None - return next((found for item_path, item in items if (found := _local_reference(item, item_path)) is not None), None) + return next((found for item_path, item in items if (found := _reference(item, item_path, matches)) is not None), None) def _update_values(update: UpdateSpec) -> dict: @@ -342,7 +376,7 @@ def latest(store: DocumentStore, **page: Any) -> list[dict]: readers = self._readers(collection, filter) if len(readers) == 1: return self._read(readers[0], lambda: latest(readers[0], limit=limit, offset=offset)) - if _ids_split_by_namespace(collection, id_field): + if self._ids_split(collection, id_field): window = (offset or 0) + limit if limit else None merged = [doc for store in readers for doc in self._read(store, lambda: latest(store, limit=window))] else: @@ -369,7 +403,7 @@ def latest_per_id_page( sort=sort, limit=limit, offset=offset, ) ) - if _ids_split_by_namespace(collection, id_field): + if self._ids_split(collection, id_field): window = (offset or 0) + limit if limit else None pages_and_totals = [ self._read(store, lambda: store.latest_per_id_page( @@ -389,7 +423,7 @@ def latest_per_id_page( def count_distinct(self, collection: str, filter: Filter, *, id_field: str = "id") -> int: readers = self._readers(collection, filter) - if len(readers) == 1 or _ids_split_by_namespace(collection, id_field): + if len(readers) == 1 or self._ids_split(collection, id_field): return sum( self._read(store, lambda: store.count_distinct(collection, filter, id_field=id_field)) for store in readers ) @@ -485,6 +519,9 @@ def check_id(self, entity_id: str) -> None: ) self.configured.check_id(entity_id) + def _ids_split(self, collection: str, id_field: str) -> bool: + return _ids_split_by_namespace(collection, id_field) + def _placed(self, collection: str, filter: Filter) -> Optional[str]: """The local namespace whose store holds ``filter``'s match, ``""`` for the configured store; None when it can be in any.""" @@ -567,6 +604,129 @@ def _open(self, namespace: str) -> LocalNamespaceDocumentStore: return store +class NamespaceDocumentStore(RoutingDocumentStore): + """The configured document store with each configured namespace's documents in that namespace's own store, + ``namespaces`` by name: an entity by its id; a run record by the running task's namespace, else by an owner field; + and every entity a run in a namespace writes. A read no id places sees the running task's namespace, the configured + store and the namespaces ``reading_namespaces`` opens. A run in a namespace never writes a new version of a + configured entity, so no entity id is in two stores.""" + + def __init__(self, configured: DocumentStore, namespaces: Mapping[str, DocumentStore]) -> None: + self.configured = configured + self.namespaces = dict(namespaces) + self._lock = threading.Lock() + + def ensure_index( + self, + collection: str, + fields: list[str], + unique: bool = False, + ttl_seconds: Optional[int] = None, + ) -> None: + for store in (self.configured, *self.namespaces.values()): + store.ensure_index(collection, fields, unique, ttl_seconds) + + def check_id(self, entity_id: str) -> None: + namespace = self._namespace_of(entity_id) + if namespace is not None: + validate_namespaced_id(entity_id, namespace) + return + if (inside := next((name for name in self.namespaces if namespace_prefix(name) in entity_id), None)): + raise ValueError( + f"id {entity_id!r} contains an @{inside} id but isn't one: an id derived from an @{inside} id " + f"has to keep the @{inside} namespace" + ) + self.configured.check_id(entity_id) + + def _namespace_of(self, value: object) -> Optional[str]: + namespace = parse_namespace(value) if isinstance(value, str) else None + return namespace if namespace in self.namespaces else None + + def _run(self) -> Optional[str]: + namespace = run_namespace() + return namespace if namespace in self.namespaces else None + + def _pinned(self, collection: str, filter: Filter) -> Optional[str]: + """The one namespace every id the filter pins is in: by ``id`` for an entity, by an owner field for a record.""" + fields = ("id",) if is_entity_collection(collection) else _NAMESPACE_OWNER_FIELDS + for field in fields: + values: list[Any] = [] + for pred in filter.conditions.get(field, ()): + if isinstance(pred, Eq): + values.append(pred.value) + elif isinstance(pred, In): + values.extend(pred.values) + namespaces = {self._namespace_of(value) for value in values} + if values and len(namespaces) == 1 and (namespace := namespaces.pop()) is not None: + return namespace + return None + + def _visible(self) -> list[DocumentStore]: + """The stores an unplaced read sees, the running task's namespace first.""" + run = self._run() + opened = sorted(name for name in readable_namespaces() if name in self.namespaces and name != run) + return [*([self.namespaces[run]] if run else []), self.configured, *(self.namespaces[name] for name in opened)] + + def _readers(self, collection: str, filter: Filter) -> list[DocumentStore]: + pinned = self._pinned(collection, filter) + return [self.namespaces[pinned]] if pinned is not None else self._visible() + + def _read(self, store: DocumentStore, read: Callable[[], _T]) -> _T: + return read() + + def _writer(self, collection: str, owner: Mapping, payload: Any) -> DocumentStore: + run = self._run() + entity_id = owner.get("id") + if is_entity_collection(collection): + if isinstance(entity_id, str): + self.check_id(entity_id) + namespace = self._namespace_of(entity_id) or run + if run is not None and namespace != run: + raise NamespaceWriteError(f"a run in @{run} can't write {entity_id!r}, an @{namespace} entity") + if (run is not None and self._namespace_of(entity_id) is None and isinstance(entity_id, str) + and self.configured.find_one(collection, Filter.of(id=entity_id)) is not None): + raise NamespaceWriteError( + f"a run in @{run} can't write a new version of {entity_id!r}: it is in the configured store, " + f"which a run in @{run} writes no new entities to" + ) + self._refuse_references(entity_id, payload, namespace) + else: + namespace = run or next( + (found for field in _NAMESPACE_OWNER_FIELDS if (found := self._namespace_of(owner.get(field))) is not None), None) + return self.namespaces[namespace] if namespace is not None else self.configured + + def _updaters(self, collection: str, filter: Filter, payload: Any) -> Iterator[DocumentStore]: + pinned = self._pinned(collection, filter) + run = self._run() + if pinned is not None: + if run is not None and pinned != run: + raise NamespaceWriteError(f"a run in @{run} can't write @{pinned} records") + yield self.namespaces[pinned] + return + if run is not None and not is_entity_collection(collection): + yield self.namespaces[run] + if self.configured.find_one(collection, filter) is not None: + raise NamespaceWriteError(f"a run in @{run} can't update {collection!r}: its record is in the configured store") + return + for store in self._visible(): + if store is self.configured and is_entity_collection(collection): + self._refuse_references(_filter_owner(filter).get("id"), payload, None) + yield store + + def _refuse_references(self, entity_id: object, payload: Any, namespace: Optional[str]) -> None: + """Refuse a document naming an id in a namespace other than ``namespace``, its own (None for the configured + store): a namespace's ids never leave it.""" + if payload is None: + return + others = tuple(namespace_prefix(name) for name in self.namespaces if name != namespace) + path = _reference(payload, "", lambda value: value.startswith(others)) if others else None + if path is not None: + where = f"@{namespace}" if namespace else "the configured store" + raise NamespaceWriteError( + f"{entity_id!r} is in {where}, so it can't reference another namespace's id: {path or 'its value'} does" + ) + + def object_store_holding(object_url: str, configured: ObjectStore, local: ObjectStore) -> ObjectStore: """The store an object_url addresses: the local store's own urls are local unless the configured store owns them too, and any other url is the configured store's, as it was before routing.""" @@ -576,7 +736,9 @@ def object_store_holding(object_url: str, configured: ObjectStore, local: Object def configured_store(store: Any) -> Any: """The configured store behind ``store``: a routing store's or view's configured side, else ``store`` itself. For reports that name the backend a user configured.""" - return store.configured if isinstance(store, (RoutingDocumentStore, LocalRunObjectStore, LocalRunImageStore)) else store + while isinstance(store, (RoutingDocumentStore, LocalRunObjectStore, LocalRunImageStore)): + store = store.configured + return store def image_store_holding(ref: str, configured: ImageStore, local: ImageStore) -> ImageStore: @@ -716,3 +878,126 @@ def _lifetime(expires_in: int | None) -> dict[str, int]: """``expires_in`` as a keyword only when the caller named one, so the owning store applies its own default, whatever it is.""" return {} if expires_in is None else {"expires_in": expires_in} + + +class PrefixedObjectStore(ObjectStore): + """``inner`` with every key this store makes under ``prefix``: a configured namespace's objects keep to its own + prefix even when namespaces share a bucket, and ``owns`` a url only under it. Explicit-url reads reach as far as + ``inner``'s do.""" + + def __init__(self, inner: ObjectStore, prefix: str) -> None: + if not prefix.endswith("/"): + raise ValueError(f"an object key prefix ends with '/': {prefix!r}") + self.inner = inner + self.prefix = prefix + + @property + def supports_transfer_grants(self) -> bool: + return self.inner.supports_transfer_grants + + def grants_reach(self, sandbox_type: str | None) -> bool: + return self.inner.grants_reach(sandbox_type) + + @property + def max_single_upload_bytes(self) -> int | None: + return self.inner.max_single_upload_bytes + + def put(self, key: str, data: bytes, content_type: str = DEFAULT_CONTENT_TYPE, allow_overwrite: bool = False) -> str: + return self.inner.put(self.prefix + key, data, content_type, allow_overwrite) + + def put_file(self, key: str, file_path: str, content_type: str = DEFAULT_CONTENT_TYPE) -> str: + return self.inner.put_file(self.prefix + key, file_path, content_type) + + def put_file_at(self, object_url: str, file_path: str, content_type: str = DEFAULT_CONTENT_TYPE) -> str: + self.get_object_key(object_url) + return self.inner.put_file_at(object_url, file_path, content_type) + + def get(self, object_url: str) -> bytes: + return self.inner.get(object_url) + + def exists(self, key: str) -> bool: + return self.inner.exists(self.prefix + key) + + def get_object_metadata(self, key: str) -> ObjectMetadata | None: + return self.inner.get_object_metadata(self.prefix + key) + + def get_object_metadata_at(self, object_url: str) -> ObjectMetadata | None: + return self.inner.get_object_metadata_at(object_url) + + def list(self, prefix: str) -> list[str]: + return [key.removeprefix(self.prefix) for key in self.inner.list(self.prefix + prefix)] + + def list_at(self, url_prefix: str) -> list[str]: + return self.inner.list_at(url_prefix) + + def object_url(self, key: str) -> str: + return self.inner.object_url(self.prefix + key) + + def get_object_key(self, object_url: str) -> str: + key = self.inner.get_object_key(object_url) + if not key.startswith(self.prefix): + raise ValueError(f"{object_url!r} is not under this store's prefix {self.prefix!r}") + return key.removeprefix(self.prefix) + + def owns(self, object_url: str) -> bool: + return self.inner.owns(object_url) and self.inner.get_object_key(object_url).startswith(self.prefix) + + def download_to_file(self, object_url: str, dest_path: str) -> None: + self.inner.download_to_file(object_url, dest_path) + + def open(self, object_url: str) -> BinaryIO: + return self.inner.open(object_url) + + def read(self, key: str) -> bytes: + return self.inner.read(self.prefix + key) + + def signed_get_url(self, object_url: str, expires_in: int = 3600) -> str | None: + return self.inner.signed_get_url(object_url, expires_in) + + def signed_put_url(self, object_url: str, expires_in: int = 3600) -> str | None: + self.get_object_key(object_url) + return self.inner.signed_put_url(object_url, expires_in) + + def shared_credentials_env(self) -> dict[str, str]: + """None: credentials would reach every prefix of the bucket, not this store's alone.""" + return {} + + def issue_read_grant(self, object_url: str, *, expires_in: int | None = None) -> HttpGetGrant: + return self.inner.issue_read_grant(object_url, **_lifetime(expires_in)) + + def issue_write_grant( + self, object_url: str, *, media_type: str, max_bytes: int, expires_in: int | None = None + ) -> HttpPutGrant: + self.get_object_key(object_url) + return self.inner.issue_write_grant(object_url, media_type=media_type, max_bytes=max_bytes, **_lifetime(expires_in)) + + def issue_upload_policy(self, prefix_url: str, *, max_object_bytes: int, expires_in: int) -> UploadPolicy: + self.get_object_key(prefix_url) + return self.inner.issue_upload_policy(prefix_url, max_object_bytes=max_object_bytes, expires_in=expires_in) + + +class NamespaceRunObjectStore(LocalRunObjectStore): + """The object store a run in a configured namespace sees: keys go to the namespace's store, an object_url to the + store it addresses, and a write anywhere but the namespace's store, or a read of another namespace's, is refused.""" + + def __init__(self, configured: ObjectStore, own: ObjectStore, namespace: str, + others: Mapping[str, ObjectStore]) -> None: + super().__init__(configured, own) + self.namespace = namespace + self.others = dict(others) + + def shared_credentials_env(self) -> dict[str, str]: + return self.local.shared_credentials_env() + + def _at(self, object_url: str) -> ObjectStore: + if self.local.owns(object_url): + return self.local + if (other := next((name for name, store in self.others.items() if store.owns(object_url)), None)): + raise NamespaceWriteError(f"a run in @{self.namespace} can't reach {object_url!r}, an @{other} object") + return self.configured + + def _writing(self, object_url: str) -> ObjectStore: + store = self._at(object_url) + if store is not self.local: + raise NamespaceWriteError(f"a run in @{self.namespace} writes objects only to its own store, not {object_url!r}") + return store diff --git a/tst/unit/config/test_describe.py b/tst/unit/config/test_describe.py index f630636f5..bb1fadf45 100644 --- a/tst/unit/config/test_describe.py +++ b/tst/unit/config/test_describe.py @@ -301,6 +301,14 @@ def test_render_puts_the_file_and_every_section_on_screen(tmp_path): [task_steps] impls = ["pkg.steps:CustomStep"] +[namespaces.team-a.document] +impl = "agent_env.store.document_store:LocalSqliteDocumentStore" +config = { path = "/tmp/team-a.db" } + +[namespaces.team-a.object] +impl = "agent_env.store.object_store:LocalFilesystemObjectStore" +config = { root = "/tmp/team-a" } + [explorer] port = 9999 @@ -316,7 +324,7 @@ def test_render_puts_the_file_and_every_section_on_screen(tmp_path): _ALIASED = ("document", "object", "image", "secret", "runner") _EVERY_TABLE = ("document", "object", "image", "secret", "runner", "model", "conversations", - "agents", "sandbox", "state", "envs", "artifacts", "task_steps", "explorer") + "agents", "sandbox", "state", "envs", "artifacts", "task_steps", "namespaces", "explorer") @pytest.mark.parametrize("toml", [None, MONGO_TOML, FULL_SURFACE_TOML]) diff --git a/tst/unit/store/configured_namespace_test.py b/tst/unit/store/configured_namespace_test.py new file mode 100644 index 000000000..9ec757e87 --- /dev/null +++ b/tst/unit/store/configured_namespace_test.py @@ -0,0 +1,321 @@ +"""Configured namespaces: an ``@/`` id, what a run of its task records and every entity and object that run +writes keep to the namespace's own stores; unplaced reads see a namespace only inside its run or once a service +opens it; nothing outside a namespace may name its ids.""" + +import inspect + +import pytest + +from agent_env.artifact.artifacts.file import FileArtifact +from agent_env.artifact.store import get_artifact_store +from agent_env.config import get_config, reset_config +from agent_env.config.errors import ConfigError +from agent_env.store import Filter, LocalSqliteDocumentStore, UpdateSpec +from agent_env.store.ids import key_segment, validate_namespace_name +from agent_env.store.object_store import LocalFilesystemObjectStore, ObjectStore +from agent_env.store.routing import ( + NamespaceDocumentStore, + NamespaceRunObjectStore, + NamespaceWriteError, + PrefixedObjectStore, + enable_namespace_routing, + reading_namespaces, + run_scope, +) +from tst.conftest import _reset_store_singletons +from tst.store import conformance + +TEAM_TASK = "@team-a/tasks/t" +TEAM_ENV = "@team-a/envs/e" +_NOT_ROUTED = {"from_config"} + + +def _write_config(tmp_path, namespaces: str) -> None: + (tmp_path / "config.toml").write_text(f""" +[stores.document] +impl = "agent_env.store.document_store:LocalSqliteDocumentStore" +config = {{ path = "{tmp_path / 'shared.db'}" }} + +[stores.object] +impl = "agent_env.store.object_store:LocalFilesystemObjectStore" +config = {{ root = "{tmp_path / 'shared-objects'}" }} +{namespaces} +""") + + +def _namespace(tmp_path, name: str, objects: str = "") -> str: + return f""" +[namespaces.{name}.document] +impl = "agent_env.store.document_store:LocalSqliteDocumentStore" +config = {{ path = "{tmp_path / f'{name}.db'}" }} + +[namespaces.{name}.object] +impl = "agent_env.store.object_store:LocalFilesystemObjectStore" +config = {{ root = "{tmp_path / (objects or f'{name}-objects')}" }} +""" + + +@pytest.fixture +def configured(tmp_path, monkeypatch): + """A shared store and two configured namespaces, ``team-a`` and ``team-b``, sharing one object root.""" + _write_config(tmp_path, _namespace(tmp_path, "team-a", "restricted") + _namespace(tmp_path, "team-b", "restricted")) + monkeypatch.setenv("AGENT_ENV_CONFIG", str(tmp_path / "config.toml")) + reset_config() + _reset_store_singletons() + yield get_config() + reset_config() + _reset_store_singletons() + + +@pytest.fixture +def stores(configured): + router = configured.get_document_store() + assert isinstance(router, NamespaceDocumentStore) + return router, router.configured, router.namespaces["team-a"] + + +def _ids(store, collection, field="id"): + return {doc[field] for doc in store.query(collection, Filter())} + + +# The suite's own collection is no entity's, so a run's writes all land in the run's namespace. +_CONFORMANCE = [ + pytest.param(case, mode, id=f"{mode}-{case.__name__}") + for mode in ("shared", "opened", "run") + for case in conformance.CASES +] + + +@pytest.mark.parametrize("case, mode", _CONFORMANCE) +def test_the_router_passes_the_conformance_suite(case, mode, stores): + router, _, team = stores + team.insert("seed", {"id": "seed"}) + if mode == "opened": + with reading_namespaces("team-a"): + case(router, "coll") + elif mode == "run": + with run_scope(TEAM_TASK): + case(router, "coll") + else: + case(router, "coll") + + +def test_no_namespaces_leave_the_configured_store_unwrapped(tmp_path, monkeypatch): + _write_config(tmp_path, "") + monkeypatch.setenv("AGENT_ENV_CONFIG", str(tmp_path / "config.toml")) + reset_config() + + assert isinstance(get_config().get_document_store(), LocalSqliteDocumentStore) + assert isinstance(get_config().get_object_store(), LocalFilesystemObjectStore) + assert get_config().configured_namespaces() == () + + +@pytest.mark.parametrize("table, message", [ + ('[namespaces.Team.document]\nimpl = "x:Y"\n[namespaces.Team.object]\nimpl = "x:Y"\n', "lowercase"), + ('[namespaces.local.document]\nimpl = "x:Y"\n[namespaces.local.object]\nimpl = "x:Y"\n', "local namespace"), + ('[namespaces.team-a.document]\nimpl = "x:Y"\n', "a document and an object table"), +]) +def test_a_namespace_table_is_checked(tmp_path, monkeypatch, table, message): + _write_config(tmp_path, table) + monkeypatch.setenv("AGENT_ENV_CONFIG", str(tmp_path / "config.toml")) + reset_config() + + with pytest.raises(ConfigError, match=message): + get_config().get_document_store() + + +def test_ids_are_admitted_only_in_a_configured_namespace(stores): + router, _, _ = stores + router.check_id(TEAM_ENV) + router.check_id("bare-env") + with pytest.raises(ValueError, match="reserved"): + router.check_id("@team-c/envs/e") + with pytest.raises(ValueError, match="has to keep the @team-a namespace"): + router.check_id(f"copy-of-{TEAM_ENV}") + with pytest.raises(ValueError, match="path segment"): + router.check_id("@team-a/envs/../e") + + +def test_the_shared_store_itself_still_refuses_a_namespaced_id(stores): + _, shared, _ = stores + with pytest.raises(ValueError, match="reserved"): + shared.check_id(TEAM_ENV) + + +def test_namespace_names_and_key_segments(): + validate_namespace_name("r-team") + for bad in ("", "-a", "a-", "A", "a_b", "x" * 65, "local", "hf"): + with pytest.raises(ValueError): + validate_namespace_name(bad) + assert key_segment("@r-team/winter sale").startswith("r-team/winter-sale-") + assert key_segment("@local/x").startswith("local/x-") + assert key_segment("bare") == "bare" + + +def test_entities_route_by_their_id_and_unplaced_reads_see_only_opened_namespaces(stores): + router, shared, team = stores + router.insert("envs", {"id": TEAM_ENV, "version": 1}) + router.insert("envs", {"id": "rocket", "version": 1}) + + assert _ids(team, "envs") == {TEAM_ENV} + assert _ids(shared, "envs") == {"rocket"} + assert router.find_one("envs", Filter.of(id=TEAM_ENV))["id"] == TEAM_ENV + assert _ids(router, "envs") == {"rocket"} + with reading_namespaces("team-a"): + assert _ids(router, "envs") == {TEAM_ENV, "rocket"} + with run_scope(TEAM_TASK): + assert _ids(router, "envs") == {TEAM_ENV, "rocket"} + with reading_namespaces("team-b"): + assert _ids(router, "envs") == {"rocket"} + + +def test_a_runs_records_and_the_records_of_a_namespaced_task_keep_to_the_namespace(stores): + router, shared, team = stores + with run_scope(TEAM_TASK): + router.insert("task_instances", {"instance_id": "ti_1", "task_id": TEAM_TASK}) + router.insert("task_step_journal", {"instance_id": "ti_1", "step": "a"}) + router.insert("task_instances", {"instance_id": "ti_2", "task_id": TEAM_TASK}) + router.insert("task_instances", {"instance_id": "ti_3", "task_id": "bare-task"}) + + assert _ids(team, "task_instances", "instance_id") == {"ti_1", "ti_2"} + assert _ids(team, "task_step_journal", "instance_id") == {"ti_1"} + assert _ids(shared, "task_instances", "instance_id") == {"ti_3"} + assert router.find_one("task_instances", Filter.of(instance_id="ti_1")) is None + assert router.find_one("task_instances", Filter.of(task_id=TEAM_TASK)) is not None + with reading_namespaces("team-a"): + assert router.find_one("task_instances", Filter.of(instance_id="ti_1"))["task_id"] == TEAM_TASK + + +def test_a_nested_bare_run_stays_in_the_namespace(stores): + router, _, team = stores + with run_scope(TEAM_TASK), run_scope("bare-task"): + router.insert("task_instances", {"instance_id": "ti_nested", "task_id": "bare-task"}) + assert _ids(team, "task_instances", "instance_id") == {"ti_nested"} + + +def test_a_run_writes_new_bare_entities_to_its_namespace_but_no_new_version_of_a_shared_one(stores): + router, shared, team = stores + router.insert("artifacts", {"id": "shared-file", "version": 1}) + with run_scope(TEAM_TASK): + router.insert("artifacts", {"id": "ti_1__recording", "version": 1}) + with pytest.raises(NamespaceWriteError, match="new version of 'shared-file'"): + router.insert("artifacts", {"id": "shared-file", "version": 2}) + with pytest.raises(NamespaceWriteError, match="an @team-b entity"): + router.insert("artifacts", {"id": "@team-b/x", "version": 1}) + + assert _ids(team, "artifacts") == {"ti_1__recording"} + assert _ids(shared, "artifacts") == {"shared-file"} + + +def test_a_run_may_update_a_shared_entity_without_naming_its_namespace(stores): + router, shared, _ = stores + router.insert("envs", {"id": "rocket", "version": 1, "metadata": {}}) + with run_scope(TEAM_TASK): + router.update("envs", Filter.of(id="rocket", version=1), UpdateSpec(set={"metadata.checked": True})) + with pytest.raises(NamespaceWriteError, match="reference another namespace's id"): + router.update("envs", Filter.of(id="rocket", version=1), UpdateSpec(set={"metadata.task": TEAM_TASK})) + assert shared.find_one("envs", Filter.of(id="rocket"))["metadata"] == {"checked": True} + + +def test_a_run_cannot_update_a_shared_run_record(stores): + router, shared, _ = stores + shared.insert("task_instances", {"instance_id": "ti_shared", "status": "running"}) + with run_scope(TEAM_TASK), pytest.raises(NamespaceWriteError, match="in the configured store"): + router.update("task_instances", Filter.of(instance_id="ti_shared"), UpdateSpec(set={"status": "done"})) + assert shared.find_one("task_instances", Filter.of(instance_id="ti_shared"))["status"] == "running" + + +def test_nothing_outside_a_namespace_names_its_ids(stores): + router, _, _ = stores + with pytest.raises(NamespaceWriteError, match="tasks\\[0\\]"): + router.insert("evals", {"id": "shared-eval", "version": 1, "tasks": [TEAM_TASK]}) + with pytest.raises(NamespaceWriteError, match="another namespace's id"): + router.insert("evals", {"id": "@team-b/eval", "version": 1, "tasks": [TEAM_TASK]}) + router.insert("tasks", {"id": TEAM_TASK, "version": 1, "env_id": "rocket"}) + + +def test_local_routing_runs_on_top_of_configured_namespaces(stores, configured): + enable_namespace_routing() + router = configured.get_document_store() + router.insert("envs", {"id": "@local/~/b/envs/e", "version": 1}) + router.insert("envs", {"id": TEAM_ENV, "version": 1}) + + assert _ids(router.local, "envs") == {"@local/~/b/envs/e"} + assert _ids(stores[2], "envs") == {TEAM_ENV} + + +def test_a_runs_objects_go_to_its_namespace_under_its_prefix(configured, tmp_path): + shared = configured.get_object_store() + shared_url = shared.put("inputs/a.txt", b"shared") + with run_scope(TEAM_TASK): + store = configured.get_object_store() + assert isinstance(store, NamespaceRunObjectStore) + url = store.put("trajectories/ti_1.json", b"{}") + assert store.get(shared_url) == b"shared" + with pytest.raises(NamespaceWriteError, match="writes objects only to its own store"): + store.signed_put_url(shared_url) + other = configured.namespace_object_store("team-b").put("x", b"b") + with pytest.raises(NamespaceWriteError, match="an @team-b object"): + store.get(other) + + assert (tmp_path / "restricted" / "team-a" / "trajectories" / "ti_1.json").read_bytes() == b"{}" + assert not (tmp_path / "shared-objects" / "trajectories").exists() + assert configured.get_object_store() is shared + assert configured.get_object_store_at(url) is configured.namespace_object_store("team-a") + + +def test_entity_objects_follow_the_entity_and_the_run(configured): + team = configured.namespace_object_store("team-a") + assert configured.get_object_store_for(TEAM_ENV) is team + assert configured.get_object_store_for("bare") is configured.get_object_store() + with run_scope(TEAM_TASK): + assert configured.get_object_store_for("bare") is team + + +def test_an_object_url_is_recorded_only_in_its_namespace(configured): + team_url = configured.namespace_object_store("team-a").object_url("artifacts/x") + shared_url = configured.get_object_store().object_url("artifacts/x") + configured.check_object_url(TEAM_ENV, team_url) + configured.check_object_url("bare", shared_url) + with pytest.raises(ValueError, match="keeps its objects in the @team-a"): + configured.check_object_url(TEAM_ENV, shared_url) + with pytest.raises(ValueError, match="an @team-a object"): + configured.check_object_url("bare", team_url) + with pytest.raises(ValueError, match="keeps its objects in the @team-a"): + configured.check_object_url("@team-a/y", configured.namespace_object_store("team-b").object_url("y")) + + +def test_a_namespaced_artifact_round_trips_through_its_own_stores(stores, configured, tmp_path): + _, shared, team = stores + stored = FileArtifact.put_bytes(TEAM_ENV + "-file", description="d", filename="f.txt", content=b"hi") + + assert stored.object_url.startswith(configured.namespace_object_store("team-a").object_url("")) + assert _ids(team, "artifacts") == {TEAM_ENV + "-file"} + assert _ids(shared, "artifacts") == set() + assert get_artifact_store().get(TEAM_ENV + "-file").object_url == stored.object_url + assert configured.get_object_store_at(stored.object_url).get(stored.object_url) == b"hi" + assert list((tmp_path / "shared-objects").rglob("f.txt")) == [] + + +def test_the_prefixed_store_keeps_to_its_prefix(tmp_path): + inner = LocalFilesystemObjectStore(str(tmp_path)) + store = PrefixedObjectStore(inner, "team-a/") + url = store.put("k/v", b"1") + outside = inner.put("k/v", b"2") + + assert store.owns(url) and not store.owns(outside) + assert store.get_object_key(url) == "k/v" + assert store.list("k/") == ["k/v"] + assert store.read("k/v") == b"1" + with pytest.raises(ValueError, match="prefix"): + store.put_file_at(outside, str(tmp_path / "team-a" / "k" / "v")) + with pytest.raises(ValueError, match="ends with '/'"): + PrefixedObjectStore(inner, "team-a") + + +@pytest.mark.parametrize("wrapper", [PrefixedObjectStore, NamespaceRunObjectStore]) +def test_the_object_wrappers_cover_every_store_method(wrapper): + methods = {name for name, member in inspect.getmembers(ObjectStore) if callable(member) and not name.startswith("_")} + covered = {name for name in methods + if any(name in vars(cls) for cls in wrapper.__mro__ if cls is not ObjectStore and cls is not object)} + assert methods - covered - _NOT_ROUTED == set() diff --git a/tst/unit/store/ids_test.py b/tst/unit/store/ids_test.py index 985bfd3b2..43d2138d4 100644 --- a/tst/unit/store/ids_test.py +++ b/tst/unit/store/ids_test.py @@ -56,7 +56,6 @@ "", " ", "__", - "@acme/tickets", "@LOCAL/upper-namespace", ] @@ -122,6 +121,13 @@ def test_a_malformed_local_id_is_not_encoded_as_one(): assert key_segment("@local") == "@local" +def test_an_id_in_another_namespace_encodes_under_that_namespace(): + segment = key_segment("@acme/tickets") + assert segment == "acme/tickets-" + hashlib.sha256(b"@acme/tickets").hexdigest()[:12] + assert _OCI_REPOSITORY.fullmatch(image_repository("@acme/tickets")) + assert fs_safe("@acme/tickets") == "@acme/tickets" + + @pytest.mark.parametrize("entity_id", [ "@local/~/work/triage/tickets", "@local/a", From 4fc792641c7532fc4c6f89fd6c4843a6b4397f46 Mon Sep 17 00:00:00 2001 From: Yzabelle Go Date: Fri, 13 Nov 2026 11:51:23 -0500 Subject: [PATCH 2/3] refactor(store): place ids in configured namespaces through a resolver A [namespace_resolver] (module:Class, namespace_of(entity_id, fresh=)) now says which ids are in which configured namespace, instead of an @/ id prefix. Ids derived from a placed id (__...) follow it (id_bases). Writes ask the resolver fresh and fail when it raises; a run asks once where its task is, an enclosing run's namespace winning. Configured stores refuse every @ id again, key_segment is unchanged from main, and the reference guard (a shared document naming a namespaced id) is gone: with a resolver it would cost a lookup per string written. TableNamespaceResolver lists ids in config, for tests and small deployments. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/config/describe.py | 1 + src/agent_env/config/runtime.py | 57 +++++--- src/agent_env/env/snapshot_store.py | 2 +- src/agent_env/store/ids.py | 72 ++++------ src/agent_env/store/namespace_resolver.py | 39 +++++ src/agent_env/store/routing.py | 140 +++++++++--------- tst/unit/config/test_describe.py | 6 +- tst/unit/store/configured_namespace_test.py | 150 +++++++++++++------- tst/unit/store/ids_test.py | 8 +- 9 files changed, 286 insertions(+), 189 deletions(-) create mode 100644 src/agent_env/store/namespace_resolver.py diff --git a/src/agent_env/config/describe.py b/src/agent_env/config/describe.py index 88c3a42fc..229d9a778 100644 --- a/src/agent_env/config/describe.py +++ b/src/agent_env/config/describe.py @@ -284,6 +284,7 @@ class FileSection: FileSection("artifacts", ("artifacts",), "artifact.registry", {"impls": None, "type_aliases": None}), FileSection("task_steps", ("task_steps",), "task_step.registry", {"impls": None}), FileSection("namespaces", ("namespaces",), "config.runtime", Named({"document": _SEAM, "object": _SEAM})), + FileSection("resolver", ("namespace_resolver",), "config.runtime", _SEAM), FileSection("explorer", ("explorer",), "explorer.app", {**dict.fromkeys(("port", "cors_origins", "allowed_hosts", "static_dir")), "plugins": {"impls": None}}), diff --git a/src/agent_env/config/runtime.py b/src/agent_env/config/runtime.py index 060f2a233..a3f6af884 100644 --- a/src/agent_env/config/runtime.py +++ b/src/agent_env/config/runtime.py @@ -534,12 +534,41 @@ def namespace_object_store(self, namespace: str) -> ObjectStore: """The configured ``namespace``'s own object store: its table's, with every key under ``/``.""" return self._namespace_store(namespace)[1] - def namespace_of(self, entity_id: str) -> Optional[str]: - """The configured namespace ``entity_id`` is in, or None.""" - from agent_env.store.ids import parse_namespace + def namespace_of(self, entity_id: str, *, fresh: bool = False) -> Optional[str]: + """The configured namespace ``[namespace_resolver]`` places ``entity_id`` in, or None. ``fresh`` asks for an + answer read now, as before a write.""" + from agent_env.store.ids import is_local_id + + if not self.configured_namespaces() or not entity_id or is_local_id(entity_id): + return None + return self._checked(self.namespace_resolver().namespace_of(entity_id, fresh=fresh), entity_id) + + def run_namespace(self) -> Optional[str]: + """The configured namespace of the task running (``run_scope``), or None.""" + from agent_env.store import routing + + if not self.configured_namespaces(): + return None + return self._checked(routing.resolved_run_namespace(self.namespace_resolver()), "this run") - namespace = parse_namespace(entity_id) - return namespace if namespace is not None and namespace in self.configured_namespaces() else None + def namespace_resolver(self) -> Any: + """The ``[namespace_resolver]`` that places ids in configured namespaces (cached); required with any.""" + from agent_env.store.namespace_resolver import NamespaceResolver + + with self._routing_lock: + if "resolver" not in self._namespaced: + section = self.section("namespace_resolver") + if not section.get("impl"): + raise ConfigError("[namespaces] needs a [namespace_resolver] table with an impl, which says which ids " + "are in which namespace") + self._namespaced["resolver"] = (None, config_loader.build_store( + dict(section), NamespaceResolver, secret_resolver=self._resolve_secret)) + return self._namespaced["resolver"][1] + + def _checked(self, namespace: Optional[str], what: str) -> Optional[str]: + if namespace is not None and namespace not in self.configured_namespaces(): + raise ConfigError(f"the namespace resolver placed {what!r} in {namespace!r}, which no [namespaces] table configures") + return namespace def _namespace_store(self, namespace: str) -> tuple[Any, Any]: from agent_env.store.document_store import DocumentStore @@ -568,16 +597,10 @@ def _with_namespaces(self, configured: "DocumentStore") -> "DocumentStore": held = self._namespaced.get("document") if held is None or held[0] is not configured: held = (configured, routing.NamespaceDocumentStore( - configured, {name: self.namespace_document_store(name) for name in names})) + configured, {name: self.namespace_document_store(name) for name in names}, self.namespace_resolver())) self._namespaced["document"] = held return held[1] - def _run_namespace(self) -> Optional[str]: - from agent_env.store import routing - - namespace = routing.run_namespace() - return namespace if namespace is not None and namespace in self.configured_namespaces() else None - def _namespace_owning(self, object_url: str) -> Optional[str]: return next((name for name in self.configured_namespaces() if self.namespace_object_store(name).owns(object_url)), None) @@ -747,7 +770,7 @@ def get_object_store(self) -> ObjectStore: from agent_env.store import routing configured = self._configured_object_store() - if (namespace := self._run_namespace()) is not None: + if (namespace := self.run_namespace()) is not None: return self._namespace_run_object_store(namespace, configured) if not routing.in_local_run(): return configured @@ -772,7 +795,7 @@ def get_object_store_for(self, entity_id: str) -> ObjectStore: from agent_env.store import routing from agent_env.store.ids import is_local_id - if (namespace := self.namespace_of(entity_id) or self._run_namespace()) is not None: + if (namespace := self.namespace_of(entity_id, fresh=True) or self.run_namespace()) is not None: return self.namespace_object_store(namespace) configured, local = self._namespace_stores("object") if local is None: @@ -805,12 +828,12 @@ def check_object_url(self, entity_id: str, object_url: str) -> None: from agent_env.store import routing from agent_env.store.ids import is_local_id - namespace = self.namespace_of(entity_id) or self._run_namespace() + namespace = self.namespace_of(entity_id, fresh=True) or self.run_namespace() owner = self._namespace_owning(object_url) if namespace != owner and namespace is not None: - raise ValueError(f"{entity_id!r} keeps its objects in the @{namespace} namespace's store, not {object_url!r}") + raise ValueError(f"{entity_id!r} keeps its objects in the {namespace!r} namespace's store, not {object_url!r}") if namespace != owner: - raise ValueError(f"{entity_id!r} is in no configured namespace, so it can't record {object_url!r}, an @{owner} object") + raise ValueError(f"{entity_id!r} is in no configured namespace, so it can't record {object_url!r}, an object of {owner!r}") if namespace is not None: return configured, local = self._namespace_stores("object") diff --git a/src/agent_env/env/snapshot_store.py b/src/agent_env/env/snapshot_store.py index ac4198ba6..66b8f8892 100644 --- a/src/agent_env/env/snapshot_store.py +++ b/src/agent_env/env/snapshot_store.py @@ -113,7 +113,7 @@ async def create(cls, instance_id: str, on_progress: ProgressCallback | None = N universe_id = environment_universe["id"] universe_version = environment_universe["version"] if is_local_id(env_id) != is_local_id(universe_id) or ( - get_config().namespace_of(env_id) != get_config().namespace_of(universe_id)): + get_config().namespace_of(env_id, fresh=True) != get_config().namespace_of(universe_id, fresh=True)): raise ValueError( f"env {env_id!r} and universe {universe_id!r} are in different namespaces; a snapshot is recorded " "under both, so both have to be in the same one" diff --git a/src/agent_env/store/ids.py b/src/agent_env/store/ids.py index 2391f17da..23fad8b94 100644 --- a/src/agent_env/store/ids.py +++ b/src/agent_env/store/ids.py @@ -1,11 +1,10 @@ """Namespaced entity ids, and the object keys, image repositories and filenames derived from them. An id opening ``@/`` names the store it lives in. The local namespaces, ``@local/…`` and ``@hf/…`` -(a bundle ``agent-env run hf://`` downloaded from the Hugging Face Hub), are this machine's local stores; a -configured namespace (``[namespaces.]``) has stores of its own beside the configured ones. Documents keep an -id verbatim. Everything derived from a local id uses one bounded segment, ``local/-``, and -from another namespace's id ``/-``; an object key or image repository takes a -bare id over 200 bytes as ``-``; every other id passes through byte-identical. +(a bundle ``agent-env run hf://`` downloaded from the Hugging Face Hub), are this machine's local stores; no other +exists today. Documents keep an id verbatim. Everything derived from a local id uses one bounded +segment, ``local/-``; an object key or image repository takes a bare id over +200 bytes as ``-``; every other id passes through byte-identical. The encoders never raise: they expect ids that ``validate_local_id`` accepts, and still return a safe segment for any other input. An id may contain spaces, ``&`` and parentheses, so quote one before it reaches a shell or a URL, and derive object keys, image repositories and filenames with @@ -75,43 +74,32 @@ def validate_local_id(entity_id: str) -> None: namespace = local_namespace(entity_id) if namespace is None: raise ValueError(f"local id {entity_id!r} must start with {LOCAL_PREFIX!r} or {HF_PREFIX!r}") - _validate_in(entity_id, namespace, "local id") - - -def validate_namespace_name(namespace: str) -> None: - """Raise ValueError unless ``namespace`` can name a configured namespace: 1 to 64 lowercase letters, digits and - inner ``-``, and not a local namespace.""" - if namespace in LOCAL_NAMESPACES: - raise ValueError(f"@{namespace} is a local namespace, so it can't be configured") - if not isinstance(namespace, str) or not _NAMESPACE_NAME.fullmatch(namespace): - raise ValueError(f"namespace {namespace!r} must be 1 to 64 lowercase letters, digits and inner '-'") - - -def validate_namespaced_id(entity_id: str, namespace: str) -> None: - """``validate_local_id``'s rules for an id in the configured ``namespace``.""" - if not entity_id.startswith(namespace_prefix(namespace)): - raise ValueError(f"id {entity_id!r} must start with {namespace_prefix(namespace)!r}") - _validate_in(entity_id, namespace, f"@{namespace} id") - - -def _validate_in(entity_id: str, namespace: str, what: str) -> None: prefix = namespace_prefix(namespace) size = len(entity_id.encode("utf-8", "surrogatepass")) if size > MAX_LOCAL_ID_BYTES: - raise ValueError(f"{what} is {size} bytes; the limit is {MAX_LOCAL_ID_BYTES}: {entity_id[:64]!r}…") + raise ValueError(f"local id is {size} bytes; the limit is {MAX_LOCAL_ID_BYTES}: {entity_id[:64]!r}…") disallowed = sorted({c for c in entity_id[len(prefix):] if not _allowed_in_id(c)}) if disallowed: raise ValueError( - f"{what} {entity_id!r} contains {ascii(''.join(disallowed))}; " - f"a {what} may contain only letters, digits, spaces and . _ - ~ / @ ( ) + , &" + f"local id {entity_id!r} contains {ascii(''.join(disallowed))}; " + "a local id may contain only letters, digits, spaces and . _ - ~ / @ ( ) + , &" ) segments = entity_id[len(prefix):].split("/") if any(segment in ("", ".", "..") for segment in segments): - raise ValueError(f"{what} {entity_id!r} has an empty, '.' or '..' path segment") + raise ValueError(f"local id {entity_id!r} has an empty, '.' or '..' path segment") if any(segment != segment.strip(" ") for segment in segments): - raise ValueError(f"{what} {entity_id!r} has a path segment that opens or closes with a space") + raise ValueError(f"local id {entity_id!r} has a path segment that opens or closes with a space") if any(_detached_mark(segment) for segment in segments): - raise ValueError(f"{what} {entity_id!r} has a combining mark that does not follow a letter or digit") + raise ValueError(f"local id {entity_id!r} has a combining mark that does not follow a letter or digit") + + +def validate_namespace_name(namespace: str) -> None: + """Raise ValueError unless ``namespace`` can name a configured namespace: 1 to 64 lowercase letters, digits and + inner ``-``, and not a local namespace.""" + if namespace in LOCAL_NAMESPACES: + raise ValueError(f"@{namespace} is a local namespace, so it can't be configured") + if not isinstance(namespace, str) or not _NAMESPACE_NAME.fullmatch(namespace): + raise ValueError(f"namespace {namespace!r} must be 1 to 64 lowercase letters, digits and inner '-'") def _allowed_in_id(character: str) -> bool: @@ -125,12 +113,12 @@ def _detached_mark(segment: str) -> bool: return any(this == "M" and before not in _ID_CATEGORIES for before, this in zip(["Z", *categories], categories)) -def _local_segment(entity_id: str, namespace: str = LOCAL_NAMESPACE) -> str: +def _local_segment(entity_id: str) -> str: digest = hashlib.sha256(entity_id.encode("utf-8", "surrogatepass")).hexdigest()[:_HASH_LENGTH] - decomposed = unicodedata.normalize("NFKD", entity_id.removeprefix(namespace_prefix(namespace))) + decomposed = unicodedata.normalize("NFKD", entity_id.removeprefix(LOCAL_PREFIX)) folded = "".join(c for c in decomposed if not unicodedata.combining(c)).casefold().replace("~", "") slug = _NON_SLUG_RUN.sub("-", folded).strip("-")[:_SLUG_LENGTH].rstrip("-") - return f"{namespace}/{slug}-{digest}" if slug else f"{namespace}/{digest}" + return f"{LOCAL_NAMESPACE}/{slug}-{digest}" if slug else f"{LOCAL_NAMESPACE}/{digest}" def aliases_local_encoding(entity_id: str) -> bool: @@ -139,19 +127,10 @@ def aliases_local_encoding(entity_id: str) -> bool: return _ENCODED_LOCAL_ID.match(entity_id) is not None -def _namespaced(entity_id: str) -> bool: - """Whether ``entity_id`` is ``@/…`` for a name a namespace could be configured under.""" - namespace = parse_namespace(entity_id) - return (namespace is not None and namespace not in LOCAL_NAMESPACES and "/" in entity_id - and _NAMESPACE_NAME.fullmatch(namespace) is not None) - - def key_segment(entity_id: str) -> str: """The object-key segment for ``entity_id`` (``artifacts////…``).""" if is_local_id(entity_id): return _local_segment(entity_id) - if _namespaced(entity_id): - return _local_segment(entity_id, parse_namespace(entity_id)) if len(entity_id.encode("utf-8", "surrogatepass")) > MAX_BARE_SEGMENT_BYTES: return _local_segment(entity_id).removeprefix(f"{LOCAL_NAMESPACE}/") return entity_id @@ -190,3 +169,10 @@ def derive_path_id(base: str, path: str) -> str: """The id of the file at ``path`` written with ``base``, as a file universe's files are: ``__<16 hex of the path's sha256>``, which any path fits in.""" return derive_id(base, hashlib.sha256(path.encode("utf-8")).hexdigest()[:16]) + + +def id_bases(entity_id: str) -> list[str]: + """``entity_id`` and every id it was derived from by ``derive_id``, longest first: ``a__b__c`` gives + ``a__b__c``, ``a__b`` and ``a``.""" + parts = entity_id.split("__") + return ["__".join(parts[:end]) for end in range(len(parts), 0, -1) if parts[:end] != [""]] diff --git a/src/agent_env/store/namespace_resolver.py b/src/agent_env/store/namespace_resolver.py new file mode 100644 index 000000000..618ea70ac --- /dev/null +++ b/src/agent_env/store/namespace_resolver.py @@ -0,0 +1,39 @@ +"""Which configured namespace an entity id is in (``[namespace_resolver]``). + +A configured namespace keeps its entities, its tasks' run records and objects in stores of its own +(``agent_env.store.routing``). The resolver says which ids are its: a deployment points it at whatever records +that, such as a service's list of restricted objects. An id derived from one of them (``__…``) is the +resolver's to place too; ``id_bases`` lists what it was derived from. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from collections.abc import Mapping +from typing import Optional + +from agent_env.store.ids import id_bases + + +class NamespaceResolver(ABC): + """Places entity ids in configured namespaces. ``fresh`` asks for an answer read now rather than a cached one: + the stores ask so before a write, so an id placed a moment ago is never written outside its namespace.""" + + @classmethod + def from_config(cls, **config) -> NamespaceResolver: + return cls(**config) + + @abstractmethod + def namespace_of(self, entity_id: str, *, fresh: bool = False) -> Optional[str]: + """The configured namespace ``entity_id`` is in, or None for the configured stores. Raises when it can't + tell, so a write is refused rather than placed by a guess.""" + + +class TableNamespaceResolver(NamespaceResolver): + """Ids listed by namespace in config, ``ids = { = [, …] }``, each with every id derived from it.""" + + def __init__(self, ids: Optional[Mapping[str, list[str]]] = None) -> None: + self._placed = {entity_id: namespace for namespace, listed in (ids or {}).items() for entity_id in listed} + + def namespace_of(self, entity_id: str, *, fresh: bool = False) -> Optional[str]: + return next((self._placed[base] for base in id_bases(entity_id) if base in self._placed), None) diff --git a/src/agent_env/store/routing.py b/src/agent_env/store/routing.py index f9195d522..bc54a529b 100644 --- a/src/agent_env/store/routing.py +++ b/src/agent_env/store/routing.py @@ -15,10 +15,10 @@ store is refused (the write fence), and a bare entity may never name a local id (references point outward only). -A configured namespace (``[namespaces.]``) is routed whether or not local routing is on: its ids, what a run -of its tasks records and every entity and object that run writes go to that namespace's own document and object -stores. A read no id places sees the configured store and the running task's namespace, plus only the namespaces -``reading_namespaces`` opens, so a service decides who reads which. Nothing outside a namespace may name its ids. +A configured namespace (``[namespaces.]``) is routed whether or not local routing is on: the ids the +``[namespace_resolver]`` places in it, what a run of its tasks records and every entity and object that run writes go +to that namespace's own document and object stores. A read no id places sees the configured store and the running +task's namespace, plus only the namespaces ``reading_namespaces`` opens, so a service decides who reads which. """ from __future__ import annotations @@ -53,7 +53,6 @@ namespace_prefix, parse_namespace, validate_local_id, - validate_namespaced_id, ) from agent_env.store.image_store import ImageStore, OciRegistryImageStore from agent_env.store.image_store.oci_registry_credentials import ( @@ -82,6 +81,7 @@ _scopes = 0 _scopes_lock = threading.Lock() _run_namespace: ContextVar[Optional[str]] = ContextVar("agent_env_run_namespace", default=None) +_run_owner: ContextVar[Optional["_RunOwner"]] = ContextVar("agent_env_run_owner", default=None) _readable: ContextVar[frozenset[str]] = ContextVar("agent_env_readable_namespaces", default=frozenset()) @@ -130,17 +130,45 @@ def namespace_routing() -> Iterator[None]: @contextmanager def run_scope(owner_id: str) -> Iterator[None]: """The run in progress, a task's or a deployment the CLI makes: what it records follows its owner id's - namespace. A run started inside a run in a namespace stays in that run's namespace unless its own id is local, - so a nested registry task can't reopen the configured stores.""" + namespace. A run started inside a local run stays in that run's namespace unless its own id is local, so a + nested registry task can't reopen the configured stores; one started inside a run in a configured namespace + stays in that namespace (``resolved_run_namespace``).""" outer = _run_namespace.get() - inherited = outer or parse_namespace(owner_id) or "" + inherited = outer if outer in LOCAL_NAMESPACES else parse_namespace(owner_id) or "" token = _run_namespace.set(local_namespace(owner_id) or inherited) + owner = _run_owner.set(_RunOwner(owner_id, _run_owner.get())) try: yield finally: + _run_owner.reset(owner) _run_namespace.reset(token) +class _RunOwner: + """A run's owner id and the run it was started in, with the configured namespace each resolver placed it in.""" + + def __init__(self, owner_id: str, outer: Optional[_RunOwner]) -> None: + self.owner_id = owner_id + self.outer = outer + self.placed: dict[Any, Optional[str]] = {} + + +def resolved_run_namespace(resolver: Any) -> Optional[str]: + """The configured namespace of the task running, as ``resolver`` places it: the enclosing run's when that is in + one, else its own owner's, asked once per run and fresh. None outside a run, or for a local run.""" + return _owner_namespace(_run_owner.get(), resolver) + + +def _owner_namespace(owner: Optional[_RunOwner], resolver: Any) -> Optional[str]: + if owner is None: + return None + if resolver not in owner.placed: + outer = _owner_namespace(owner.outer, resolver) + owner.placed[resolver] = outer or (None if is_local_id(owner.owner_id) + else resolver.namespace_of(owner.owner_id, fresh=True)) + return owner.placed[resolver] + + def local_run_namespace() -> Optional[str]: """The local namespace of the task running, which its records go to; None outside a local run.""" namespace = _run_namespace.get() @@ -151,11 +179,6 @@ def in_local_run() -> bool: return local_run_namespace() is not None -def run_namespace() -> Optional[str]: - """The namespace of the task running, local or configured; None outside a run or in a run of a bare id.""" - return _run_namespace.get() or None - - @contextmanager def reading_namespaces(*namespaces: str) -> Iterator[None]: """Let reads no id places also see these configured namespaces' stores for the length of the block, as a service @@ -181,24 +204,23 @@ def refuse_in_local_run(what: str) -> None: def refuse_local_references(entity_id: object, value: Any) -> None: """Refuse a bare entity naming a local id anywhere in ``value``: a shared store must never point into one person's local stores.""" - path = _reference(value, "", is_local_id) + path = _local_reference(value, "") if path is not None: raise OutwardReferenceError( f"{entity_id!r} is not a local id, so it can't reference one: {path or 'its value'} names a local id" ) -def _reference(value: Any, path: str, matches: Callable[[str], bool]) -> Optional[str]: - """The path in ``value`` of the first string ``matches`` accepts, ``""`` for ``value`` itself; None when none does.""" +def _local_reference(value: Any, path: str) -> Optional[str]: if isinstance(value, str): - return path if matches(value) else None + return path if is_local_id(value) else None if isinstance(value, Mapping): items = ((f"{path}.{key}" if path else str(key), item) for key, item in value.items()) elif isinstance(value, (list, tuple, set)): items = ((f"{path}[{index}]", item) for index, item in enumerate(value)) else: return None - return next((found for item_path, item in items if (found := _reference(item, item_path, matches)) is not None), None) + return next((found for item_path, item in items if (found := _local_reference(item, item_path)) is not None), None) def _update_values(update: UpdateSpec) -> dict: @@ -606,14 +628,16 @@ def _open(self, namespace: str) -> LocalNamespaceDocumentStore: class NamespaceDocumentStore(RoutingDocumentStore): """The configured document store with each configured namespace's documents in that namespace's own store, - ``namespaces`` by name: an entity by its id; a run record by the running task's namespace, else by an owner field; - and every entity a run in a namespace writes. A read no id places sees the running task's namespace, the configured - store and the namespaces ``reading_namespaces`` opens. A run in a namespace never writes a new version of a + ``namespaces`` by name: an entity where ``resolver`` places its id; a run record by the running task's + namespace, else by an owner field; and every entity a run in a namespace writes. A read no id places sees the + running task's namespace, the configured store and the namespaces ``reading_namespaces`` opens. A write asks + the resolver fresh, and is refused when it can't answer. A run in a namespace never writes a new version of a configured entity, so no entity id is in two stores.""" - def __init__(self, configured: DocumentStore, namespaces: Mapping[str, DocumentStore]) -> None: + def __init__(self, configured: DocumentStore, namespaces: Mapping[str, DocumentStore], resolver: Any) -> None: self.configured = configured self.namespaces = dict(namespaces) + self.resolver = resolver self._lock = threading.Lock() def ensure_index( @@ -627,26 +651,23 @@ def ensure_index( store.ensure_index(collection, fields, unique, ttl_seconds) def check_id(self, entity_id: str) -> None: - namespace = self._namespace_of(entity_id) - if namespace is not None: - validate_namespaced_id(entity_id, namespace) - return - if (inside := next((name for name in self.namespaces if namespace_prefix(name) in entity_id), None)): - raise ValueError( - f"id {entity_id!r} contains an @{inside} id but isn't one: an id derived from an @{inside} id " - f"has to keep the @{inside} namespace" - ) self.configured.check_id(entity_id) - def _namespace_of(self, value: object) -> Optional[str]: - namespace = parse_namespace(value) if isinstance(value, str) else None - return namespace if namespace in self.namespaces else None + def _namespace_of(self, value: object, *, fresh: bool = False) -> Optional[str]: + if not isinstance(value, str) or not value or is_local_id(value): + return None + namespace = self.resolver.namespace_of(value, fresh=fresh) + if namespace is not None and namespace not in self.namespaces: + raise ConfigError(f"the namespace resolver placed {value!r} in {namespace!r}, which no [namespaces] table configures") + return namespace def _run(self) -> Optional[str]: - namespace = run_namespace() - return namespace if namespace in self.namespaces else None + namespace = resolved_run_namespace(self.resolver) + if namespace is not None and namespace not in self.namespaces: + raise ConfigError(f"the namespace resolver placed this run in {namespace!r}, which no [namespaces] table configures") + return namespace - def _pinned(self, collection: str, filter: Filter) -> Optional[str]: + def _pinned(self, collection: str, filter: Filter, *, fresh: bool = False) -> Optional[str]: """The one namespace every id the filter pins is in: by ``id`` for an entity, by an owner field for a record.""" fields = ("id",) if is_entity_collection(collection) else _NAMESPACE_OWNER_FIELDS for field in fields: @@ -656,7 +677,7 @@ def _pinned(self, collection: str, filter: Filter) -> Optional[str]: values.append(pred.value) elif isinstance(pred, In): values.extend(pred.values) - namespaces = {self._namespace_of(value) for value in values} + namespaces = {self._namespace_of(value, fresh=fresh) for value in values} if values and len(namespaces) == 1 and (namespace := namespaces.pop()) is not None: return namespace return None @@ -680,51 +701,36 @@ def _writer(self, collection: str, owner: Mapping, payload: Any) -> DocumentStor if is_entity_collection(collection): if isinstance(entity_id, str): self.check_id(entity_id) - namespace = self._namespace_of(entity_id) or run + placed = self._namespace_of(entity_id, fresh=True) + namespace = placed or run if run is not None and namespace != run: - raise NamespaceWriteError(f"a run in @{run} can't write {entity_id!r}, an @{namespace} entity") - if (run is not None and self._namespace_of(entity_id) is None and isinstance(entity_id, str) + raise NamespaceWriteError(f"a run in {run!r} can't write {entity_id!r}, which is in {namespace!r}") + if (run is not None and placed is None and isinstance(entity_id, str) and self.configured.find_one(collection, Filter.of(id=entity_id)) is not None): raise NamespaceWriteError( - f"a run in @{run} can't write a new version of {entity_id!r}: it is in the configured store, " - f"which a run in @{run} writes no new entities to" + f"a run in {run!r} can't write a new version of {entity_id!r}: it is in the configured store, " + f"which a run in {run!r} writes no new entities to" ) - self._refuse_references(entity_id, payload, namespace) else: namespace = run or next( - (found for field in _NAMESPACE_OWNER_FIELDS if (found := self._namespace_of(owner.get(field))) is not None), None) + (found for field in _NAMESPACE_OWNER_FIELDS + if (found := self._namespace_of(owner.get(field), fresh=True)) is not None), None) return self.namespaces[namespace] if namespace is not None else self.configured def _updaters(self, collection: str, filter: Filter, payload: Any) -> Iterator[DocumentStore]: - pinned = self._pinned(collection, filter) + pinned = self._pinned(collection, filter, fresh=True) run = self._run() if pinned is not None: if run is not None and pinned != run: - raise NamespaceWriteError(f"a run in @{run} can't write @{pinned} records") + raise NamespaceWriteError(f"a run in {run!r} can't write records of {pinned!r}") yield self.namespaces[pinned] return if run is not None and not is_entity_collection(collection): yield self.namespaces[run] if self.configured.find_one(collection, filter) is not None: - raise NamespaceWriteError(f"a run in @{run} can't update {collection!r}: its record is in the configured store") + raise NamespaceWriteError(f"a run in {run!r} can't update {collection!r}: its record is in the configured store") return - for store in self._visible(): - if store is self.configured and is_entity_collection(collection): - self._refuse_references(_filter_owner(filter).get("id"), payload, None) - yield store - - def _refuse_references(self, entity_id: object, payload: Any, namespace: Optional[str]) -> None: - """Refuse a document naming an id in a namespace other than ``namespace``, its own (None for the configured - store): a namespace's ids never leave it.""" - if payload is None: - return - others = tuple(namespace_prefix(name) for name in self.namespaces if name != namespace) - path = _reference(payload, "", lambda value: value.startswith(others)) if others else None - if path is not None: - where = f"@{namespace}" if namespace else "the configured store" - raise NamespaceWriteError( - f"{entity_id!r} is in {where}, so it can't reference another namespace's id: {path or 'its value'} does" - ) + yield from self._visible() def object_store_holding(object_url: str, configured: ObjectStore, local: ObjectStore) -> ObjectStore: @@ -993,11 +999,11 @@ def _at(self, object_url: str) -> ObjectStore: if self.local.owns(object_url): return self.local if (other := next((name for name, store in self.others.items() if store.owns(object_url)), None)): - raise NamespaceWriteError(f"a run in @{self.namespace} can't reach {object_url!r}, an @{other} object") + raise NamespaceWriteError(f"a run in {self.namespace!r} can't reach {object_url!r}, an object of {other!r}") return self.configured def _writing(self, object_url: str) -> ObjectStore: store = self._at(object_url) if store is not self.local: - raise NamespaceWriteError(f"a run in @{self.namespace} writes objects only to its own store, not {object_url!r}") + raise NamespaceWriteError(f"a run in {self.namespace!r} writes objects only to its own store, not {object_url!r}") return store diff --git a/tst/unit/config/test_describe.py b/tst/unit/config/test_describe.py index bb1fadf45..dfd5a633b 100644 --- a/tst/unit/config/test_describe.py +++ b/tst/unit/config/test_describe.py @@ -309,6 +309,10 @@ def test_render_puts_the_file_and_every_section_on_screen(tmp_path): impl = "agent_env.store.object_store:LocalFilesystemObjectStore" config = { root = "/tmp/team-a" } +[namespace_resolver] +impl = "agent_env.store.namespace_resolver:TableNamespaceResolver" +config = { ids = { team-a = ["team-task"] } } + [explorer] port = 9999 @@ -324,7 +328,7 @@ def test_render_puts_the_file_and_every_section_on_screen(tmp_path): _ALIASED = ("document", "object", "image", "secret", "runner") _EVERY_TABLE = ("document", "object", "image", "secret", "runner", "model", "conversations", - "agents", "sandbox", "state", "envs", "artifacts", "task_steps", "namespaces", "explorer") + "agents", "sandbox", "state", "envs", "artifacts", "task_steps", "namespaces", "resolver", "explorer") @pytest.mark.parametrize("toml", [None, MONGO_TOML, FULL_SURFACE_TOML]) diff --git a/tst/unit/store/configured_namespace_test.py b/tst/unit/store/configured_namespace_test.py index 9ec757e87..4498c818c 100644 --- a/tst/unit/store/configured_namespace_test.py +++ b/tst/unit/store/configured_namespace_test.py @@ -1,6 +1,6 @@ -"""Configured namespaces: an ``@/`` id, what a run of its task records and every entity and object that run -writes keep to the namespace's own stores; unplaced reads see a namespace only inside its run or once a service -opens it; nothing outside a namespace may name its ids.""" +"""Configured namespaces: the ids the namespace resolver places in a namespace, ids derived from them, what a run of +their tasks records and every entity and object that run writes keep to the namespace's own stores; unplaced reads see +a namespace only inside its run or once a service opens it; a write asks the resolver fresh and fails when it can't.""" import inspect @@ -11,7 +11,8 @@ from agent_env.config import get_config, reset_config from agent_env.config.errors import ConfigError from agent_env.store import Filter, LocalSqliteDocumentStore, UpdateSpec -from agent_env.store.ids import key_segment, validate_namespace_name +from agent_env.store.ids import id_bases, validate_namespace_name +from agent_env.store.namespace_resolver import NamespaceResolver, TableNamespaceResolver from agent_env.store.object_store import LocalFilesystemObjectStore, ObjectStore from agent_env.store.routing import ( NamespaceDocumentStore, @@ -25,9 +26,15 @@ from tst.conftest import _reset_store_singletons from tst.store import conformance -TEAM_TASK = "@team-a/tasks/t" -TEAM_ENV = "@team-a/envs/e" +TEAM_TASK = "team-task" +TEAM_ENV = "team-env" +OTHER_ENV = "other-env" _NOT_ROUTED = {"from_config"} +_RESOLVER = f""" +[namespace_resolver] +impl = "agent_env.store.namespace_resolver:TableNamespaceResolver" +config = {{ ids = {{ team-a = ["{TEAM_TASK}", "{TEAM_ENV}"], team-b = ["{OTHER_ENV}"] }} }} +""" def _write_config(tmp_path, namespaces: str) -> None: @@ -58,7 +65,8 @@ def _namespace(tmp_path, name: str, objects: str = "") -> str: @pytest.fixture def configured(tmp_path, monkeypatch): """A shared store and two configured namespaces, ``team-a`` and ``team-b``, sharing one object root.""" - _write_config(tmp_path, _namespace(tmp_path, "team-a", "restricted") + _namespace(tmp_path, "team-b", "restricted")) + _write_config(tmp_path, _namespace(tmp_path, "team-a", "restricted") + _namespace(tmp_path, "team-b", "restricted") + + _RESOLVER) monkeypatch.setenv("AGENT_ENV_CONFIG", str(tmp_path / "config.toml")) reset_config() _reset_store_singletons() @@ -74,6 +82,19 @@ def stores(configured): return router, router.configured, router.namespaces["team-a"] +class _Recording(NamespaceResolver): + """Places ids from a table, recording each question, and fails while ``down``.""" + + def __init__(self, table): + self.table, self.asked, self.down = table, [], False + + def namespace_of(self, entity_id, *, fresh=False): + self.asked.append((entity_id, fresh)) + if self.down: + raise RuntimeError("restriction rows unreadable") + return self.table.get(entity_id) + + def _ids(store, collection, field="id"): return {doc[field] for doc in store.query(collection, Filter())} @@ -108,6 +129,7 @@ def test_no_namespaces_leave_the_configured_store_unwrapped(tmp_path, monkeypatc assert isinstance(get_config().get_document_store(), LocalSqliteDocumentStore) assert isinstance(get_config().get_object_store(), LocalFilesystemObjectStore) assert get_config().configured_namespaces() == () + assert get_config().namespace_of(TEAM_TASK) is None @pytest.mark.parametrize("table, message", [ @@ -124,51 +146,79 @@ def test_a_namespace_table_is_checked(tmp_path, monkeypatch, table, message): get_config().get_document_store() -def test_ids_are_admitted_only_in_a_configured_namespace(stores): +def test_namespaces_need_a_resolver(tmp_path, monkeypatch): + _write_config(tmp_path, _namespace(tmp_path, "team-a")) + monkeypatch.setenv("AGENT_ENV_CONFIG", str(tmp_path / "config.toml")) + reset_config() + + with pytest.raises(ConfigError, match=r"\[namespace_resolver\]"): + get_config().get_document_store() + + +def test_a_resolver_naming_an_unconfigured_namespace_is_refused(stores): router, _, _ = stores - router.check_id(TEAM_ENV) - router.check_id("bare-env") - with pytest.raises(ValueError, match="reserved"): - router.check_id("@team-c/envs/e") - with pytest.raises(ValueError, match="has to keep the @team-a namespace"): - router.check_id(f"copy-of-{TEAM_ENV}") - with pytest.raises(ValueError, match="path segment"): - router.check_id("@team-a/envs/../e") + router.resolver = TableNamespaceResolver({"team-c": ["stray"]}) + with pytest.raises(ConfigError, match="no \\[namespaces\\] table"): + router.insert("envs", {"id": "stray", "version": 1}) -def test_the_shared_store_itself_still_refuses_a_namespaced_id(stores): - _, shared, _ = stores +def test_ids_keep_the_configured_stores_rules(stores): + router, _, _ = stores + router.check_id(TEAM_ENV) with pytest.raises(ValueError, match="reserved"): - shared.check_id(TEAM_ENV) + router.check_id("@team-a/envs/e") -def test_namespace_names_and_key_segments(): - validate_namespace_name("r-team") +def test_namespace_names_and_id_bases(): + validate_namespace_name("team-a") for bad in ("", "-a", "a-", "A", "a_b", "x" * 65, "local", "hf"): with pytest.raises(ValueError): validate_namespace_name(bad) - assert key_segment("@r-team/winter sale").startswith("r-team/winter-sale-") - assert key_segment("@local/x").startswith("local/x-") - assert key_segment("bare") == "bare" + assert id_bases("u__files__ab12") == ["u__files__ab12", "u__files", "u"] + assert TableNamespaceResolver({"team-a": ["u"]}).namespace_of("u__files__ab12") == "team-a" + assert TableNamespaceResolver({"team-a": ["u"]}).namespace_of("u2") is None -def test_entities_route_by_their_id_and_unplaced_reads_see_only_opened_namespaces(stores): +def test_entities_route_by_resolver_and_unplaced_reads_see_only_opened_namespaces(stores): router, shared, team = stores router.insert("envs", {"id": TEAM_ENV, "version": 1}) + router.insert("envs", {"id": f"{TEAM_ENV}__cli", "version": 1}) router.insert("envs", {"id": "rocket", "version": 1}) - assert _ids(team, "envs") == {TEAM_ENV} + assert _ids(team, "envs") == {TEAM_ENV, f"{TEAM_ENV}__cli"} assert _ids(shared, "envs") == {"rocket"} assert router.find_one("envs", Filter.of(id=TEAM_ENV))["id"] == TEAM_ENV assert _ids(router, "envs") == {"rocket"} with reading_namespaces("team-a"): - assert _ids(router, "envs") == {TEAM_ENV, "rocket"} + assert _ids(router, "envs") == {TEAM_ENV, f"{TEAM_ENV}__cli", "rocket"} with run_scope(TEAM_TASK): - assert _ids(router, "envs") == {TEAM_ENV, "rocket"} + assert _ids(router, "envs") == {TEAM_ENV, f"{TEAM_ENV}__cli", "rocket"} with reading_namespaces("team-b"): assert _ids(router, "envs") == {"rocket"} +def test_writes_ask_the_resolver_fresh_and_fail_when_it_cant_answer(stores): + router, shared, _ = stores + router.resolver = resolver = _Recording({TEAM_ENV: "team-a"}) + router.insert("envs", {"id": TEAM_ENV, "version": 1}) + assert (TEAM_ENV, True) in resolver.asked + + resolver.down = True + with pytest.raises(RuntimeError, match="unreadable"): + router.insert("envs", {"id": "rocket", "version": 1}) + assert _ids(shared, "envs") == set() + + +def test_a_run_asks_once_where_its_task_is(stores): + router, _, team = stores + router.resolver = resolver = _Recording({TEAM_TASK: "team-a"}) + with run_scope(TEAM_TASK): + router.insert("task_step_journal", {"instance_id": "ti_1", "step": "a"}) + router.insert("task_step_journal", {"instance_id": "ti_1", "step": "b"}) + assert resolver.asked.count((TEAM_TASK, True)) == 1 + assert len(team.query("task_step_journal", Filter())) == 2 + + def test_a_runs_records_and_the_records_of_a_namespaced_task_keep_to_the_namespace(stores): router, shared, team = stores with run_scope(TEAM_TASK): @@ -186,11 +236,14 @@ def test_a_runs_records_and_the_records_of_a_namespaced_task_keep_to_the_namespa assert router.find_one("task_instances", Filter.of(instance_id="ti_1"))["task_id"] == TEAM_TASK -def test_a_nested_bare_run_stays_in_the_namespace(stores): - router, _, team = stores +def test_a_nested_run_keeps_the_outer_namespace_and_a_bare_outer_one_takes_the_inner_ones(stores): + router, shared, team = stores with run_scope(TEAM_TASK), run_scope("bare-task"): router.insert("task_instances", {"instance_id": "ti_nested", "task_id": "bare-task"}) - assert _ids(team, "task_instances", "instance_id") == {"ti_nested"} + with run_scope("bare-task"), run_scope(TEAM_TASK): + router.insert("task_instances", {"instance_id": "ti_inner", "task_id": TEAM_TASK}) + assert _ids(team, "task_instances", "instance_id") == {"ti_nested", "ti_inner"} + assert _ids(shared, "task_instances", "instance_id") == set() def test_a_run_writes_new_bare_entities_to_its_namespace_but_no_new_version_of_a_shared_one(stores): @@ -200,20 +253,18 @@ def test_a_run_writes_new_bare_entities_to_its_namespace_but_no_new_version_of_a router.insert("artifacts", {"id": "ti_1__recording", "version": 1}) with pytest.raises(NamespaceWriteError, match="new version of 'shared-file'"): router.insert("artifacts", {"id": "shared-file", "version": 2}) - with pytest.raises(NamespaceWriteError, match="an @team-b entity"): - router.insert("artifacts", {"id": "@team-b/x", "version": 1}) + with pytest.raises(NamespaceWriteError, match="which is in 'team-b'"): + router.insert("artifacts", {"id": OTHER_ENV, "version": 1}) assert _ids(team, "artifacts") == {"ti_1__recording"} assert _ids(shared, "artifacts") == {"shared-file"} -def test_a_run_may_update_a_shared_entity_without_naming_its_namespace(stores): +def test_a_run_may_update_a_shared_entity(stores): router, shared, _ = stores router.insert("envs", {"id": "rocket", "version": 1, "metadata": {}}) with run_scope(TEAM_TASK): router.update("envs", Filter.of(id="rocket", version=1), UpdateSpec(set={"metadata.checked": True})) - with pytest.raises(NamespaceWriteError, match="reference another namespace's id"): - router.update("envs", Filter.of(id="rocket", version=1), UpdateSpec(set={"metadata.task": TEAM_TASK})) assert shared.find_one("envs", Filter.of(id="rocket"))["metadata"] == {"checked": True} @@ -225,15 +276,6 @@ def test_a_run_cannot_update_a_shared_run_record(stores): assert shared.find_one("task_instances", Filter.of(instance_id="ti_shared"))["status"] == "running" -def test_nothing_outside_a_namespace_names_its_ids(stores): - router, _, _ = stores - with pytest.raises(NamespaceWriteError, match="tasks\\[0\\]"): - router.insert("evals", {"id": "shared-eval", "version": 1, "tasks": [TEAM_TASK]}) - with pytest.raises(NamespaceWriteError, match="another namespace's id"): - router.insert("evals", {"id": "@team-b/eval", "version": 1, "tasks": [TEAM_TASK]}) - router.insert("tasks", {"id": TEAM_TASK, "version": 1, "env_id": "rocket"}) - - def test_local_routing_runs_on_top_of_configured_namespaces(stores, configured): enable_namespace_routing() router = configured.get_document_store() @@ -255,7 +297,7 @@ def test_a_runs_objects_go_to_its_namespace_under_its_prefix(configured, tmp_pat with pytest.raises(NamespaceWriteError, match="writes objects only to its own store"): store.signed_put_url(shared_url) other = configured.namespace_object_store("team-b").put("x", b"b") - with pytest.raises(NamespaceWriteError, match="an @team-b object"): + with pytest.raises(NamespaceWriteError, match="an object of 'team-b'"): store.get(other) assert (tmp_path / "restricted" / "team-a" / "trajectories" / "ti_1.json").read_bytes() == b"{}" @@ -267,6 +309,7 @@ def test_a_runs_objects_go_to_its_namespace_under_its_prefix(configured, tmp_pat def test_entity_objects_follow_the_entity_and_the_run(configured): team = configured.namespace_object_store("team-a") assert configured.get_object_store_for(TEAM_ENV) is team + assert configured.get_object_store_for(f"{TEAM_ENV}__files__ab12") is team assert configured.get_object_store_for("bare") is configured.get_object_store() with run_scope(TEAM_TASK): assert configured.get_object_store_for("bare") is team @@ -277,22 +320,23 @@ def test_an_object_url_is_recorded_only_in_its_namespace(configured): shared_url = configured.get_object_store().object_url("artifacts/x") configured.check_object_url(TEAM_ENV, team_url) configured.check_object_url("bare", shared_url) - with pytest.raises(ValueError, match="keeps its objects in the @team-a"): + with pytest.raises(ValueError, match="keeps its objects in the 'team-a'"): configured.check_object_url(TEAM_ENV, shared_url) - with pytest.raises(ValueError, match="an @team-a object"): + with pytest.raises(ValueError, match="an object of 'team-a'"): configured.check_object_url("bare", team_url) - with pytest.raises(ValueError, match="keeps its objects in the @team-a"): - configured.check_object_url("@team-a/y", configured.namespace_object_store("team-b").object_url("y")) + with pytest.raises(ValueError, match="keeps its objects in the 'team-a'"): + configured.check_object_url(f"{TEAM_ENV}__y", configured.namespace_object_store("team-b").object_url("y")) def test_a_namespaced_artifact_round_trips_through_its_own_stores(stores, configured, tmp_path): _, shared, team = stores - stored = FileArtifact.put_bytes(TEAM_ENV + "-file", description="d", filename="f.txt", content=b"hi") + file_id = f"{TEAM_ENV}__readme" + stored = FileArtifact.put_bytes(file_id, description="d", filename="f.txt", content=b"hi") assert stored.object_url.startswith(configured.namespace_object_store("team-a").object_url("")) - assert _ids(team, "artifacts") == {TEAM_ENV + "-file"} + assert _ids(team, "artifacts") == {file_id} assert _ids(shared, "artifacts") == set() - assert get_artifact_store().get(TEAM_ENV + "-file").object_url == stored.object_url + assert get_artifact_store().get(file_id).object_url == stored.object_url assert configured.get_object_store_at(stored.object_url).get(stored.object_url) == b"hi" assert list((tmp_path / "shared-objects").rglob("f.txt")) == [] diff --git a/tst/unit/store/ids_test.py b/tst/unit/store/ids_test.py index 43d2138d4..985bfd3b2 100644 --- a/tst/unit/store/ids_test.py +++ b/tst/unit/store/ids_test.py @@ -56,6 +56,7 @@ "", " ", "__", + "@acme/tickets", "@LOCAL/upper-namespace", ] @@ -121,13 +122,6 @@ def test_a_malformed_local_id_is_not_encoded_as_one(): assert key_segment("@local") == "@local" -def test_an_id_in_another_namespace_encodes_under_that_namespace(): - segment = key_segment("@acme/tickets") - assert segment == "acme/tickets-" + hashlib.sha256(b"@acme/tickets").hexdigest()[:12] - assert _OCI_REPOSITORY.fullmatch(image_repository("@acme/tickets")) - assert fs_safe("@acme/tickets") == "@acme/tickets" - - @pytest.mark.parametrize("entity_id", [ "@local/~/work/triage/tickets", "@local/a", From 78e9357890bb937212b15fa1b047ce77e0fa49a6 Mon Sep 17 00:00:00 2001 From: Yzabelle Go Date: Fri, 13 Nov 2026 12:08:48 -0500 Subject: [PATCH 3/3] fix(store): refuse a run's upload for another namespace's entity; read every store a multi-id filter names Co-Authored-By: Claude Opus 5.5 --- src/agent_env/config/runtime.py | 14 +++++++++-- src/agent_env/store/routing.py | 28 +++++++++++++++------ tst/unit/store/configured_namespace_test.py | 23 ++++++++++++++++- 3 files changed, 55 insertions(+), 10 deletions(-) diff --git a/src/agent_env/config/runtime.py b/src/agent_env/config/runtime.py index a3f6af884..3fee529fa 100644 --- a/src/agent_env/config/runtime.py +++ b/src/agent_env/config/runtime.py @@ -565,6 +565,16 @@ def namespace_resolver(self) -> Any: dict(section), NamespaceResolver, secret_resolver=self._resolve_secret)) return self._namespaced["resolver"][1] + def _writing_namespace(self, entity_id: str) -> Optional[str]: + """The namespace ``entity_id``'s objects are written to: its own, else the running task's. A run in one + namespace may not write another's, so this refuses before any bytes are uploaded.""" + from agent_env.store import routing + + placed, run = self.namespace_of(entity_id, fresh=True), self.run_namespace() + if placed is not None and run is not None and placed != run: + raise routing.NamespaceWriteError(f"a run in {run!r} can't write {entity_id!r}, which is in {placed!r}") + return placed or run + def _checked(self, namespace: Optional[str], what: str) -> Optional[str]: if namespace is not None and namespace not in self.configured_namespaces(): raise ConfigError(f"the namespace resolver placed {what!r} in {namespace!r}, which no [namespaces] table configures") @@ -795,7 +805,7 @@ def get_object_store_for(self, entity_id: str) -> ObjectStore: from agent_env.store import routing from agent_env.store.ids import is_local_id - if (namespace := self.namespace_of(entity_id, fresh=True) or self.run_namespace()) is not None: + if (namespace := self._writing_namespace(entity_id)) is not None: return self.namespace_object_store(namespace) configured, local = self._namespace_stores("object") if local is None: @@ -828,7 +838,7 @@ def check_object_url(self, entity_id: str, object_url: str) -> None: from agent_env.store import routing from agent_env.store.ids import is_local_id - namespace = self.namespace_of(entity_id, fresh=True) or self.run_namespace() + namespace = self._writing_namespace(entity_id) owner = self._namespace_owning(object_url) if namespace != owner and namespace is not None: raise ValueError(f"{entity_id!r} keeps its objects in the {namespace!r} namespace's store, not {object_url!r}") diff --git a/src/agent_env/store/routing.py b/src/agent_env/store/routing.py index bc54a529b..11c2ecea6 100644 --- a/src/agent_env/store/routing.py +++ b/src/agent_env/store/routing.py @@ -667,8 +667,9 @@ def _run(self) -> Optional[str]: raise ConfigError(f"the namespace resolver placed this run in {namespace!r}, which no [namespaces] table configures") return namespace - def _pinned(self, collection: str, filter: Filter, *, fresh: bool = False) -> Optional[str]: - """The one namespace every id the filter pins is in: by ``id`` for an entity, by an owner field for a record.""" + def _pinned_namespaces(self, collection: str, filter: Filter, *, fresh: bool = False) -> Optional[list[Optional[str]]]: + """The namespaces of the ids the filter pins, None for an id the resolver places in none: by ``id`` for an + entity, by the first owner field pinning one for a record. None when it pins no id.""" fields = ("id",) if is_entity_collection(collection) else _NAMESPACE_OWNER_FIELDS for field in fields: values: list[Any] = [] @@ -677,11 +678,16 @@ def _pinned(self, collection: str, filter: Filter, *, fresh: bool = False) -> Op values.append(pred.value) elif isinstance(pred, In): values.extend(pred.values) - namespaces = {self._namespace_of(value, fresh=fresh) for value in values} - if values and len(namespaces) == 1 and (namespace := namespaces.pop()) is not None: - return namespace + namespaces = list(dict.fromkeys(self._namespace_of(value, fresh=fresh) for value in values)) + if values and namespaces != [None]: + return namespaces return None + def _pinned(self, collection: str, filter: Filter, *, fresh: bool = False) -> Optional[str]: + """The one namespace every id the filter pins is in, or None.""" + namespaces = self._pinned_namespaces(collection, filter, fresh=fresh) + return namespaces[0] if namespaces is not None and len(namespaces) == 1 else None + def _visible(self) -> list[DocumentStore]: """The stores an unplaced read sees, the running task's namespace first.""" run = self._run() @@ -689,8 +695,16 @@ def _visible(self) -> list[DocumentStore]: return [*([self.namespaces[run]] if run else []), self.configured, *(self.namespaces[name] for name in opened)] def _readers(self, collection: str, filter: Filter) -> list[DocumentStore]: - pinned = self._pinned(collection, filter) - return [self.namespaces[pinned]] if pinned is not None else self._visible() + """The stores of the namespaces the filter's ids are in, an unplaced id's being those an unplaced read sees.""" + namespaces = self._pinned_namespaces(collection, filter) + if namespaces is None: + return self._visible() + stores: list[DocumentStore] = [] + for namespace in namespaces: + for store in ([self.namespaces[namespace]] if namespace is not None else self._visible()): + if all(store is not held for held in stores): + stores.append(store) + return stores def _read(self, store: DocumentStore, read: Callable[[], _T]) -> _T: return read() diff --git a/tst/unit/store/configured_namespace_test.py b/tst/unit/store/configured_namespace_test.py index 4498c818c..524715f66 100644 --- a/tst/unit/store/configured_namespace_test.py +++ b/tst/unit/store/configured_namespace_test.py @@ -10,7 +10,7 @@ from agent_env.artifact.store import get_artifact_store from agent_env.config import get_config, reset_config from agent_env.config.errors import ConfigError -from agent_env.store import Filter, LocalSqliteDocumentStore, UpdateSpec +from agent_env.store import Filter, In, LocalSqliteDocumentStore, UpdateSpec from agent_env.store.ids import id_bases, validate_namespace_name from agent_env.store.namespace_resolver import NamespaceResolver, TableNamespaceResolver from agent_env.store.object_store import LocalFilesystemObjectStore, ObjectStore @@ -260,6 +260,27 @@ def test_a_run_writes_new_bare_entities_to_its_namespace_but_no_new_version_of_a assert _ids(shared, "artifacts") == {"shared-file"} +def test_a_read_of_ids_in_several_stores_finds_each(stores): + router, _, _ = stores + router.insert("artifacts", {"id": "shared-file", "version": 1, "type": "file"}) + router.insert("artifacts", {"id": TEAM_ENV, "version": 1, "type": "file"}) + router.insert("artifacts", {"id": OTHER_ENV, "version": 1, "type": "file"}) + + found = router.query("artifacts", Filter().where("id", In(["shared-file", TEAM_ENV, OTHER_ENV]))) + assert {doc["id"] for doc in found} == {"shared-file", TEAM_ENV, OTHER_ENV} + found = router.latest_per_id("artifacts", Filter().where("id", In([TEAM_ENV, "shared-file"]))) + assert {doc["id"] for doc in found} == {TEAM_ENV, "shared-file"} + + +def test_a_run_uploads_no_bytes_for_another_namespaces_entity(configured, tmp_path): + with run_scope(TEAM_TASK): + with pytest.raises(NamespaceWriteError, match="which is in 'team-b'"): + FileArtifact.put_bytes(OTHER_ENV + "__f", description="d", filename="f.txt", content=b"x") + with pytest.raises(NamespaceWriteError, match="which is in 'team-b'"): + configured.check_object_url(OTHER_ENV, configured.namespace_object_store("team-b").object_url("y")) + assert list((tmp_path / "restricted").rglob("f.txt")) == [] + + def test_a_run_may_update_a_shared_entity(stores): router, shared, _ = stores router.insert("envs", {"id": "rocket", "version": 1, "metadata": {}})