From 8ac276d8f7fb90a71aeab0bc36745820ffcf5a51 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 1 Nov 2026 05:18:01 -0700 Subject: [PATCH 1/5] feat: keep Hermes's home in the agent's state folder The Hermes adapter now accepts `runtime.keep_state`. When the runtime context has a state folder, the adapter uses it as the Hermes home, so every runtime of the agent uses the same session database, memory, skills and logs; otherwise the home stays `/.fabric/hermes/runtimes/`. Each runtime still starts a new session, and earlier sessions stay in the database. The API server's token moves with the home, to `interfaces/api.token`. The two descriptor copies and the catalog are mechanical. Signed-off-by: Carlos Villela --- .../python/hermes/hermes.fabric-adapter.json | 3 ++- .../nemo_fabric_adapters/hermes/adapter.py | 22 ++++++++++++++----- .../hermes/hermes.fabric-adapter.json | 3 ++- .../harness/hermes/hermes-api.mdx | 2 +- docs/integrations/harness/hermes/hermes.mdx | 2 +- docs/sdk/python.mdx | 2 +- .../hermes/hermes.fabric-adapter.json | 3 ++- tests/adapters/test_hermes_adapter.py | 19 ++++++++++++++++ 8 files changed, 44 insertions(+), 12 deletions(-) diff --git a/adapters/python/hermes/hermes.fabric-adapter.json b/adapters/python/hermes/hermes.fabric-adapter.json index 458ce7fdf..5071a5318 100644 --- a/adapters/python/hermes/hermes.fabric-adapter.json +++ b/adapters/python/hermes/hermes.fabric-adapter.json @@ -84,7 +84,7 @@ "port" ], "additionalProperties": false, - "description": "Hermes's API server, run by a gateway child. After every enabled interface starts, the adapter writes its bearer token to /.fabric/hermes/runtimes//interfaces/api.token." + "description": "Hermes's API server, run by a gateway child. After every enabled interface starts, the adapter writes its bearer token to interfaces/api.token in the Hermes home: /.fabric/hermes/runtimes/, or the agent's state folder with runtime.keep_state." }, "dashboard": { "type": "object", @@ -249,6 +249,7 @@ "models.max_tokens", "instructions.system", "runtime.max_turns", + "runtime.keep_state", "tools.enabled", "tools.blocked", "mcp", diff --git a/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py b/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py index ac586fb71..496d435f6 100755 --- a/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py +++ b/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py @@ -132,12 +132,8 @@ async def start(self, payload: dict[str, Any]) -> None: model_config = configuration._selected_model(agent_config) self._model_config = model_config self._runtime_id = runtime_context.runtime_id - self._hermes_home = ( - _artifact_root(runtime_context, common_utils.base_dir(payload)) - / ".fabric" - / "hermes" - / "runtimes" - / runtime_context.runtime_id + self._hermes_home = _hermes_home( + runtime_context, common_utils.base_dir(payload) ) self._hermes_home.mkdir(parents=True, exist_ok=True) os.environ["HOME"] = str(self._hermes_home) @@ -831,6 +827,20 @@ async def stop(self) -> None: ) from errors[0] +def _hermes_home(runtime_context: RuntimeContext, base_dir: str) -> Path: + """Return the agent's state folder when Fabric keeps state across runtimes, + otherwise this runtime's own folder.""" + if runtime_context.state_dir: + return Path(runtime_context.state_dir) + return ( + _artifact_root(runtime_context, base_dir) + / ".fabric" + / "hermes" + / "runtimes" + / runtime_context.runtime_id + ) + + def _artifact_root(runtime_context: RuntimeContext, base_dir: str) -> Path: root = runtime_context.artifacts.root if root: diff --git a/crates/fabric-cli/assets/adapters/hermes/hermes.fabric-adapter.json b/crates/fabric-cli/assets/adapters/hermes/hermes.fabric-adapter.json index 458ce7fdf..5071a5318 100644 --- a/crates/fabric-cli/assets/adapters/hermes/hermes.fabric-adapter.json +++ b/crates/fabric-cli/assets/adapters/hermes/hermes.fabric-adapter.json @@ -84,7 +84,7 @@ "port" ], "additionalProperties": false, - "description": "Hermes's API server, run by a gateway child. After every enabled interface starts, the adapter writes its bearer token to /.fabric/hermes/runtimes//interfaces/api.token." + "description": "Hermes's API server, run by a gateway child. After every enabled interface starts, the adapter writes its bearer token to interfaces/api.token in the Hermes home: /.fabric/hermes/runtimes/, or the agent's state folder with runtime.keep_state." }, "dashboard": { "type": "object", @@ -249,6 +249,7 @@ "models.max_tokens", "instructions.system", "runtime.max_turns", + "runtime.keep_state", "tools.enabled", "tools.blocked", "mcp", diff --git a/docs/integrations/harness/hermes/hermes-api.mdx b/docs/integrations/harness/hermes/hermes-api.mdx index a0aef2b37..1db511872 100644 --- a/docs/integrations/harness/hermes/hermes-api.mdx +++ b/docs/integrations/harness/hermes/hermes-api.mdx @@ -45,7 +45,7 @@ This disables embedded SDK execution explicitly. Omit `sdk` to also start the em The API server binds to `127.0.0.1` at its configured port. After every enabled interface passes readiness, the adapter writes a per-start bearer token with mode `0600` to: ```text -/.fabric/hermes/runtimes//interfaces/api.token +/interfaces/api.token ``` Clients calling the native API must read the generated token file and include `Authorization: Bearer ` in each HTTP request. Only the API gateway child receives it in its environment. Processes running as the runtime's user, including agent tools, can read the token file. diff --git a/docs/integrations/harness/hermes/hermes.mdx b/docs/integrations/harness/hermes/hermes.mdx index 9e63b16b7..9b4f46741 100644 --- a/docs/integrations/harness/hermes/hermes.mdx +++ b/docs/integrations/harness/hermes/hermes.mdx @@ -120,7 +120,7 @@ Model credentials are supplied through child process environments, not public at ### Shared Runtime Home -The runtime home is `/.fabric/hermes/runtimes/`. Agents share persisted resources in this directory while retaining separate live conversations. Keeping these files does not establish cross-runtime resume. +The default Hermes home is `/.fabric/hermes/runtimes/`. With `runtime.keep_state`, the Hermes home is `/.fabric/state/nvidia.fabric.hermes/`, so runtimes of the agent share the session database, memory, skills, and logs. The SDK starts a new conversation for each runtime, and earlier sessions remain in the database. The adapter rewrites `config.yaml` at each start. ### Process Ownership diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 9335d94b6..2c4cfdade 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -902,7 +902,7 @@ Only one runtime can use the folder at a time: `Fabric.start_runtime(...)` fails while another runtime of the agent runs. A service and one runtime connected to it share the folder; the service holds it until `Service.release()`, even with no runtime connected. On Linux and macOS, each adapter host inherits a descriptor for the state lock, including a runtime connected to a service. The folder stays locked while any process holds an open descriptor for that lock. A completed stop closes the runtime's descriptors, and the operating system releases the lock after all descriptors close. On Windows, adapter hosts do not inherit the lock, so only the NeMo Fabric process holds the lock. If the NeMo Fabric process exits while an adapter host is still running, the folder becomes available. If all descriptors close while a native process started in a separate session keeps running, the folder becomes available while that native process can still write. Planning fails without an artifact directory, or for an -adapter that doesn't declare `runtime.keep_state`. +adapter that doesn't declare `runtime.keep_state`. The Hermes Agent adapter declares `runtime.keep_state`. ## Runtime Resource Snapshots diff --git a/examples/harbor/swebench/adapters/hermes/hermes.fabric-adapter.json b/examples/harbor/swebench/adapters/hermes/hermes.fabric-adapter.json index 458ce7fdf..5071a5318 100644 --- a/examples/harbor/swebench/adapters/hermes/hermes.fabric-adapter.json +++ b/examples/harbor/swebench/adapters/hermes/hermes.fabric-adapter.json @@ -84,7 +84,7 @@ "port" ], "additionalProperties": false, - "description": "Hermes's API server, run by a gateway child. After every enabled interface starts, the adapter writes its bearer token to /.fabric/hermes/runtimes//interfaces/api.token." + "description": "Hermes's API server, run by a gateway child. After every enabled interface starts, the adapter writes its bearer token to interfaces/api.token in the Hermes home: /.fabric/hermes/runtimes/, or the agent's state folder with runtime.keep_state." }, "dashboard": { "type": "object", @@ -249,6 +249,7 @@ "models.max_tokens", "instructions.system", "runtime.max_turns", + "runtime.keep_state", "tools.enabled", "tools.blocked", "mcp", diff --git a/tests/adapters/test_hermes_adapter.py b/tests/adapters/test_hermes_adapter.py index a93c24b5e..53f870bec 100644 --- a/tests/adapters/test_hermes_adapter.py +++ b/tests/adapters/test_hermes_adapter.py @@ -217,6 +217,7 @@ def test_descriptor_uses_the_typed_agent_config_contract(): "models.max_tokens", "instructions.system", "runtime.max_turns", + "runtime.keep_state", "tools.enabled", "tools.blocked", "mcp", @@ -1485,6 +1486,24 @@ def stop_after_environment_setup(*_args, **_kwargs): await adapter.HermesRuntime().start(payload) +@pytest.mark.parametrize("kept", [False, True]) +def test_hermes_home_is_the_agents_state_folder_when_fabric_keeps_state( + tmp_path: Path, kept: bool +): + state_dir = tmp_path / "artifacts/.fabric/state/nvidia.fabric.hermes/assistant" + context = _runtime_context(artifact_root="artifacts").to_mapping() + if kept: + context["state_dir"] = str(state_dir) + + home = adapter._hermes_home(RuntimeContext.from_mapping(context), str(tmp_path)) + + assert home == ( + state_dir + if kept + else tmp_path / "artifacts/.fabric/hermes/runtimes/runtime-1" + ) + + def test_artifact_root_resolves_relative_to_base_dir(tmp_path: Path): assert ( adapter._artifact_root( From dc3df71484f1f9b7b11ab9b6d1b6f0cbf9eec075 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 1 Nov 2026 17:05:15 -0700 Subject: [PATCH 2/5] fix: remove a stale Hermes API token from a kept home at start With runtime.keep_state, every runtime of an agent uses the same Hermes home. A runtime that ended without stop left interfaces/api.token there, and a later start that failed before writing a new token kept it. Start now removes that token first. The Hermes README also describes the kept home: runtimes then share the session database, and each still starts a new session. Signed-off-by: Carlos Villela --- adapters/python/hermes/README.md | 2 +- .../nemo_fabric_adapters/hermes/adapter.py | 6 ++++- .../harness/hermes/hermes-api.mdx | 2 ++ .../nemo_fabric_adapter_catalog/catalog.json | 5 +++-- tests/adapters/test_hermes_interfaces.py | 22 +++++++++++++++++++ 5 files changed, 33 insertions(+), 4 deletions(-) diff --git a/adapters/python/hermes/README.md b/adapters/python/hermes/README.md index d2a786dea..25ddcf4ae 100644 --- a/adapters/python/hermes/README.md +++ b/adapters/python/hermes/README.md @@ -151,7 +151,7 @@ harness = HarnessConfig( Managed native processes use the adapter's launcher to apply explicit instructions and behavioral settings before constructing agents. Explicit enabled tools, including an empty list, override native selection; a nonempty blocked list overrides native exclusions. Unspecified tool policies can differ between SDK and native paths. -The runtime home is `/.fabric/hermes/runtimes/`. The adapter writes native configuration and optional NeMo Relay plugin configuration there. When native interfaces are enabled, an owner-only `interfaces/agent-config.json` snapshot supplies the constructor mapping and is removed on stop. Model credentials are provided through child environments rather than public attachment metadata. +The default Hermes home is `/.fabric/hermes/runtimes/`. With `runtime.keep_state`, the adapter uses the agent state folder described in the [Hermes guide](../../../docs/integrations/harness/hermes/hermes.mdx#state-and-session-boundaries). The adapter writes native configuration and optional NeMo Relay plugin configuration there. When native interfaces are enabled, an owner-only `interfaces/agent-config.json` snapshot supplies the constructor mapping and is removed on stop. Model credentials are provided through child environments rather than public attachment metadata. The adapter leaves the installed Hermes source unchanged. Independently started processes do not automatically receive its mapping. Refer to [Shared Configuration and Ownership](../../../docs/integrations/harness/hermes/hermes.mdx#shared-configuration-and-ownership) for precedence and managed API gateway scope. diff --git a/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py b/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py index 496d435f6..7fd8e6984 100755 --- a/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py +++ b/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py @@ -136,6 +136,10 @@ async def start(self, payload: dict[str, Any]) -> None: runtime_context, common_utils.base_dir(payload) ) self._hermes_home.mkdir(parents=True, exist_ok=True) + # A runtime that ended without stop can leave its API token in a + # kept Hermes home, for a server that no longer runs. + token_path = self._hermes_home / "interfaces" / "api.token" + token_path.unlink(missing_ok=True) os.environ["HOME"] = str(self._hermes_home) os.environ["HERMES_HOME"] = str(self._hermes_home) os.environ.setdefault("HERMES_YOLO_MODE", "1") @@ -258,7 +262,7 @@ async def start(self, payload: dict[str, Any]) -> None: await self._interfaces.start( interface_ports, hermes_home=self._hermes_home, - token_path=self._hermes_home / "interfaces" / "api.token", + token_path=token_path, timeout=max( interfaces.startup_timeout(self._settings) - (asyncio.get_running_loop().time() - started_at), diff --git a/docs/integrations/harness/hermes/hermes-api.mdx b/docs/integrations/harness/hermes/hermes-api.mdx index 1db511872..17e952efd 100644 --- a/docs/integrations/harness/hermes/hermes-api.mdx +++ b/docs/integrations/harness/hermes/hermes-api.mdx @@ -48,6 +48,8 @@ The API server binds to `127.0.0.1` at its configured port. After every enabled /interfaces/api.token ``` +Before starting Hermes, the adapter removes any API token left by an earlier runtime. + Clients calling the native API must read the generated token file and include `Authorization: Bearer ` in each HTTP request. Only the API gateway child receives it in its environment. Processes running as the runtime's user, including agent tools, can read the token file. This token authenticates access to the API gateway. It is distinct from the model-provider API key and the TUI backend credential used by dashboard and terminal clients. API clients share the per-start token; it does not provide per-user isolation. diff --git a/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json b/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json index ee36b2836..a3c67923e 100644 --- a/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json +++ b/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json @@ -812,6 +812,7 @@ "models.max_tokens", "instructions.system", "runtime.max_turns", + "runtime.keep_state", "tools.enabled", "tools.blocked", "mcp", @@ -923,7 +924,7 @@ "properties": { "api": { "additionalProperties": false, - "description": "Hermes's API server, run by a gateway child. After every enabled interface starts, the adapter writes its bearer token to /.fabric/hermes/runtimes//interfaces/api.token.", + "description": "Hermes's API server, run by a gateway child. After every enabled interface starts, the adapter writes its bearer token to interfaces/api.token in the Hermes home: /.fabric/hermes/runtimes/, or the agent's state folder with runtime.keep_state.", "properties": { "enabled": { "default": true, @@ -1078,7 +1079,7 @@ "name": "nemo-fabric-adapters-hermes", "version": "0.7.0" }, - "sha256": "6b75b6803c8a4fac6cac5b96aa68b32e7a1b058a4c33f24041d22124b9caae85", + "sha256": "d4ffb1125719a5e7ddaf1f58eb45f4f7b10e877288abc2a2abcc848075192301", "source": "adapters/python/hermes/hermes.fabric-adapter.json" }, "nvidia.fabric.kilo": { diff --git a/tests/adapters/test_hermes_interfaces.py b/tests/adapters/test_hermes_interfaces.py index 53c2c39eb..2fe45757f 100644 --- a/tests/adapters/test_hermes_interfaces.py +++ b/tests/adapters/test_hermes_interfaces.py @@ -494,6 +494,28 @@ async def test_hermes_interface_port_in_use_fails_start( assert not _token_path(tmp_path).exists() +async def test_hermes_failed_interface_start_removes_a_token_left_in_kept_state( + tmp_path: Path, api_server: str +): + state_dir = tmp_path / "artifacts/.fabric/state/nvidia.fabric.hermes/assistant" + stale_token = state_dir / "interfaces" / "api.token" + stale_token.parent.mkdir(parents=True) + stale_token.write_text("token-from-a-crashed-runtime", encoding="utf-8") + runtime = adapter.HermesRuntime() + with socket.create_server(("127.0.0.1", 0)) as server: + payload = _payload( + tmp_path, + {"interfaces": {"api": {"port": server.getsockname()[1]}}}, + base_url=f"{api_server}/v1", + ) + payload["runtime_context"]["state_dir"] = str(state_dir) + with pytest.raises(adapter.lifecycle.LifecycleError) as caught: + await runtime.start(payload) + + assert caught.value.code == "hermes_interface_port_unavailable" + assert not stale_token.exists() + + def test_hermes_disabled_interface_is_not_started(): settings = { "interfaces": { From f2e59860dff976b0de9f2db9217811744fee7c0a Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Tue, 10 Nov 2026 14:54:51 -0700 Subject: [PATCH 3/5] fix: clean stale Hermes tokens before platform validation Signed-off-by: Sara Tadayon --- adapters/python/hermes/README.md | 2 +- .../nemo_fabric_adapters/hermes/adapter.py | 8 +- .../tutorials/adapter-descriptor.md | 2 + docs/sdk/python.mdx | 2 +- tests/adapters/test_hermes_adapter.py | 76 ++++++++++++++++++- tests/adapters/test_hermes_interfaces.py | 25 +++--- 6 files changed, 97 insertions(+), 18 deletions(-) diff --git a/adapters/python/hermes/README.md b/adapters/python/hermes/README.md index 25ddcf4ae..0f189a40a 100644 --- a/adapters/python/hermes/README.md +++ b/adapters/python/hermes/README.md @@ -151,7 +151,7 @@ harness = HarnessConfig( Managed native processes use the adapter's launcher to apply explicit instructions and behavioral settings before constructing agents. Explicit enabled tools, including an empty list, override native selection; a nonempty blocked list overrides native exclusions. Unspecified tool policies can differ between SDK and native paths. -The default Hermes home is `/.fabric/hermes/runtimes/`. With `runtime.keep_state`, the adapter uses the agent state folder described in the [Hermes guide](../../../docs/integrations/harness/hermes/hermes.mdx#state-and-session-boundaries). The adapter writes native configuration and optional NeMo Relay plugin configuration there. When native interfaces are enabled, an owner-only `interfaces/agent-config.json` snapshot supplies the constructor mapping and is removed on stop. Model credentials are provided through child environments rather than public attachment metadata. +The default Hermes home is `/.fabric/hermes/runtimes/`. With `runtime.keep_state`, the adapter uses the agent state folder described in the [Hermes guide](../../../docs/integrations/harness/hermes/hermes.mdx#shared-runtime-home). The adapter writes native configuration and optional NeMo Relay plugin configuration there. When native interfaces are enabled, an owner-only `interfaces/agent-config.json` snapshot supplies the constructor mapping and is removed on stop. Model credentials are provided through child environments rather than public attachment metadata. The adapter leaves the installed Hermes source unchanged. Independently started processes do not automatically receive its mapping. Refer to [Shared Configuration and Ownership](../../../docs/integrations/harness/hermes/hermes.mdx#shared-configuration-and-ownership) for precedence and managed API gateway scope. diff --git a/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py b/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py index 7fd8e6984..317de041b 100755 --- a/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py +++ b/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py @@ -125,10 +125,6 @@ async def start(self, payload: dict[str, Any]) -> None: "hermes_no_execution_path", "Enable the Hermes SDK or at least one native interface", ) - if interface_ports: - # Refuse before Hermes's setup, whose MCP discovery can take - # minutes and outlast the caller's start deadline. - interfaces.require_supported_platform() model_config = configuration._selected_model(agent_config) self._model_config = model_config self._runtime_id = runtime_context.runtime_id @@ -140,6 +136,10 @@ async def start(self, payload: dict[str, Any]) -> None: # kept Hermes home, for a server that no longer runs. token_path = self._hermes_home / "interfaces" / "api.token" token_path.unlink(missing_ok=True) + if interface_ports: + # Refuse before Hermes's setup, whose MCP discovery can take + # minutes and outlast the caller's start deadline. + interfaces.require_supported_platform() os.environ["HOME"] = str(self._hermes_home) os.environ["HERMES_HOME"] = str(self._hermes_home) os.environ.setdefault("HERMES_YOLO_MODE", "1") diff --git a/docs/adapter-contract/tutorials/adapter-descriptor.md b/docs/adapter-contract/tutorials/adapter-descriptor.md index c2c1b5c31..5b33d86ea 100644 --- a/docs/adapter-contract/tutorials/adapter-descriptor.md +++ b/docs/adapter-contract/tutorials/adapter-descriptor.md @@ -111,6 +111,8 @@ children that keep inherited file descriptors open: such a child keeps the agent's state locked after the host exits, and later runtimes of the agent fail to start. +A detached child that does not hold the lock can outlive the adapter host and access the state folder after another runtime acquires the lock. Ensure that all children stop accessing retained state before the lock is released. + When an adapter accepts `instructions.system`, declare the exact supported modes in `config.system_instruction_modes`. `replace` discards the harness default system instruction. `append` preserves the harness default and adds the diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 2c4cfdade..01c23a861 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -901,7 +901,7 @@ as `CON` or `NUL`. On Linux and macOS, only the folder's owner can read it. Only one runtime can use the folder at a time: `Fabric.start_runtime(...)` fails while another runtime of the agent runs. A service and one runtime connected to it share the folder; the service holds it until `Service.release()`, even with -no runtime connected. On Linux and macOS, each adapter host inherits a descriptor for the state lock, including a runtime connected to a service. The folder stays locked while any process holds an open descriptor for that lock. A completed stop closes the runtime's descriptors, and the operating system releases the lock after all descriptors close. On Windows, adapter hosts do not inherit the lock, so only the NeMo Fabric process holds the lock. If the NeMo Fabric process exits while an adapter host is still running, the folder becomes available. If all descriptors close while a native process started in a separate session keeps running, the folder becomes available while that native process can still write. Planning fails without an artifact directory, or for an +no runtime connected. On Linux and macOS, each adapter host inherits a descriptor for the state lock, including a runtime connected to a service. The folder stays locked while any process holds an open descriptor for that lock. A completed stop closes the runtime's descriptors, and the operating system releases the lock after all descriptors close. Stopping a standalone runtime normally releases the lock while the application continues running. Killing only the adapter host leaves the application's lock descriptor open. On Windows, adapter hosts do not inherit the lock, so only the NeMo Fabric process holds the lock. If the NeMo Fabric process exits while an adapter host is still running, the folder becomes available. If all descriptors close while a native process started in a separate session keeps running, the folder becomes available while that native process can still write. Planning fails without an artifact directory, or for an adapter that doesn't declare `runtime.keep_state`. The Hermes Agent adapter declares `runtime.keep_state`. ## Runtime Resource Snapshots diff --git a/tests/adapters/test_hermes_adapter.py b/tests/adapters/test_hermes_adapter.py index 53f870bec..eb237c491 100644 --- a/tests/adapters/test_hermes_adapter.py +++ b/tests/adapters/test_hermes_adapter.py @@ -1498,12 +1498,82 @@ def test_hermes_home_is_the_agents_state_folder_when_fabric_keeps_state( home = adapter._hermes_home(RuntimeContext.from_mapping(context), str(tmp_path)) assert home == ( - state_dir - if kept - else tmp_path / "artifacts/.fabric/hermes/runtimes/runtime-1" + state_dir if kept else tmp_path / "artifacts/.fabric/hermes/runtimes/runtime-1" ) +@pytest.mark.parametrize("kept", [False, True]) +async def test_hermes_sessions_survive_runtime_replacement_only_with_kept_state( + tmp_path: Path, api_server: str, kept: bool +): + os.environ["TEST_API_KEY"] = "fabric-test-model-key" + config = _agent_config( + { + "models": { + "default": { + "provider": "openai", + "model": "fabric-echo", + "base_url": f"{api_server}/v1", + "api_key_env": "TEST_API_KEY", + } + }, + "tools": {"enabled": []}, + } + ) + contexts = [] + for runtime_id in ("runtime-first", "runtime-second"): + context = _runtime_context( + runtime_id=runtime_id, + workspace=str(tmp_path), + artifact_root=str(tmp_path / "artifacts"), + ).to_mapping() + if kept: + context["state_dir"] = str(tmp_path / "kept-state") + contexts.append(RuntimeContext.from_mapping(context)) + first = adapter.HermesRuntime() + try: + await first.start( + { + "config": config, + "runtime_context": contexts[0].to_mapping(), + "base_dir": str(tmp_path), + } + ) + result = await first.invoke( + AgentRunRequest.from_mapping({"input": "retained session marker"}), + contexts[0], + ) + assert result.status is AgentRunStatus.SUCCEEDED + session_id = first._agent.session_id + assert first._session_db.get_session(session_id) is not None + saved_messages = first._session_db.get_messages(session_id) + assert any( + message["content"] == "retained session marker" + for message in saved_messages + ) + finally: + await first.stop() + + second = adapter.HermesRuntime() + try: + await second.start( + { + "config": config, + "runtime_context": contexts[1].to_mapping(), + "base_dir": str(tmp_path), + } + ) + if kept: + assert second._session_db.get_session(session_id) is not None + assert second._session_db.get_messages(session_id) == saved_messages + else: + assert second._session_db.get_session(session_id) is None + assert second._agent.session_id == "runtime-second" + assert second._conversation_history is None + finally: + await second.stop() + + def test_artifact_root_resolves_relative_to_base_dir(tmp_path: Path): assert ( adapter._artifact_root( diff --git a/tests/adapters/test_hermes_interfaces.py b/tests/adapters/test_hermes_interfaces.py index 2fe45757f..f949c9fb0 100644 --- a/tests/adapters/test_hermes_interfaces.py +++ b/tests/adapters/test_hermes_interfaces.py @@ -442,8 +442,9 @@ async def test_hermes_stop_cuts_off_an_in_flight_api_request( assert not _token_path(tmp_path).exists() +@pytest.mark.parametrize("kept", [False, True]) async def test_hermes_interfaces_fail_start_on_windows_before_any_child_starts( - tmp_path: Path, api_server: str, monkeypatch: pytest.MonkeyPatch + tmp_path: Path, api_server: str, monkeypatch: pytest.MonkeyPatch, kept: bool ): monkeypatch.setattr(interfaces, "POSIX", False) # start() stops whatever it started when it fails, so check the spawn @@ -455,20 +456,26 @@ async def test_hermes_interfaces_fail_start_on_windows_before_any_child_starts( write_config = MagicMock(wraps=configuration.write_hermes_config) monkeypatch.setattr(configuration, "write_hermes_config", write_config) runtime = adapter.HermesRuntime() + payload = _payload( + tmp_path, + {"interfaces": {"api": {"port": _free_port()}}}, + base_url=f"{api_server}/v1", + ) + token_path = _token_path(tmp_path) + if kept: + state_dir = tmp_path / "artifacts/.fabric/state/nvidia.fabric.hermes/assistant" + payload["runtime_context"]["state_dir"] = str(state_dir) + token_path = state_dir / "interfaces/api.token" + token_path.parent.mkdir(parents=True) + token_path.write_text("token-from-a-crashed-runtime", encoding="utf-8") with pytest.raises(adapter.lifecycle.LifecycleError) as caught: - await runtime.start( - _payload( - tmp_path, - {"interfaces": {"api": {"port": _free_port()}}}, - base_url=f"{api_server}/v1", - ) - ) + await runtime.start(payload) assert caught.value.code == "hermes_interfaces_unsupported_platform" spawn.assert_not_called() write_config.assert_not_called() - assert not _token_path(tmp_path).exists() + assert not token_path.exists() async def test_hermes_interface_port_in_use_fails_start( From 1f78f481bbaf2536387f9f8170a1d9863c750a40 Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Tue, 10 Nov 2026 15:40:10 -0700 Subject: [PATCH 4/5] fix: clear stale Hermes interface files before startup Signed-off-by: Sara Tadayon --- .../nemo_fabric_adapters/hermes/adapter.py | 13 +++- .../tutorials/adapter-descriptor.md | 11 +-- docs/integrations/harness/hermes/hermes.mdx | 4 +- docs/sdk/python.mdx | 14 ++-- tests/adapters/test_hermes_interfaces.py | 75 +++++++++++++++++++ 5 files changed, 98 insertions(+), 19 deletions(-) diff --git a/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py b/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py index 317de041b..9dda0920f 100755 --- a/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py +++ b/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py @@ -132,10 +132,17 @@ async def start(self, payload: dict[str, Any]) -> None: runtime_context, common_utils.base_dir(payload) ) self._hermes_home.mkdir(parents=True, exist_ok=True) - # A runtime that ended without stop can leave its API token in a - # kept Hermes home, for a server that no longer runs. + # An abruptly ended runtime can leave credentials and attachment + # records in a kept Hermes home. token_path = self._hermes_home / "interfaces" / "api.token" - token_path.unlink(missing_ok=True) + try: + for name in ("tui.json", "tui.token", "api.token", "agent-config.json"): + token_path.with_name(name).unlink(missing_ok=True) + except OSError as error: + raise lifecycle.LifecycleError( + "hermes_interface_cleanup_failed", + "Hermes interface files could not be removed", + ) from error if interface_ports: # Refuse before Hermes's setup, whose MCP discovery can take # minutes and outlast the caller's start deadline. diff --git a/docs/adapter-contract/tutorials/adapter-descriptor.md b/docs/adapter-contract/tutorials/adapter-descriptor.md index 5b33d86ea..f23112930 100644 --- a/docs/adapter-contract/tutorials/adapter-descriptor.md +++ b/docs/adapter-contract/tutorials/adapter-descriptor.md @@ -102,16 +102,7 @@ schema enum for the exact accepted values. Planning rejects configured normalized behavior outside the declared surface. NeMo Fabric does not silently remove unsupported fields. -Accept `runtime.keep_state` only when the adapter keeps all of its native -state in `RuntimeContext.state_dir` when that field is present. Every runtime -of the agent then receives the same folder, so a later runtime finds the -earlier runtime's sessions, memory, and other harness files. On Linux and -macOS, the adapter host inherits the folder's lock. Don't leave detached -children that keep inherited file descriptors open: such a child keeps the -agent's state locked after the host exits, and later runtimes of the agent -fail to start. - -A detached child that does not hold the lock can outlive the adapter host and access the state folder after another runtime acquires the lock. Ensure that all children stop accessing retained state before the lock is released. +Accept `runtime.keep_state` only when the adapter keeps all of its native state in `RuntimeContext.state_dir` when that field is present. Every runtime of the agent then receives the same folder, so a later runtime finds the earlier runtime's sessions, memory, and other harness files. On Linux and macOS, the adapter host inherits the folder's lock. Detached children with inherited lock descriptors can keep the state locked after the host exits and prevent later runtimes from starting. A detached child without the lock can outlive the host and access the state folder after another runtime acquires the lock. Ensure that all children stop accessing retained state before the lock is released. When an adapter accepts `instructions.system`, declare the exact supported modes in `config.system_instruction_modes`. `replace` discards the harness diff --git a/docs/integrations/harness/hermes/hermes.mdx b/docs/integrations/harness/hermes/hermes.mdx index 9b4f46741..57dcc40f3 100644 --- a/docs/integrations/harness/hermes/hermes.mdx +++ b/docs/integrations/harness/hermes/hermes.mdx @@ -108,7 +108,7 @@ Use a single `FabricConfig` to define supported settings for all four Hermes pat NeMo Fabric resolves `FabricConfig` into `AgentConfig`. The adapter translates it into Hermes's native configuration and agent settings, applying the same explicit instructions and behavioral settings to agents in managed native processes. -The adapter writes native `config.yaml` and optional NeMo Relay plugin configuration under the runtime's Hermes home. When native interfaces are enabled, it also writes an owner-only `interfaces/agent-config.json` snapshot for the managed processes. +The adapter writes native `config.yaml` and optional NeMo Relay plugin configuration under the Hermes home. When native interfaces are enabled, it also writes an owner-only `interfaces/agent-config.json` snapshot for the managed processes. Model credentials are supplied through child process environments, not public attachment metadata. For OpenAI without an explicit base URL, the adapter removes inherited `OPENAI_BASE_URL` from managed children so it cannot redirect the configured credential. @@ -122,6 +122,8 @@ Model credentials are supplied through child process environments, not public at The default Hermes home is `/.fabric/hermes/runtimes/`. With `runtime.keep_state`, the Hermes home is `/.fabric/state/nvidia.fabric.hermes/`, so runtimes of the agent share the session database, memory, skills, and logs. The SDK starts a new conversation for each runtime, and earlier sessions remain in the database. The adapter rewrites `config.yaml` at each start. +Startup removes stale `interfaces/api.token`, `interfaces/tui.json`, `interfaces/tui.token`, and `interfaces/agent-config.json` files from the Hermes home. A removal failure stops startup with `hermes_interface_cleanup_failed` before Hermes setup. + ### Process Ownership Managed native processes run through the adapter's launcher without modifying the installed Hermes source. Independently started processes do not automatically receive the adapter's configuration mapping. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 01c23a861..94610cd22 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -898,11 +898,15 @@ artifact directory, and `metadata.name` and the adapter ID must each be usable as one folder name on Linux, macOS, and Windows: no `/`, `\`, `<>:"|?*`, or control characters, no trailing space or `.`, and not a Windows device name such as `CON` or `NUL`. On Linux and macOS, only the folder's owner can read it. -Only one runtime can use the folder at a time: `Fabric.start_runtime(...)` fails -while another runtime of the agent runs. A service and one runtime connected to -it share the folder; the service holds it until `Service.release()`, even with -no runtime connected. On Linux and macOS, each adapter host inherits a descriptor for the state lock, including a runtime connected to a service. The folder stays locked while any process holds an open descriptor for that lock. A completed stop closes the runtime's descriptors, and the operating system releases the lock after all descriptors close. Stopping a standalone runtime normally releases the lock while the application continues running. Killing only the adapter host leaves the application's lock descriptor open. On Windows, adapter hosts do not inherit the lock, so only the NeMo Fabric process holds the lock. If the NeMo Fabric process exits while an adapter host is still running, the folder becomes available. If all descriptors close while a native process started in a separate session keeps running, the folder becomes available while that native process can still write. Planning fails without an artifact directory, or for an -adapter that doesn't declare `runtime.keep_state`. The Hermes Agent adapter declares `runtime.keep_state`. +Only one runtime can use the folder at a time: `Fabric.start_runtime(...)` fails while another runtime of the agent runs. A service and one connected runtime share the folder. The service holds the lock until `Service.release()`, even with no runtime connected. + +The operating system releases the lock after all copies of the lock have closed: + +- **Linux and macOS:** Each adapter host inherits a lock descriptor, including a runtime connected to a service. Stopping a standalone runtime normally releases the lock while the application continues running. Killing only the adapter host leaves the application's lock descriptor open. +- **Windows:** Adapter hosts do not inherit the lock. Only the NeMo Fabric process holds the lock, so the folder becomes available if that process exits while an adapter host is still running. +- **Detached processes:** A native process can continue accessing the state folder after all lock descriptors close. A replacement runtime can then acquire the lock while the detached process still writes to the folder. + +Planning fails without an artifact directory or for an adapter that doesn't declare `runtime.keep_state`. The Hermes Agent adapter declares `runtime.keep_state`. ## Runtime Resource Snapshots diff --git a/tests/adapters/test_hermes_interfaces.py b/tests/adapters/test_hermes_interfaces.py index f949c9fb0..4aec29f06 100644 --- a/tests/adapters/test_hermes_interfaces.py +++ b/tests/adapters/test_hermes_interfaces.py @@ -523,6 +523,81 @@ async def test_hermes_failed_interface_start_removes_a_token_left_in_kept_state( assert not stale_token.exists() +@pytest.mark.parametrize("kept", [False, True]) +async def test_sdk_only_start_removes_stale_interface_files( + tmp_path, api_server, monkeypatch, kept +): + payload = _payload(tmp_path, {}, base_url=f"{api_server}/v1") + home = _token_path(tmp_path).parent.parent + if kept: + home = tmp_path / "kept-state" + payload["runtime_context"]["state_dir"] = str(home) + interface_dir = home / "interfaces" + interface_dir.mkdir(parents=True) + stale_files = [ + interface_dir / name + for name in ("api.token", "tui.json", "tui.token", "agent-config.json") + ] + for path in stale_files: + path.write_text("stale interface data", encoding="utf-8") + unrelated = interface_dir / "keep.txt" + unrelated.write_text("retained data", encoding="utf-8") + spawn = AsyncMock() + monkeypatch.setattr(interfaces.asyncio, "create_subprocess_exec", spawn) + + runtime = adapter.HermesRuntime() + try: + await runtime.start(payload) + assert all(not path.exists() for path in stale_files) + assert unrelated.read_text(encoding="utf-8") == "retained data" + spawn.assert_not_called() + finally: + await runtime.stop() + + +@pytest.mark.parametrize( + "name", ["api.token", "tui.json", "tui.token", "agent-config.json"] +) +@pytest.mark.parametrize("failure", ["directory", "permission"]) +async def test_start_reports_interface_cleanup_failures_before_hermes_setup( + tmp_path, api_server, monkeypatch, name, failure +): + payload = _payload(tmp_path, {}, base_url=f"{api_server}/v1") + home = tmp_path / "kept-state" + payload["runtime_context"]["state_dir"] = str(home) + path = home / "interfaces" / name + path.parent.mkdir(parents=True) + if failure == "directory": + path.mkdir() + else: + path.write_text("stale interface data", encoding="utf-8") + unlink = Path.unlink + + def deny_unlink(candidate, *args, **kwargs): + if candidate == path: + raise PermissionError("PRIVATE_UNLINK_DETAIL") + return unlink(candidate, *args, **kwargs) + + monkeypatch.setattr(Path, "unlink", deny_unlink) + write_config = MagicMock(wraps=configuration.write_hermes_config) + monkeypatch.setattr(configuration, "write_hermes_config", write_config) + spawn = AsyncMock() + monkeypatch.setattr(interfaces.asyncio, "create_subprocess_exec", spawn) + + runtime = adapter.HermesRuntime() + try: + with pytest.raises(adapter.lifecycle.LifecycleError) as caught: + await runtime.start(payload) + finally: + await runtime.stop() + + assert caught.value.code == "hermes_interface_cleanup_failed" + assert isinstance(caught.value.__cause__, OSError) + assert "PRIVATE_UNLINK_DETAIL" not in str(caught.value) + write_config.assert_not_called() + spawn.assert_not_called() + + def test_hermes_disabled_interface_is_not_started(): settings = { "interfaces": { From 739027106f116a4d93a8ac007715c3df668d42fb Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Wed, 11 Nov 2026 08:33:25 -0700 Subject: [PATCH 5/5] docs: clarify Hermes retention and Relay configuration Signed-off-by: Sara Tadayon --- docs/integrations/harness/hermes/hermes.mdx | 2 +- docs/sdk/python.mdx | 2 ++ tests/adapters/test_hermes_adapter.py | 2 ++ tests/adapters/test_hermes_interfaces.py | 5 +++++ 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/integrations/harness/hermes/hermes.mdx b/docs/integrations/harness/hermes/hermes.mdx index 57dcc40f3..628d1bc5c 100644 --- a/docs/integrations/harness/hermes/hermes.mdx +++ b/docs/integrations/harness/hermes/hermes.mdx @@ -108,7 +108,7 @@ Use a single `FabricConfig` to define supported settings for all four Hermes pat NeMo Fabric resolves `FabricConfig` into `AgentConfig`. The adapter translates it into Hermes's native configuration and agent settings, applying the same explicit instructions and behavioral settings to agents in managed native processes. -The adapter writes native `config.yaml` and optional NeMo Relay plugin configuration under the Hermes home. When native interfaces are enabled, it also writes an owner-only `interfaces/agent-config.json` snapshot for the managed processes. +The adapter writes native `config.yaml` under the Hermes home. When NeMo Relay is enabled, the adapter creates a `relay-config` directory beside the file specified by `FABRIC_RELAY_CONFIG_PATH` and writes `plugins.toml` there. This location follows `FABRIC_RELAY_CONFIG_PATH` even with `runtime.keep_state` enabled. When native interfaces are enabled, it also writes an owner-only `interfaces/agent-config.json` snapshot for the managed processes. Model credentials are supplied through child process environments, not public attachment metadata. For OpenAI without an explicit base URL, the adapter removes inherited `OPENAI_BASE_URL` from managed children so it cannot redirect the configured credential. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 94610cd22..23e9ec581 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -908,6 +908,8 @@ The operating system releases the lock after all copies of the lock have closed: Planning fails without an artifact directory or for an adapter that doesn't declare `runtime.keep_state`. The Hermes Agent adapter declares `runtime.keep_state`. +Enabling `runtime.keep_state` preserves earlier Hermes sessions in the database. Each new Hermes SDK runtime starts a fresh conversation. + ## Runtime Resource Snapshots `Runtime.resource_snapshot()` returns a timestamped `RuntimeResourceSnapshot` diff --git a/tests/adapters/test_hermes_adapter.py b/tests/adapters/test_hermes_adapter.py index eb237c491..1d410a530 100644 --- a/tests/adapters/test_hermes_adapter.py +++ b/tests/adapters/test_hermes_adapter.py @@ -1490,6 +1490,7 @@ def stop_after_environment_setup(*_args, **_kwargs): def test_hermes_home_is_the_agents_state_folder_when_fabric_keeps_state( tmp_path: Path, kept: bool ): + """Select the shared state folder or the current runtime's home.""" state_dir = tmp_path / "artifacts/.fabric/state/nvidia.fabric.hermes/assistant" context = _runtime_context(artifact_root="artifacts").to_mapping() if kept: @@ -1506,6 +1507,7 @@ def test_hermes_home_is_the_agents_state_folder_when_fabric_keeps_state( async def test_hermes_sessions_survive_runtime_replacement_only_with_kept_state( tmp_path: Path, api_server: str, kept: bool ): + """Preserve native session records when runtimes share a state folder.""" os.environ["TEST_API_KEY"] = "fabric-test-model-key" config = _agent_config( { diff --git a/tests/adapters/test_hermes_interfaces.py b/tests/adapters/test_hermes_interfaces.py index 4aec29f06..36bc41397 100644 --- a/tests/adapters/test_hermes_interfaces.py +++ b/tests/adapters/test_hermes_interfaces.py @@ -446,6 +446,7 @@ async def test_hermes_stop_cuts_off_an_in_flight_api_request( async def test_hermes_interfaces_fail_start_on_windows_before_any_child_starts( tmp_path: Path, api_server: str, monkeypatch: pytest.MonkeyPatch, kept: bool ): + """Clear stale credentials and reject native interfaces before setup.""" monkeypatch.setattr(interfaces, "POSIX", False) # start() stops whatever it started when it fails, so check the spawn # itself rather than the children left afterwards. @@ -504,6 +505,7 @@ async def test_hermes_interface_port_in_use_fails_start( async def test_hermes_failed_interface_start_removes_a_token_left_in_kept_state( tmp_path: Path, api_server: str ): + """Remove a retained API token even when an occupied port blocks startup.""" state_dir = tmp_path / "artifacts/.fabric/state/nvidia.fabric.hermes/assistant" stale_token = state_dir / "interfaces" / "api.token" stale_token.parent.mkdir(parents=True) @@ -527,6 +529,7 @@ async def test_hermes_failed_interface_start_removes_a_token_left_in_kept_state( async def test_sdk_only_start_removes_stale_interface_files( tmp_path, api_server, monkeypatch, kept ): + """Remove stale native attachment files while preserving unrelated data.""" payload = _payload(tmp_path, {}, base_url=f"{api_server}/v1") home = _token_path(tmp_path).parent.parent if kept: @@ -562,6 +565,7 @@ async def test_sdk_only_start_removes_stale_interface_files( async def test_start_reports_interface_cleanup_failures_before_hermes_setup( tmp_path, api_server, monkeypatch, name, failure ): + """Return a stable error before setup when attachment cleanup fails.""" payload = _payload(tmp_path, {}, base_url=f"{api_server}/v1") home = tmp_path / "kept-state" payload["runtime_context"]["state_dir"] = str(home) @@ -574,6 +578,7 @@ async def test_start_reports_interface_cleanup_failures_before_hermes_setup( unlink = Path.unlink def deny_unlink(candidate, *args, **kwargs): + """Simulate a permission failure for the selected attachment.""" if candidate == path: raise PermissionError("PRIVATE_UNLINK_DETAIL") return unlink(candidate, *args, **kwargs)