Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions .github/workflows/ci_python.yml
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,17 @@ jobs:
set -euo pipefail
just --set no_uv true test-python

- name: Test skill loading with pinned native OpenClaw
if: matrix.platform == 'linux-amd64' && matrix.python-version == '3.12'
run: |
set -euo pipefail
openclaw_root="$RUNNER_TEMP/fabric-openclaw-skills"
npm exec --yes --package=npm@11.16.0 -- npm install --global \
--prefix "$openclaw_root" --no-audit --no-fund \
--allow-scripts=openclaw openclaw@2026.9.34
export OPENCLAW_TEST_PACKAGE="$openclaw_root/lib/node_modules/openclaw"
uv run --no-sync pytest tests/e2e/test_openclaw_skills.py -q

- name: Test Kiro health with the pinned native CLI
if: matrix.platform == 'linux-amd64' && matrix.python-version == '3.12'
env:
Expand Down
37 changes: 37 additions & 0 deletions tests/_utils/openclaw_skills.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import assert from "node:assert/strict";
import { readFileSync, readdirSync } from "node:fs";
import { join } from "node:path";
import { pathToFileURL } from "node:url";

// The pinned release exposes its native loader through a hashed bundle facade.
// Do not replace discovery or file reading with a test implementation.
const dist = join(process.argv[2], "dist");
const loaders = [];
for (const file of readdirSync(dist).filter((name) =>
/^workspace-skill-loader-.*\.mjs$/.test(name),
)) {
const module = await import(pathToFileURL(join(dist, file)).href);
if (typeof module.loadWorkspaceSkills === "function") {
loaders.push(module.loadWorkspaceSkills);
}
}
assert.equal(
loaders.length,
1,
"the pinned package must expose one native workspace loader",
);
const config = JSON.parse(readFileSync(process.argv[3], "utf8"));
const entries = loaders[0](config.agents.defaults.workspace, { config });
process.stdout.write(
JSON.stringify(
entries.map(({ skill }) => ({
name: skill.name,
source: skill.source,
path: skill.filePath,
contentHash: skill.contentHash,
})),
),
);
49 changes: 34 additions & 15 deletions tests/e2e/test_hermes_e2e.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
import pytest
import requests
import yaml
from _utils.utils import assert_atof_model, assert_atof_skill_selection
from _utils.utils import assert_atof_model, assert_atof_skill_selection, atof_records

from examples.code_review_agent import (
hermes_config,
Expand Down Expand Up @@ -73,6 +73,7 @@ async def test_hermes_persistent_host_with_relay(
code_review_agent_dir: Path,
api_server: str,
):
"""Verify reused Hermes sessions retain Relay artifacts across invocations."""
os.environ["ADAPTER_PYTHON"] = sys.executable
config = with_relay(hermes_config())
config.models["default"].base_url = f"{api_server}/v1"
Expand Down Expand Up @@ -104,8 +105,7 @@ async def test_hermes_persistent_host_with_relay(
]
assert (
sum(
record["name"] == "hermes.session"
and record.get("scope_category") == "end"
record["name"] == "hermes.session" and record.get("scope_category") == "end"
for record in atof_records
)
== 2
Expand Down Expand Up @@ -266,6 +266,7 @@ async def test_skill_selection(
default_skill: Path,
alternate_skill: Path,
):
"""Verify Hermes returns the full supplied skill through a successful tool call."""
os.environ["ADAPTER_PYTHON"] = sys.executable
config = with_relay(hermes_config())
config.models["default"].model = "fabric-echo"
Expand All @@ -279,18 +280,18 @@ async def test_skill_selection(
if skill == "alternate":
config.add_skill_path(alternate_skill)

if skill is not None:
scenario_response = requests.post(
f"{api_server}/_scenario",
json={
"tool_call": {
"name": "skill_view",
"arguments": {"name": skill},
}
},
timeout=5,
)
scenario_response.raise_for_status()
tool_name = "skill_view" if skill is not None else "skills_list"
scenario_response = requests.post(
f"{api_server}/_scenario",
json={
"tool_call": {
"name": tool_name,
"arguments": {"name": skill} if skill is not None else {},
}
},
timeout=5,
)
scenario_response.raise_for_status()

result = await Fabric().run(
config,
Expand All @@ -300,6 +301,24 @@ async def test_skill_selection(

assert result["status"] == "succeeded", result.to_mapping()
assert_atof_skill_selection(result["output"], skill)
skill_results = [
json.loads(record["data"])
for record in atof_records(result["output"])
if record["kind"] == "scope"
and record["category"] == "tool"
and record["name"] == tool_name
and record["scope_category"] == "end"
]
assert len(skill_results) == 1, skill_results
loaded = skill_results[0]
assert loaded["success"] is True, loaded
if skill is None:
assert not {"default", "alternate"}.intersection(
entry["name"] for entry in loaded["skills"]
), loaded
else:
skill_dir = {"default": default_skill, "alternate": alternate_skill}[skill]
assert loaded["content"] == (skill_dir / "SKILL.md").read_text(), loaded


@pytest.mark.usefixtures("mock_nvidia_api_key", "nemo_relay")
Expand Down
101 changes: 101 additions & 0 deletions tests/e2e/test_openclaw_skills.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Fabric configuration through the installed native loader, without inference.

OPENCLAW_TEST_PACKAGE selects the pinned package installed by the native CI step.
The loader hashes the bytes it reads; this does not prove model use or mounts.
"""

import hashlib
import json
import os
from pathlib import Path
import subprocess

import pytest
from nemo_fabric_adapter_contract.models import AgentConfig, RuntimeContext
from nemo_fabric_adapters.openclaw import adapter


@pytest.fixture(name="openclaw_package")
def openclaw_package_fixture() -> Path:
"""Require a supported native package when the dedicated CI step enables it."""
configured = os.environ.get("OPENCLAW_TEST_PACKAGE")
if configured is None:
pytest.skip("OPENCLAW_TEST_PACKAGE must name the installed native package")
package = Path(configured).resolve()
manifest = json.loads((package / "package.json").read_text())
assert manifest["version"] in adapter.SUPPORTED_OPENCLAW_VERSIONS
return package


@pytest.mark.parametrize("skill", ["default", "alternate", None])
def test_configured_skill_is_loaded_by_native_openclaw(
tmp_path: Path,
openclaw_package: Path,
default_skill: Path,
alternate_skill: Path,
skill: str | None,
):
"""Check native selection, source path, and content hash for supplied skills."""
sources = {"default": default_skill, "alternate": alternate_skill}
workspace = tmp_path / "workspace"
workspace.mkdir()
config = AgentConfig.from_mapping(
{
"models": {"default": {"provider": "test", "model": "unused"}},
"skills": {"paths": [sources[skill]] if skill else []},
}
)
context = RuntimeContext.from_mapping(
{
"runtime_id": "skill-loading",
"invocation_id": "skill-loading",
"request_id": "skill-loading",
"environment": {
"environment_id": "skill-loading",
"provider": "local",
"control_location": "in_env_control",
"workspace": str(workspace),
"env": {},
"ownership": "caller_owned",
},
"artifacts": {},
}
)
native = adapter._openclaw_config(
config, context, base_dir=tmp_path, port=18789, token_env="UNUSED_TEST_TOKEN"
)
config_path = tmp_path / "openclaw.json"
config_path.write_text(json.dumps(native))
home = tmp_path / "home"
home.mkdir()
probe = Path(__file__).parents[1] / "_utils" / "openclaw_skills.mjs"
result = subprocess.run(
["node", str(probe), str(openclaw_package), str(config_path)],
cwd=workspace,
env={
"PATH": os.environ["PATH"],
"HOME": str(home),
"OPENCLAW_HOME": str(home),
"OPENCLAW_STATE_DIR": str(home / ".openclaw"),
},
capture_output=True,
text=True,
timeout=30,
)
assert result.returncode == 0, result.stderr
loaded = [entry for entry in json.loads(result.stdout) if entry["name"] in sources]
if skill is None:
assert loaded == []
else:
source = (sources[skill] / "SKILL.md").resolve()
assert loaded == [
{
"name": skill,
"source": "openclaw-extra",
"path": str(source),
"contentHash": hashlib.sha256(source.read_bytes()).hexdigest(),
}
]
Loading