diff --git a/.github/workflows/ci_python.yml b/.github/workflows/ci_python.yml index 72a0ec20c..b7ef8c7bc 100644 --- a/.github/workflows/ci_python.yml +++ b/.github/workflows/ci_python.yml @@ -122,6 +122,17 @@ jobs: set -euo pipefail just --set no_uv true test-python + - name: Test skill loading with pinned native OpenClaw + if: matrix.platform == 'linux-amd64' && matrix.python-version == '3.12' + run: | + set -euo pipefail + openclaw_root="$RUNNER_TEMP/fabric-openclaw-skills" + npm exec --yes --package=npm@11.16.0 -- npm install --global \ + --prefix "$openclaw_root" --no-audit --no-fund \ + --allow-scripts=openclaw openclaw@2026.9.34 + export OPENCLAW_TEST_PACKAGE="$openclaw_root/lib/node_modules/openclaw" + uv run --no-sync pytest tests/e2e/test_openclaw_skills.py -q + - name: Test Kiro health with the pinned native CLI if: matrix.platform == 'linux-amd64' && matrix.python-version == '3.12' env: diff --git a/tests/_utils/openclaw_skills.mjs b/tests/_utils/openclaw_skills.mjs new file mode 100644 index 000000000..94c7ad1ad --- /dev/null +++ b/tests/_utils/openclaw_skills.mjs @@ -0,0 +1,37 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { readFileSync, readdirSync } from "node:fs"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; + +// The pinned release exposes its native loader through a hashed bundle facade. +// Do not replace discovery or file reading with a test implementation. +const dist = join(process.argv[2], "dist"); +const loaders = []; +for (const file of readdirSync(dist).filter((name) => + /^workspace-skill-loader-.*\.mjs$/.test(name), +)) { + const module = await import(pathToFileURL(join(dist, file)).href); + if (typeof module.loadWorkspaceSkills === "function") { + loaders.push(module.loadWorkspaceSkills); + } +} +assert.equal( + loaders.length, + 1, + "the pinned package must expose one native workspace loader", +); +const config = JSON.parse(readFileSync(process.argv[3], "utf8")); +const entries = loaders[0](config.agents.defaults.workspace, { config }); +process.stdout.write( + JSON.stringify( + entries.map(({ skill }) => ({ + name: skill.name, + source: skill.source, + path: skill.filePath, + contentHash: skill.contentHash, + })), + ), +); diff --git a/tests/e2e/test_hermes_e2e.py b/tests/e2e/test_hermes_e2e.py index 20472ff62..08d334306 100644 --- a/tests/e2e/test_hermes_e2e.py +++ b/tests/e2e/test_hermes_e2e.py @@ -15,7 +15,7 @@ import pytest import requests import yaml -from _utils.utils import assert_atof_model, assert_atof_skill_selection +from _utils.utils import assert_atof_model, assert_atof_skill_selection, atof_records from examples.code_review_agent import ( hermes_config, @@ -73,6 +73,7 @@ async def test_hermes_persistent_host_with_relay( code_review_agent_dir: Path, api_server: str, ): + """Verify reused Hermes sessions retain Relay artifacts across invocations.""" os.environ["ADAPTER_PYTHON"] = sys.executable config = with_relay(hermes_config()) config.models["default"].base_url = f"{api_server}/v1" @@ -104,8 +105,7 @@ async def test_hermes_persistent_host_with_relay( ] assert ( sum( - record["name"] == "hermes.session" - and record.get("scope_category") == "end" + record["name"] == "hermes.session" and record.get("scope_category") == "end" for record in atof_records ) == 2 @@ -266,6 +266,7 @@ async def test_skill_selection( default_skill: Path, alternate_skill: Path, ): + """Verify Hermes returns the full supplied skill through a successful tool call.""" os.environ["ADAPTER_PYTHON"] = sys.executable config = with_relay(hermes_config()) config.models["default"].model = "fabric-echo" @@ -279,18 +280,18 @@ async def test_skill_selection( if skill == "alternate": config.add_skill_path(alternate_skill) - if skill is not None: - scenario_response = requests.post( - f"{api_server}/_scenario", - json={ - "tool_call": { - "name": "skill_view", - "arguments": {"name": skill}, - } - }, - timeout=5, - ) - scenario_response.raise_for_status() + tool_name = "skill_view" if skill is not None else "skills_list" + scenario_response = requests.post( + f"{api_server}/_scenario", + json={ + "tool_call": { + "name": tool_name, + "arguments": {"name": skill} if skill is not None else {}, + } + }, + timeout=5, + ) + scenario_response.raise_for_status() result = await Fabric().run( config, @@ -300,6 +301,24 @@ async def test_skill_selection( assert result["status"] == "succeeded", result.to_mapping() assert_atof_skill_selection(result["output"], skill) + skill_results = [ + json.loads(record["data"]) + for record in atof_records(result["output"]) + if record["kind"] == "scope" + and record["category"] == "tool" + and record["name"] == tool_name + and record["scope_category"] == "end" + ] + assert len(skill_results) == 1, skill_results + loaded = skill_results[0] + assert loaded["success"] is True, loaded + if skill is None: + assert not {"default", "alternate"}.intersection( + entry["name"] for entry in loaded["skills"] + ), loaded + else: + skill_dir = {"default": default_skill, "alternate": alternate_skill}[skill] + assert loaded["content"] == (skill_dir / "SKILL.md").read_text(), loaded @pytest.mark.usefixtures("mock_nvidia_api_key", "nemo_relay") diff --git a/tests/e2e/test_openclaw_skills.py b/tests/e2e/test_openclaw_skills.py new file mode 100644 index 000000000..13031861a --- /dev/null +++ b/tests/e2e/test_openclaw_skills.py @@ -0,0 +1,101 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Fabric configuration through the installed native loader, without inference. + +OPENCLAW_TEST_PACKAGE selects the pinned package installed by the native CI step. +The loader hashes the bytes it reads; this does not prove model use or mounts. +""" + +import hashlib +import json +import os +from pathlib import Path +import subprocess + +import pytest +from nemo_fabric_adapter_contract.models import AgentConfig, RuntimeContext +from nemo_fabric_adapters.openclaw import adapter + + +@pytest.fixture(name="openclaw_package") +def openclaw_package_fixture() -> Path: + """Require a supported native package when the dedicated CI step enables it.""" + configured = os.environ.get("OPENCLAW_TEST_PACKAGE") + if configured is None: + pytest.skip("OPENCLAW_TEST_PACKAGE must name the installed native package") + package = Path(configured).resolve() + manifest = json.loads((package / "package.json").read_text()) + assert manifest["version"] in adapter.SUPPORTED_OPENCLAW_VERSIONS + return package + + +@pytest.mark.parametrize("skill", ["default", "alternate", None]) +def test_configured_skill_is_loaded_by_native_openclaw( + tmp_path: Path, + openclaw_package: Path, + default_skill: Path, + alternate_skill: Path, + skill: str | None, +): + """Check native selection, source path, and content hash for supplied skills.""" + sources = {"default": default_skill, "alternate": alternate_skill} + workspace = tmp_path / "workspace" + workspace.mkdir() + config = AgentConfig.from_mapping( + { + "models": {"default": {"provider": "test", "model": "unused"}}, + "skills": {"paths": [sources[skill]] if skill else []}, + } + ) + context = RuntimeContext.from_mapping( + { + "runtime_id": "skill-loading", + "invocation_id": "skill-loading", + "request_id": "skill-loading", + "environment": { + "environment_id": "skill-loading", + "provider": "local", + "control_location": "in_env_control", + "workspace": str(workspace), + "env": {}, + "ownership": "caller_owned", + }, + "artifacts": {}, + } + ) + native = adapter._openclaw_config( + config, context, base_dir=tmp_path, port=18789, token_env="UNUSED_TEST_TOKEN" + ) + config_path = tmp_path / "openclaw.json" + config_path.write_text(json.dumps(native)) + home = tmp_path / "home" + home.mkdir() + probe = Path(__file__).parents[1] / "_utils" / "openclaw_skills.mjs" + result = subprocess.run( + ["node", str(probe), str(openclaw_package), str(config_path)], + cwd=workspace, + env={ + "PATH": os.environ["PATH"], + "HOME": str(home), + "OPENCLAW_HOME": str(home), + "OPENCLAW_STATE_DIR": str(home / ".openclaw"), + }, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr + loaded = [entry for entry in json.loads(result.stdout) if entry["name"] in sources] + if skill is None: + assert loaded == [] + else: + source = (sources[skill] / "SKILL.md").resolve() + assert loaded == [ + { + "name": skill, + "source": "openclaw-extra", + "path": str(source), + "contentHash": hashlib.sha256(source.read_bytes()).hexdigest(), + } + ]