Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -192,8 +192,10 @@ jobs:

# τ²-bench requires Python 3.12+.
- name: Install τ²-bench environment dependencies
working-directory: envs/tau2_env
run: uv sync --python 3.12 --frozen --all-extras
working-directory: tests/tau2_runtime
run: |
uv lock --check
uv sync --python 3.12 --frozen --all-extras

# The tasks and databases live in τ²-bench's repository, at the commit pinned in
# envs/tau2_env/pyproject.toml.
Expand All @@ -206,7 +208,7 @@ jobs:

- name: Run τ²-bench environment and Claude Code recipe tests
run: |
uv run --python 3.12 --frozen --project envs/tau2_env --all-extras \
uv run --python 3.12 --frozen --project tests/tau2_runtime --all-extras \
pytest tests/envs/test_tau2_env.py \
tests/scripts/test_claude_code_harness_eval_example.py -v --tb=short
env:
Expand Down
27 changes: 27 additions & 0 deletions docs/source/guides/simulation-vs-production.md
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,33 @@ class MyEnv(MCPEnvironment):
return "Used by live MCP clients"
```

## FastMCP Runtime and Sessions

OpenEnv uses FastMCP 4 and creates its internal client with `mode="legacy"`.
This selects FastMCP's session-based handshake so `ctx.get_state()` and
`ctx.set_state()` persist across tool calls within a managed OpenEnv session.
HTTP sessions hold the MCP connection until the session closes; WebSocket
sessions hold it for the connection lifetime. A standalone tool call without
a persistent session does not provide this cross-call state guarantee.

FastMCP's Python tool descriptors use `input_schema`; MCP tool discovery
responses continue to use the wire field `inputSchema`.

### Harness sampling

FastMCP 4 removes `Context.sample()` even on legacy connections. A tool that
needs the harness client's model now returns an `InputRequiredResult` containing
a `CreateMessageRequest`, then reads the answer from `ctx.input_responses` on
the next invocation. The client's `sampling_handler` registration is unchanged.
See FastMCP's [guard pattern](https://gofastmcp.com/servers/elicitation#sampling-and-roots).

Sampling guards require a modern MCP connection (`Client(..., mode="auto")`).
The harness bridge mirrors the front connection's protocol onto its backend,
including when tools are renamed. Legacy harness connections still retain
per-session state; a modern guard is not supported by the legacy internal
client used for OpenEnv-managed tool calls. Tools that generate with their own
model credentials can instead call their model provider directly.

## Harness Environments: `WS /harness`

When the environment factory produces a [`HarnessEnvironment`](https://github.com/openenv/OpenEnv/blob/main/rfcs/005-agentic-harnesses.md) (an agent harness such as Claude Code running inside the environment), production mode also registers a `/harness` WebSocket, so clients talk to the harness itself:
Expand Down
3 changes: 1 addition & 2 deletions envs/tau2_env/server/tau2_environment.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,7 @@

import tau2.evaluator.evaluator_nl_assertions as nl_assertions
from fastmcp import FastMCP
from fastmcp.tools import Tool
from fastmcp.tools.tool import ToolResult
from fastmcp.tools import Tool, ToolResult
from openenv.core.env_server.mcp_environment import MCPEnvironment
from openenv.core.env_server.types import Action, Observation
from tau2.data_model.message import AssistantMessage
Expand Down
6 changes: 3 additions & 3 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ dependencies = [
# Core shared dependencies - minimal set required for all environments
# Heavy dependencies (torch, numpy, smolagents, etc.) should be in
# individual environment pyproject.toml files
"fastapi>=0.116.0",
"pydantic>=2.0.0",
"fastapi>=0.133.0",
"pydantic>=2.12.0",
"packaging>=24.2",
"typing_extensions>=4.0.0",
"uvicorn>=0.24.0",
Expand All @@ -30,7 +30,7 @@ dependencies = [
"tomli-w>=1.2.0",
"websockets>=15.0.1",
# MCP support
"fastmcp>=3.0.0,<4.0.0",
"fastmcp>=4.0.10,<5.0.0",
# Web UI dependencies
"gradio>=4.0.0",
"httpx>=0.28.1",
Expand Down
2 changes: 1 addition & 1 deletion src/openenv/core/env_server/http_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -1435,7 +1435,7 @@ async def call_mcp_style_step(action: Action) -> Observation:
return JsonRpcResponse.success(
result={
"tools": [
t.model_dump()
t.model_dump(by_alias=True)
if hasattr(t, "model_dump")
else dict(t)
for t in tools
Expand Down
14 changes: 8 additions & 6 deletions src/openenv/core/env_server/mcp_environment.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,17 +86,17 @@ def state(self):

def get_server_tools(mcp_server: Any) -> Dict[str, Any]:
"""
Get tools from a FastMCP server, compatible with both 2.x and 3.x.
Get tools from a FastMCP server.

Returns:
Dictionary mapping tool names to tool objects.
"""
# FastMCP 2.x: get_tools() returns dict {name: Tool}
# FastMCP server tools are exposed as a name-to-tool mapping.
if hasattr(mcp_server, "get_tools"):
result = run_async_safely(mcp_server.get_tools())
if isinstance(result, dict):
return result
# FastMCP 3.x: list_tools() returns list of Tool objects
# Some server adapters expose the tools as a list instead.
if hasattr(mcp_server, "list_tools"):
tools_list = run_async_safely(mcp_server.list_tools())
return {t.name: t for t in tools_list}
Expand All @@ -108,7 +108,7 @@ def _tool_from_server_tool(tool: Any) -> Tool:
return Tool(
name=tool.name,
description=tool.description or "",
input_schema=tool.inputSchema if hasattr(tool, "inputSchema") else {},
input_schema=tool.input_schema,
)


Expand Down Expand Up @@ -187,7 +187,9 @@ def __init__(self, mcp_server: Any, transform: Optional[Any] = None) -> None:
self._validate_tool_names(mcp_server)

self.mcp_server = mcp_server
self.mcp_client = Client(mcp_server)
# OpenEnv sessions keep MCP state across tool calls. FastMCP 4's default
# modern protocol is sessionless, so use its supported legacy handshake.
self.mcp_client = Client(mcp_server, mode="legacy")

# Track mode-specific tools: {tool_name: {mode: func}}
# mode can be "production", "simulation", or None (available in all modes)
Expand Down Expand Up @@ -254,7 +256,7 @@ def supports_code_mode(self) -> bool:

def _get_server_tools(self, mcp_server: Any) -> Dict[str, Any]:
"""
Get tools from a FastMCP server, compatible with both 2.x and 3.x.
Get tools from a FastMCP server.

Returns:
Dictionary mapping tool names to tool objects.
Expand Down
18 changes: 14 additions & 4 deletions src/openenv/core/harness/bridge.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@


async def _source_tools(mcp_server: Any) -> dict[str, Any]:
"""Return {name: FastMCP tool}, compatible with FastMCP 2.x and 3.x.
"""Return {name: FastMCP tool}.

Async twin of [`~openenv.core.env_server.mcp_environment.get_server_tools`],
which wraps the same calls in `run_async_safely`; awaiting directly avoids
Expand Down Expand Up @@ -52,9 +52,11 @@ async def build_bridge_server(mcp_server: Any, renames: dict[str, str]) -> Any:
return mcp_server

from fastmcp import Client
from fastmcp.server.dependencies import get_context
from fastmcp.server.providers.proxy import FastMCPProxy, StatefulProxyClient
from fastmcp.server.transforms import ToolTransform
from fastmcp.tools.tool_transform import ToolTransformConfig
from mcp_types.version import MODERN_PROTOCOL_VERSIONS

source_tools = await _source_tools(mcp_server)
missing = sorted(set(renames.values()) - set(source_tools))
Expand All @@ -71,12 +73,20 @@ async def lifespan(server):
async with Client(mcp_server):
yield {}

# Forward callbacks in the current request context. Each harness MCP
# connection gets its own source session, closed on disconnect.
# Keep one backend client per front connection, and mirror its protocol era.
# StatefulProxyClient defaults to legacy, which cannot carry sampling guards.
client = StatefulProxyClient(mcp_server)

def client_factory():
backend = client.new_stateful()
if not backend.is_connected():
version = get_context().request_context.protocol_version
backend.mode = version if version in MODERN_PROTOCOL_VERSIONS else "legacy"
return backend

view = FastMCPProxy(
name=f"{getattr(mcp_server, 'name', 'openenv')}-harness-view",
client_factory=client.new_stateful,
client_factory=client_factory,
lifespan=lifespan,
)
view.add_transform(
Expand Down
91 changes: 80 additions & 11 deletions tests/core/test_agentic_harness_bridge.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,14 @@
from fastmcp import Client, Context, FastMCP
from fastmcp.exceptions import ToolError
from fastmcp.server.middleware import Middleware
from mcp.types import CreateMessageResult, TextContent
from mcp.types import (
CreateMessageRequest,
CreateMessageRequestParams,
CreateMessageResult,
InputRequiredResult,
SamplingMessage,
TextContent,
)
from openenv.core.harness import (
AgenticHarnessAdapter,
HarnessConfig,
Expand Down Expand Up @@ -335,12 +342,12 @@ async def test_renamed_tool_is_listed_and_callable(self):
"env_read_file": "read_file",
}.items():
assert (
served[name].inputSchema
== source_tools[source_name].inputSchema
served[name].input_schema
== source_tools[source_name].input_schema
)
assert (
served[name].outputSchema
== source_tools[source_name].outputSchema
served[name].output_schema
== source_tools[source_name].output_schema
)

# The renamed tool must actually resolve, not just be listed.
Expand Down Expand Up @@ -423,18 +430,42 @@ def read_file(ctx: Context) -> int:
assert resources
assert all(not resource["open"] for resource in resources)

async def test_renamed_bridge_forwards_sampling_and_progress(self):
@pytest.mark.parametrize("rename", [False, True])
async def test_bridge_forwards_sampling_guard_and_progress(self, rename):
sampled = []
progress = []
rounds = []
mcp = FastMCP("domain")

@mcp.tool
async def read_file(path: str, ctx: Context) -> str:
response = await ctx.sample(path)
async def read_file(path: str, ctx: Context) -> str | InputRequiredResult:
# FastMCP 4 asks the caller via a returned guard, not ctx.sample().
# Each continuation must survive both the proxy and tool rename.
rounds.append((path, ctx.input_responses is not None))
if ctx.input_responses is None:
return InputRequiredResult(
input_requests={
"summary": CreateMessageRequest(
params=CreateMessageRequestParams(
messages=[
SamplingMessage(
role="user",
content=TextContent(type="text", text=path),
)
],
max_tokens=80,
)
)
},
request_state=path,
)
assert ctx.request_state == path
response = ctx.input_responses["summary"]
await ctx.report_progress(1, 1, path)
return response.text
return response.content.text

async def sample(messages, params, context):
assert params.max_tokens == 80
text = messages[0].content.text
sampled.append(text)
return CreateMessageResult(
Expand All @@ -447,20 +478,58 @@ async def record_progress(completed, total, message):
progress.append((completed, total, message))

adapter = RecordingAdapter()
if not rename:
adapter.BUILTIN_TOOL_NAMES = frozenset()
env = HarnessEnvironment(adapter=adapter, mcp=mcp)
try:
await env.reset_async()
async with Client(adapter.bridge_url, sampling_handler=sample) as client:
async with Client(
adapter.bridge_url, mode="auto", sampling_handler=sample
) as client:
for path in ("first", "second"):
result = await client.call_tool(
"env_read_file",
"env_read_file" if rename else "read_file",
{"path": path},
progress_handler=record_progress,
timeout=5.0,
)
assert result.content[0].text == f"sampled {path}"
assert sampled == ["first", "second"]
assert progress == [(1, 1, "first"), (1, 1, "second")]
assert rounds == [
("first", False),
("first", True),
("second", False),
("second", True),
]
finally:
env.close()

@pytest.mark.parametrize("rename", [False, True])
async def test_bridge_preserves_legacy_session_state(self, rename):
mcp = FastMCP("domain")

@mcp.tool
async def read_file(ctx: Context) -> int:
count = await ctx.get_state("calls") or 0
await ctx.set_state("calls", count + 1)
return count + 1

adapter = RecordingAdapter()
if not rename:
adapter.BUILTIN_TOOL_NAMES = frozenset()
tool_name = "env_read_file" if rename else "read_file"
env = HarnessEnvironment(adapter=adapter, mcp=mcp)
try:
await env.reset_async()
async with Client(adapter.bridge_url, mode="legacy") as first:
assert (await first.call_tool(tool_name, {})).data == 1
async with Client(adapter.bridge_url, mode="legacy") as second:
assert (await second.call_tool(tool_name, {})).data == 1
assert (await first.call_tool(tool_name, {})).data == 2
assert (await second.call_tool(tool_name, {})).data == 2
async with Client(adapter.bridge_url, mode="legacy") as replacement:
assert (await replacement.call_tool(tool_name, {})).data == 1
finally:
env.close()

Expand Down
3 changes: 3 additions & 0 deletions tests/core/test_mcp/test_mcp_integration.py
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,9 @@ def test_fastmcp_in_mcp_environment_list_tools(self, minimal_mcp_env):
tool_names = [t.name for t in obs.tools]
assert "add" in tool_names
assert "greet" in tool_names
add_tool = next(tool for tool in obs.tools if tool.name == "add")
assert set(add_tool.input_schema["properties"]) == {"a", "b"}
assert set(add_tool.input_schema["required"]) == {"a", "b"}

def test_fastmcp_in_mcp_environment_call_add(self, minimal_mcp_env):
"""Test MCPEnvironment can call an 'add' tool from FastMCP server."""
Expand Down
7 changes: 2 additions & 5 deletions tests/core/test_production_mode_mcp.py
Original file line number Diff line number Diff line change
Expand Up @@ -184,11 +184,8 @@ def test_production_mode_mcp_tools_list_via_websocket(self, production_mcp_app):
# Verify tool has required fields
get_info_tool = next(t for t in tools if t["name"] == "get_info")
assert "description" in get_info_tool, "Tool should have description"
# Note: FastMCP may use different field names (inputSchema vs input_schema)
# Just verify the tool has some schema-related field
assert len(get_info_tool) > 2, (
"Tool should have name, description, and other metadata"
)
assert "inputSchema" in get_info_tool
assert "input_schema" not in get_info_tool

def test_production_mode_tools_list_without_reset(self, production_mcp_app):
"""
Expand Down
Loading
Loading