diff --git a/src/openenv/core/containers/runtime/hf_sandbox_provider.py b/src/openenv/core/containers/runtime/hf_sandbox_provider.py index 05d6ce250..112e0297c 100644 --- a/src/openenv/core/containers/runtime/hf_sandbox_provider.py +++ b/src/openenv/core/containers/runtime/hf_sandbox_provider.py @@ -10,6 +10,7 @@ import asyncio import hashlib +import logging import socket import threading import time @@ -24,6 +25,8 @@ from .providers import ContainerProvider +logger = logging.getLogger(__name__) + _DEFAULT_PORT = 8000 _SERVER_COMMAND = ( 'export SBX_PROXY_DIR="${SBX_PROXY_DIR:-$HOME/.sbx/proxy}"; ' @@ -95,6 +98,17 @@ def _get_pool(image: str, flavor: str) -> Any: class _LocalAuthProxy: + """Loopback proxy that adds the HF auth headers to every request. + + Copied into agent_sandbox_provider._LocalAuthProxy; keep the copies + aligned. + + Limitations: bodies are buffered (a streamed upstream response such as + SSE is withheld until it ends), only JSON text frames are relayed + upstream (binary client frames and close codes are not forwarded), and + upstream redirects are rejected. EnvClient uses none of these. + """ + def __init__(self, *, target_url: str, headers: dict[str, str]): self.target_url = target_url.rstrip("/") self.headers = headers @@ -131,13 +145,20 @@ async def proxy_http(path: str, request: Request) -> Response: data=body, headers=headers, timeout=60.0, - allow_redirects=True, + allow_redirects=False, ) except requests.RequestException: return Response( content=b"upstream HF job unreachable", status_code=502, ) + if 300 <= upstream.status_code < 400: + # Following a redirect would replay the body and the injected + # headers to another origin and could downgrade https to http. + return Response( + content=b"upstream HF job redirected; rejected", + status_code=502, + ) response_headers = { key: value for key, value in upstream.headers.items() @@ -156,7 +177,16 @@ async def proxy_websocket(path: str, websocket: WebSocket) -> None: if query: target = f"{target}?{query}" headers = {**self.headers, "accept": "*/*"} - upstream = await self._connect_upstream_websocket(target, headers) + try: + upstream = await self._connect_upstream_websocket(target, headers) + except Exception: + # Never propagate the upstream exception: its message can embed + # the target URL, which may carry credentials. Deny generically + # instead. + logger.warning("HF sandbox WebSocket upstream handshake failed") + with suppress(Exception): + await websocket.close(code=1011) + return await websocket.accept() try: to_upstream = asyncio.create_task( @@ -201,13 +231,18 @@ async def _connect_upstream_websocket( ) -> Any: last_error: Exception | None = None for _ in range(5): + connector = ws_connect( + target, + additional_headers=headers, + max_size=_MAX_WS_MESSAGE_SIZE, + compression=None, + ) + # Never follow handshake redirects: websockets replays + # additional_headers to the redirect target, which would leak the + # auth headers to another origin. + connector.process_redirect = lambda exc: exc try: - return await ws_connect( - target, - additional_headers=headers, - max_size=_MAX_WS_MESSAGE_SIZE, - compression=None, - ) + return await connector except Exception as exc: last_error = exc await asyncio.sleep(0.5) @@ -227,10 +262,18 @@ async def _upstream_to_client(self, websocket: WebSocket, upstream: Any) -> None await websocket.send_text(message) def stop(self) -> None: + """Shut the proxy down. Raises `RuntimeError` if the thread will not + exit; the handles are then retained so a later call can retry, instead + of silently leaking a thread that keeps the loopback port bound.""" if self._server is None or self._thread is None: return self._server.should_exit = True + # Bound the graceful drain: do not wait out long-lived upstream + # responses that would outlive the 5s join below. + self._server.force_exit = True self._thread.join(timeout=5.0) + if self._thread.is_alive(): + raise RuntimeError("HF sandbox auth proxy did not stop") self._server = None self._thread = None @@ -272,8 +315,11 @@ def start_container( env_vars: dict[str, str] | None = None, **kwargs: Any, ) -> str: - if self._sandbox is not None: - raise RuntimeError("HFSandboxProvider already has an active service") + if self._sandbox is not None or self._proxy is not None: + raise RuntimeError( + "HFSandboxProvider already has an active service, or a proxy " + "still awaits cleanup. Call stop_container() first." + ) if kwargs: unsupported = ", ".join(sorted(kwargs)) raise ValueError( @@ -311,14 +357,20 @@ def start_container( raise def stop_container(self) -> None: + proxy_error: RuntimeError | None = None if self._proxy is not None: - self._proxy.stop() - self._proxy = None + try: + self._proxy.stop() + self._proxy = None + except RuntimeError as exc: + proxy_error = exc if self._sandbox is not None: sandbox = self._sandbox self._sandbox = None with suppress(Exception): sandbox.kill() + if proxy_error is not None: + raise proxy_error def wait_for_ready(self, base_url: str, timeout_s: float = 120.0) -> None: deadline = time.time() + timeout_s diff --git a/tests/test_core/test_hf_sandbox_provider.py b/tests/test_core/test_hf_sandbox_provider.py new file mode 100644 index 000000000..374128dbb --- /dev/null +++ b/tests/test_core/test_hf_sandbox_provider.py @@ -0,0 +1,321 @@ +# SPDX-License-Identifier: BSD-3-Clause + +"""Unit tests for HFSandboxProvider and its local auth proxy. No Hugging Face +account or network needed: the sandbox pool is faked, and loopback FastAPI +apps stand in for the HF sandbox proxy endpoint.""" + +from __future__ import annotations + +import asyncio +import logging +import threading +import time +from unittest.mock import MagicMock, patch + +import pytest +import requests +import uvicorn +from fastapi import FastAPI, Request, WebSocket +from fastapi.responses import RedirectResponse +from openenv.core.containers.runtime import hf_sandbox_provider as hf +from openenv.core.containers.runtime.hf_sandbox_provider import ( + _find_available_port, + _LocalAuthProxy, + HFSandboxProvider, +) +from websockets.asyncio.client import connect as ws_connect + + +class _FakeUpstream: + """Loopback stand-in for the HF sandbox proxy endpoint: records headers, + answers every path, echoes one WebSocket text frame, and can emit + HTTP/WS redirects to test that the proxy never follows them.""" + + def __init__(self): + self.headers: dict[str, str] = {} + self.ws_headers: dict[str, str] = {} + self.redirect_target = "http://127.0.0.1:1/" + self.ws_redirect_target = "ws://127.0.0.1:1/" + self.port = _find_available_port() + app = FastAPI() + + @app.websocket("/ws-redirect") + async def ws_redirect(websocket: WebSocket): + # Answer the handshake with a plain HTTP 302 (Starlette's message + # types for HTTP responses inside a websocket scope). + await websocket.send( + { + "type": "websocket.http.response.start", + "status": 302, + "headers": [ + (b"location", self.ws_redirect_target.encode()), + ], + } + ) + await websocket.send({"type": "websocket.http.response.body", "body": b""}) + + @app.api_route("/{path:path}", methods=["GET", "POST"]) + async def handle(path: str, request: Request): + self.headers = dict(request.headers) + if path == "health": + return {"status": "ok"} + if path == "redirect": + return RedirectResponse(url=self.redirect_target, status_code=307) + body = await request.body() + return {"path": path, "body": body.decode()} + + @app.websocket("/{path:path}") + async def ws(websocket: WebSocket): + self.ws_headers = dict(websocket.headers) + await websocket.accept() + message = await websocket.receive_text() + await websocket.send_text(f"echo:{message}") + await websocket.close() + + config = uvicorn.Config( + app, + host="127.0.0.1", + port=self.port, + log_level="warning", + access_log=False, + ) + self._server = uvicorn.Server(config) + self._thread = threading.Thread(target=self._server.run, daemon=True) + + @property + def url(self) -> str: + return f"http://127.0.0.1:{self.port}" + + def start(self): + self._thread.start() + while not self._server.started: + time.sleep(0.05) + + def stop(self): + self._server.should_exit = True + self._thread.join(timeout=5.0) + + +@pytest.fixture +def fake_upstream(): + upstream = _FakeUpstream() + upstream.start() + yield upstream + upstream.stop() + + +def _ws_roundtrip(base: str, path: str = "/ws"): + async def attempt(): + async with ws_connect(f"{base.replace('http://', 'ws://')}{path}") as ws: + await ws.send("ping") + return await ws.recv() + + return asyncio.run(attempt()) + + +# --- proxy behavior --- + + +def test_proxy_injects_headers_on_http(fake_upstream): + proxy = _LocalAuthProxy( + target_url=fake_upstream.url, + headers={"Authorization": "Bearer hf-secret"}, + ) + base = proxy.start() + try: + response = requests.get(f"{base}/health", timeout=10) + assert response.status_code == 200 + assert fake_upstream.headers["authorization"] == "Bearer hf-secret" + finally: + proxy.stop() + + +def test_proxy_bridges_websocket_with_headers(fake_upstream): + proxy = _LocalAuthProxy( + target_url=fake_upstream.url, + headers={"Authorization": "Bearer hf-secret"}, + ) + base = proxy.start() + try: + assert _ws_roundtrip(base) == "echo:ping" + assert fake_upstream.ws_headers["authorization"] == "Bearer hf-secret" + finally: + proxy.stop() + + +def test_http_redirect_is_rejected_not_followed(fake_upstream): + attacker = _FakeUpstream() + attacker.start() + try: + fake_upstream.redirect_target = f"{attacker.url}/steal?token=s3cret" + proxy = _LocalAuthProxy( + target_url=fake_upstream.url, + headers={"Authorization": "Bearer hf-secret"}, + ) + base = proxy.start() + try: + response = requests.get(f"{base}/redirect", timeout=10) + assert response.status_code == 502 + # The redirect target must never receive the replayed request. + assert attacker.headers == {} + finally: + proxy.stop() + finally: + attacker.stop() + + +def test_ws_redirect_is_not_followed(fake_upstream): + attacker = _FakeUpstream() + attacker.start() + try: + fake_upstream.ws_redirect_target = ( + attacker.url.replace("http://", "ws://") + "/steal?token=s3cret" + ) + proxy = _LocalAuthProxy( + target_url=fake_upstream.url, + headers={"Authorization": "Bearer hf-secret"}, + ) + base = proxy.start() + try: + with pytest.raises(Exception): + _ws_roundtrip(base, "/ws-redirect") + # websockets would replay additional_headers on a cross-origin + # redirect; the proxy must refuse to follow it at all. + assert attacker.ws_headers == {} + assert attacker.headers == {} + finally: + proxy.stop() + finally: + attacker.stop() + + +def test_ws_handshake_failure_is_sanitized(fake_upstream, caplog): + from websockets.exceptions import SecurityError + + proxy = _LocalAuthProxy( + target_url=fake_upstream.url, + headers={"Authorization": "Bearer hf-secret"}, + ) + base = proxy.start() + secret = "ws://127.0.0.1:9/session?token=review-url-secret" + try: + with ( + patch( + "openenv.core.containers.runtime.hf_sandbox_provider.ws_connect", + side_effect=SecurityError( + f"cannot follow redirect to non-secure URI {secret}" + ), + ), + caplog.at_level(logging.WARNING, logger=hf.__name__), + ): + with pytest.raises(Exception): + _ws_roundtrip(base) + # The route must handle the failure itself with a generic warning: + # the upstream exception chain (which embeds the URL) must not be + # logged or propagated. + assert "handshake failed" in caplog.text + assert "review-url-secret" not in caplog.text + finally: + proxy.stop() + + +def test_stop_keeps_handles_when_thread_does_not_exit(fake_upstream): + proxy = _LocalAuthProxy(target_url=fake_upstream.url, headers={}) + proxy.start() + real_thread = proxy._thread + + class _StuckThread: + def is_alive(self): + return True + + def join(self, timeout=None): + pass + + proxy._thread = _StuckThread() + try: + with pytest.raises(RuntimeError, match="did not stop"): + proxy.stop() + # Handles are retained so a later stop() can retry. + assert proxy._thread is not None + assert proxy._server is not None + finally: + proxy._thread = real_thread + proxy.stop() + assert proxy._thread is None + + +# --- provider lifecycle --- + + +@pytest.fixture +def fake_pool(fake_upstream): + """Fake the huggingface_hub sandbox pool; the sandbox proxy points at the + loopback fake upstream.""" + sandbox = MagicMock() + sandbox.proxy_url_for.return_value = fake_upstream.url + sandbox.proxy_headers = {"Authorization": "Bearer hf-secret"} + pool = MagicMock() + pool.create.return_value = sandbox + with patch.object(hf, "_get_pool", return_value=pool): + yield sandbox + + +def test_start_container_runs_server_and_proxies(fake_pool, fake_upstream): + provider = HFSandboxProvider(image="hf.co/spaces/openenv/echo_env") + base = provider.start_container(env_vars={"A": "1"}) + try: + assert base.startswith("http://127.0.0.1:") + fake_pool.run.assert_called_once() + response = requests.get(f"{base}/health", timeout=10) + assert response.status_code == 200 + assert fake_upstream.headers["authorization"] == "Bearer hf-secret" + finally: + provider.stop_container() + fake_pool.kill.assert_called_once() + + +def test_stop_container_kills_sandbox_even_if_proxy_stop_fails(fake_pool): + provider = HFSandboxProvider(image="hf.co/spaces/openenv/echo_env") + provider.start_container() + + with patch.object( + _LocalAuthProxy, "stop", side_effect=[RuntimeError("proxy did not stop"), None] + ): + with pytest.raises(RuntimeError, match="did not stop"): + provider.stop_container() + # The sandbox was killed even though the proxy shutdown failed. + fake_pool.kill.assert_called_once() + provider.stop_container() + + assert provider._proxy is None + + +def test_start_refused_while_proxy_cleanup_pending(fake_pool): + provider = HFSandboxProvider(image="hf.co/spaces/openenv/echo_env") + provider.start_container() + + with patch.object( + _LocalAuthProxy, "stop", side_effect=[RuntimeError("proxy did not stop"), None] + ): + with pytest.raises(RuntimeError, match="did not stop"): + provider.stop_container() + # The old proxy thread may still hold the loopback port: a restart + # must not overwrite the retained handle. + with pytest.raises(RuntimeError, match="stop_container"): + provider.start_container() + provider.stop_container() + + assert provider._proxy is None + base = provider.start_container() + assert base.startswith("http://127.0.0.1:") + provider.stop_container() + + +def test_second_start_is_rejected(fake_pool): + provider = HFSandboxProvider(image="hf.co/spaces/openenv/echo_env") + provider.start_container() + try: + with pytest.raises(RuntimeError, match="already has an active service"): + provider.start_container() + finally: + provider.stop_container()