diff --git a/openhands-tools/openhands/tools/terminal/impl.py b/openhands-tools/openhands/tools/terminal/impl.py index b35ebeb3204..ef3ac289005 100644 --- a/openhands-tools/openhands/tools/terminal/impl.py +++ b/openhands-tools/openhands/tools/terminal/impl.py @@ -52,7 +52,9 @@ _TMUX_RECOVERABLE_ERROR_MARKERS = ( "no server running", + "server exited unexpectedly", "can't find session", + "can't find pane", "could not find window_id", "could not find pane_id", ) diff --git a/openhands-tools/openhands/tools/terminal/terminal/tmux_pane_pool.py b/openhands-tools/openhands/tools/terminal/terminal/tmux_pane_pool.py index a2202a46b1b..bb6262c5fa7 100644 --- a/openhands-tools/openhands/tools/terminal/terminal/tmux_pane_pool.py +++ b/openhands-tools/openhands/tools/terminal/terminal/tmux_pane_pool.py @@ -216,7 +216,9 @@ def _create_pane(self) -> PooledTmuxTerminal: self._initial_window.kill() self._initial_window = None - logger.debug(f"Created pooled pane #{len(self._all_panes)}: {active_pane}") + logger.debug( + "Created pooled pane #%d: %s", len(self._all_panes), active_pane.pane_id + ) return terminal def checkout( @@ -252,17 +254,17 @@ def checkout( with self._lock: if prefer is not None and prefer in self._available: self._available.remove(prefer) - logger.debug(f"Checked out preferred pane: {prefer.pane}") + logger.debug("Checked out preferred pane: %s", prefer.pane.pane_id) return prefer if self._available: terminal = self._available.popleft() - logger.debug(f"Checked out existing pane: {terminal.pane}") + logger.debug("Checked out existing pane: %s", terminal.pane.pane_id) return terminal # Create a new pane (still under max_panes thanks to semaphore) terminal = self._create_pane() self._all_panes.append(terminal) - logger.debug(f"Checked out new pane: {terminal.pane}") + logger.debug("Checked out new pane: %s", terminal.pane.pane_id) return terminal except Exception: self._semaphore.release() @@ -278,7 +280,7 @@ def checkin(self, terminal: PooledTmuxTerminal) -> None: self._available.append(terminal) self._semaphore.release() - logger.debug(f"Checked in pane: {terminal.pane}") + logger.debug("Checked in pane: %s", terminal.pane.pane_id) def replace(self, old_terminal: PooledTmuxTerminal) -> PooledTmuxTerminal: """Replace a checked-out pane with a fresh one. diff --git a/openhands-tools/openhands/tools/terminal/terminal/tmux_terminal.py b/openhands-tools/openhands/tools/terminal/terminal/tmux_terminal.py index bd52cd3870c..1a223eca592 100644 --- a/openhands-tools/openhands/tools/terminal/terminal/tmux_terminal.py +++ b/openhands-tools/openhands/tools/terminal/terminal/tmux_terminal.py @@ -12,6 +12,7 @@ import libtmux from filelock import FileLock +from libtmux.exc import LibTmuxException from openhands.sdk.logger import get_logger from openhands.sdk.utils.redact import redact_api_key_literals @@ -391,15 +392,39 @@ def send_keys(self, text: str, enter: bool = True) -> None: # 3) Plain text — use literal=True so tmux doesn't split on # whitespace or interpret special tokens. - self.pane.send_keys(text, enter=False, literal=True) if enter and not text.endswith("\n"): - self.pane.send_keys("Enter", enter=False) + self._send_then_enter("-l", "--", text) + else: + self.pane.send_keys(text, enter=False, literal=True) + + def _send_then_enter(self, *send_keys_args: str) -> None: + """Send keys and then Enter in one tmux process. + + tmux would read a last argument ending in ``;`` as a command separator, + so such text takes two processes, as ``pane.send_keys`` does. + """ + assert isinstance(self.pane, libtmux.Pane) + if send_keys_args[-1].endswith(";"): + self.pane.cmd("send-keys", *send_keys_args) + self.pane.enter() + return + self.pane.cmd( + "send-keys", + *send_keys_args, + ";", + "send-keys", + "-t", + str(self.pane.pane_id), + "Enter", + ) @staticmethod def _capture_pane(pane: libtmux.Pane) -> str: - return "\n".join( - line.rstrip() for line in pane.cmd("capture-pane", "-J", "-pS", "-").stdout - ) + result = pane.cmd("capture-pane", "-J", "-pS", "-") + if result.stderr: + # E.g. the shell exited and took the pane (or the server) with it. + raise LibTmuxException(*result.stderr) + return "\n".join(line.rstrip() for line in result.stdout) def read_screen(self) -> str: """Read the current tmux pane content. @@ -425,7 +450,7 @@ def clear_screen(self) -> None: if not self._initialized or not isinstance(self.pane, libtmux.Pane): raise RuntimeError("Tmux terminal is not initialized") - self.pane.send_keys("clear", enter=True) + self._send_then_enter("clear") time.sleep(0.1) self.pane.cmd("clear-history") diff --git a/tests/tools/terminal/test_pool_integration.py b/tests/tools/terminal/test_pool_integration.py index ed85414c5c2..0f8e7654410 100644 --- a/tests/tools/terminal/test_pool_integration.py +++ b/tests/tools/terminal/test_pool_integration.py @@ -144,6 +144,22 @@ def test_shell_exit_returns_actionable_error_and_rebuilds_pool(self, pool_execut assert after.exit_code == 0 assert "after_rebuild" in after.text + def test_shell_exit_beside_a_busy_pane_rebuilds_pool(self, pool_executor): + # The busy pane keeps the tmux server up, so the exited pane is gone + # while the server still answers. + busy = threading.Thread( + target=lambda: pool_executor(TerminalAction(command="sleep 3", timeout=10)) + ) + busy.start() + time.sleep(0.5) + obs = pool_executor(TerminalAction(command="exit 7", timeout=5.0)) + busy.join() + + assert obs.is_error + assert "rebuilt the terminal pool" in obs.text + after = pool_executor(TerminalAction(command="echo after_rebuild", timeout=5.0)) + assert "after_rebuild" in after.text + def test_reset_after_shell_exit_uses_rebuilt_pool(self, pool_executor): obs = pool_executor(TerminalAction(command="exit 0", timeout=1.0)) assert obs.is_error diff --git a/tests/tools/terminal/test_tmux_command_batching.py b/tests/tools/terminal/test_tmux_command_batching.py index 4377d3c9371..c9873698a81 100644 --- a/tests/tools/terminal/test_tmux_command_batching.py +++ b/tests/tools/terminal/test_tmux_command_batching.py @@ -7,6 +7,8 @@ import libtmux.common import pytest +from openhands.tools.terminal.definition import TerminalAction +from openhands.tools.terminal.impl import TerminalExecutor from openhands.tools.terminal.terminal.tmux_pane_pool import TmuxPanePool from openhands.tools.terminal.terminal.tmux_terminal import run_tmux_commands @@ -109,3 +111,42 @@ def test_batches_are_sized_in_utf8_bytes(): server.kill() assert {name: environment[name] for name in values} == values + + +@pytest.fixture +def executor(tmp_path): + executor = TerminalExecutor(working_dir=str(tmp_path)) + yield executor + executor.close() + + +def test_command_runs_in_few_tmux_processes(executor, monkeypatch): + executor(TerminalAction(command="echo warm")) + commands: list[str] = [] + real_init = libtmux.common.tmux_cmd.__init__ + + def recording_init(self, *args, **kwargs): + commands.append(next(a for a in args if not str(a).startswith("-"))) + real_init(self, *args, **kwargs) + + monkeypatch.setattr(libtmux.common.tmux_cmd, "__init__", recording_init) + observation = executor(TerminalAction(command="echo counted")) + + assert "counted" in observation.text + # Each tmux process stalls the server's other threads while it is spawned; + # a command used to take 11. How often the screen is polled depends on + # timing, so only the rest is counted. + assert len([c for c in commands if c != "capture-pane"]) <= 3 + + +def test_command_starting_with_a_dash_runs(executor): + observation = executor(TerminalAction(command="-not-a-command", timeout=10)) + + assert observation.exit_code == 127 + + +def test_command_ending_in_semicolon_runs(executor): + observation = executor(TerminalAction(command="echo semicolon_end;")) + + assert observation.exit_code == 0 + assert "semicolon_end" in observation.text diff --git a/tests/tools/terminal/test_tmux_pane_pool.py b/tests/tools/terminal/test_tmux_pane_pool.py index dd16e0d306c..ffd15f338b3 100644 --- a/tests/tools/terminal/test_tmux_pane_pool.py +++ b/tests/tools/terminal/test_tmux_pane_pool.py @@ -7,6 +7,7 @@ from pathlib import Path import pytest +from libtmux.exc import LibTmuxException from openhands.tools.terminal.constants import ( TMUX_SESSION_HEIGHT, @@ -307,7 +308,8 @@ def test_stale_terminal_cannot_access_restarted_server( time.sleep(0.05) if operation == "read": - assert "SECOND_CONVERSATION_MARKER" not in stale.read_screen() + with pytest.raises(LibTmuxException): + stale.read_screen() elif operation == "write": marker = tmp_path / "wrong-conversation" stale.send_keys(f"touch {marker}")