From 1d7a15e6a4baecd7131de12a4d663368e569d4bc Mon Sep 17 00:00:00 2001 From: VascoSch92 Date: Mon, 16 Nov 2026 09:19:57 +0100 Subject: [PATCH 1/3] perf: run a terminal command in 5 tmux processes instead of 11 Every tmux process is spawned from the agent-server, and on macOS a spawn stalls the server's other threads for about as long as it takes (10-28 ms for a 300 MB-1 GB process). A terminal command took 11 of them: - 4 came from debug log lines whose f-strings formatted a libtmux Pane, whose repr queries tmux, even with debug logging off. Log the pane id. - Sending text and then Enter took two processes, also inside clear_screen. Send both in one tmux invocation, unless the text ends in ";", which tmux would read as a separator. A dead pane used to be noticed only because one of those log lines raised. Capturing the pane now raises when tmux reports an error, so the pool recovery still runs, without waiting for the command timeout. --- .../tools/terminal/terminal/tmux_pane_pool.py | 12 +++--- .../tools/terminal/terminal/tmux_terminal.py | 37 ++++++++++++++++--- .../terminal/test_tmux_command_batching.py | 35 ++++++++++++++++++ tests/tools/terminal/test_tmux_pane_pool.py | 4 +- 4 files changed, 76 insertions(+), 12 deletions(-) diff --git a/openhands-tools/openhands/tools/terminal/terminal/tmux_pane_pool.py b/openhands-tools/openhands/tools/terminal/terminal/tmux_pane_pool.py index a2202a46b1b..bb6262c5fa7 100644 --- a/openhands-tools/openhands/tools/terminal/terminal/tmux_pane_pool.py +++ b/openhands-tools/openhands/tools/terminal/terminal/tmux_pane_pool.py @@ -216,7 +216,9 @@ def _create_pane(self) -> PooledTmuxTerminal: self._initial_window.kill() self._initial_window = None - logger.debug(f"Created pooled pane #{len(self._all_panes)}: {active_pane}") + logger.debug( + "Created pooled pane #%d: %s", len(self._all_panes), active_pane.pane_id + ) return terminal def checkout( @@ -252,17 +254,17 @@ def checkout( with self._lock: if prefer is not None and prefer in self._available: self._available.remove(prefer) - logger.debug(f"Checked out preferred pane: {prefer.pane}") + logger.debug("Checked out preferred pane: %s", prefer.pane.pane_id) return prefer if self._available: terminal = self._available.popleft() - logger.debug(f"Checked out existing pane: {terminal.pane}") + logger.debug("Checked out existing pane: %s", terminal.pane.pane_id) return terminal # Create a new pane (still under max_panes thanks to semaphore) terminal = self._create_pane() self._all_panes.append(terminal) - logger.debug(f"Checked out new pane: {terminal.pane}") + logger.debug("Checked out new pane: %s", terminal.pane.pane_id) return terminal except Exception: self._semaphore.release() @@ -278,7 +280,7 @@ def checkin(self, terminal: PooledTmuxTerminal) -> None: self._available.append(terminal) self._semaphore.release() - logger.debug(f"Checked in pane: {terminal.pane}") + logger.debug("Checked in pane: %s", terminal.pane.pane_id) def replace(self, old_terminal: PooledTmuxTerminal) -> PooledTmuxTerminal: """Replace a checked-out pane with a fresh one. diff --git a/openhands-tools/openhands/tools/terminal/terminal/tmux_terminal.py b/openhands-tools/openhands/tools/terminal/terminal/tmux_terminal.py index bd52cd3870c..d9a86345dc2 100644 --- a/openhands-tools/openhands/tools/terminal/terminal/tmux_terminal.py +++ b/openhands-tools/openhands/tools/terminal/terminal/tmux_terminal.py @@ -12,6 +12,7 @@ import libtmux from filelock import FileLock +from libtmux.exc import LibTmuxException from openhands.sdk.logger import get_logger from openhands.sdk.utils.redact import redact_api_key_literals @@ -391,15 +392,39 @@ def send_keys(self, text: str, enter: bool = True) -> None: # 3) Plain text — use literal=True so tmux doesn't split on # whitespace or interpret special tokens. - self.pane.send_keys(text, enter=False, literal=True) if enter and not text.endswith("\n"): - self.pane.send_keys("Enter", enter=False) + self._send_then_enter("-l", text) + else: + self.pane.send_keys(text, enter=False, literal=True) + + def _send_then_enter(self, *send_keys_args: str) -> None: + """Send keys and then Enter in one tmux process. + + tmux would read a last argument ending in ``;`` as a command separator, + so such text takes two processes, as ``pane.send_keys`` does. + """ + assert isinstance(self.pane, libtmux.Pane) + if send_keys_args[-1].endswith(";"): + self.pane.cmd("send-keys", *send_keys_args) + self.pane.enter() + return + self.pane.cmd( + "send-keys", + *send_keys_args, + ";", + "send-keys", + "-t", + str(self.pane.pane_id), + "Enter", + ) @staticmethod def _capture_pane(pane: libtmux.Pane) -> str: - return "\n".join( - line.rstrip() for line in pane.cmd("capture-pane", "-J", "-pS", "-").stdout - ) + result = pane.cmd("capture-pane", "-J", "-pS", "-") + if result.stderr: + # E.g. the shell exited and took the pane (or the server) with it. + raise LibTmuxException(*result.stderr) + return "\n".join(line.rstrip() for line in result.stdout) def read_screen(self) -> str: """Read the current tmux pane content. @@ -425,7 +450,7 @@ def clear_screen(self) -> None: if not self._initialized or not isinstance(self.pane, libtmux.Pane): raise RuntimeError("Tmux terminal is not initialized") - self.pane.send_keys("clear", enter=True) + self._send_then_enter("clear") time.sleep(0.1) self.pane.cmd("clear-history") diff --git a/tests/tools/terminal/test_tmux_command_batching.py b/tests/tools/terminal/test_tmux_command_batching.py index 4377d3c9371..f78522b5be4 100644 --- a/tests/tools/terminal/test_tmux_command_batching.py +++ b/tests/tools/terminal/test_tmux_command_batching.py @@ -7,6 +7,8 @@ import libtmux.common import pytest +from openhands.tools.terminal.definition import TerminalAction +from openhands.tools.terminal.impl import TerminalExecutor from openhands.tools.terminal.terminal.tmux_pane_pool import TmuxPanePool from openhands.tools.terminal.terminal.tmux_terminal import run_tmux_commands @@ -109,3 +111,36 @@ def test_batches_are_sized_in_utf8_bytes(): server.kill() assert {name: environment[name] for name in values} == values + + +@pytest.fixture +def executor(tmp_path): + executor = TerminalExecutor(working_dir=str(tmp_path)) + yield executor + executor.close() + + +def test_command_runs_in_few_tmux_processes(executor, monkeypatch): + executor(TerminalAction(command="echo warm")) + calls = 0 + real_init = libtmux.common.tmux_cmd.__init__ + + def counting_init(self, *args, **kwargs): + nonlocal calls + calls += 1 + real_init(self, *args, **kwargs) + + monkeypatch.setattr(libtmux.common.tmux_cmd, "__init__", counting_init) + observation = executor(TerminalAction(command="echo counted")) + + assert "counted" in observation.text + # Each tmux process stalls the server's other threads while it is spawned; + # a command used to take 11. + assert calls <= 6 + + +def test_command_ending_in_semicolon_runs(executor): + observation = executor(TerminalAction(command="echo semicolon_end;")) + + assert observation.exit_code == 0 + assert "semicolon_end" in observation.text diff --git a/tests/tools/terminal/test_tmux_pane_pool.py b/tests/tools/terminal/test_tmux_pane_pool.py index dd16e0d306c..ffd15f338b3 100644 --- a/tests/tools/terminal/test_tmux_pane_pool.py +++ b/tests/tools/terminal/test_tmux_pane_pool.py @@ -7,6 +7,7 @@ from pathlib import Path import pytest +from libtmux.exc import LibTmuxException from openhands.tools.terminal.constants import ( TMUX_SESSION_HEIGHT, @@ -307,7 +308,8 @@ def test_stale_terminal_cannot_access_restarted_server( time.sleep(0.05) if operation == "read": - assert "SECOND_CONVERSATION_MARKER" not in stale.read_screen() + with pytest.raises(LibTmuxException): + stale.read_screen() elif operation == "write": marker = tmp_path / "wrong-conversation" stale.send_keys(f"touch {marker}") From 3def75fea02b988a8bbedb114f243ee64f1b2b52 Mon Sep 17 00:00:00 2001 From: VascoSch92 Date: Mon, 16 Nov 2026 10:18:05 +0100 Subject: [PATCH 2/3] fix: recover the tmux pool when tmux reports the server exited On Linux a capture against a pane whose shell exited fails with "server exited unexpectedly" rather than "no server running". --- openhands-tools/openhands/tools/terminal/impl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openhands-tools/openhands/tools/terminal/impl.py b/openhands-tools/openhands/tools/terminal/impl.py index b35ebeb3204..f2ca6283526 100644 --- a/openhands-tools/openhands/tools/terminal/impl.py +++ b/openhands-tools/openhands/tools/terminal/impl.py @@ -52,6 +52,7 @@ _TMUX_RECOVERABLE_ERROR_MARKERS = ( "no server running", + "server exited unexpectedly", "can't find session", "could not find window_id", "could not find pane_id", From eecd315d00225d34364b39c8ef107d08f3925b43 Mon Sep 17 00:00:00 2001 From: VascoSch92 Date: Mon, 16 Nov 2026 11:28:31 +0100 Subject: [PATCH 3/3] fix: recover a dead pane beside live ones; send text after -- With two or more panes, a shell exit leaves the tmux server up and the capture fails with "can't find pane", which was not a recoverable marker, so the dead pane went back to the pool. Before, a debug log line happened to raise a recoverable error at checkin. Literal text starting with "-" was read as send-keys flags; with Enter in the same invocation it was dropped too and the command hung until the timeout. Pass -- before the text. The process-count test now ignores capture-pane polls, whose number depends on timing. --- .../openhands/tools/terminal/impl.py | 1 + .../tools/terminal/terminal/tmux_terminal.py | 2 +- tests/tools/terminal/test_pool_integration.py | 16 +++++++++++++++ .../terminal/test_tmux_command_batching.py | 20 ++++++++++++------- 4 files changed, 31 insertions(+), 8 deletions(-) diff --git a/openhands-tools/openhands/tools/terminal/impl.py b/openhands-tools/openhands/tools/terminal/impl.py index f2ca6283526..ef3ac289005 100644 --- a/openhands-tools/openhands/tools/terminal/impl.py +++ b/openhands-tools/openhands/tools/terminal/impl.py @@ -54,6 +54,7 @@ "no server running", "server exited unexpectedly", "can't find session", + "can't find pane", "could not find window_id", "could not find pane_id", ) diff --git a/openhands-tools/openhands/tools/terminal/terminal/tmux_terminal.py b/openhands-tools/openhands/tools/terminal/terminal/tmux_terminal.py index d9a86345dc2..1a223eca592 100644 --- a/openhands-tools/openhands/tools/terminal/terminal/tmux_terminal.py +++ b/openhands-tools/openhands/tools/terminal/terminal/tmux_terminal.py @@ -393,7 +393,7 @@ def send_keys(self, text: str, enter: bool = True) -> None: # 3) Plain text — use literal=True so tmux doesn't split on # whitespace or interpret special tokens. if enter and not text.endswith("\n"): - self._send_then_enter("-l", text) + self._send_then_enter("-l", "--", text) else: self.pane.send_keys(text, enter=False, literal=True) diff --git a/tests/tools/terminal/test_pool_integration.py b/tests/tools/terminal/test_pool_integration.py index ed85414c5c2..0f8e7654410 100644 --- a/tests/tools/terminal/test_pool_integration.py +++ b/tests/tools/terminal/test_pool_integration.py @@ -144,6 +144,22 @@ def test_shell_exit_returns_actionable_error_and_rebuilds_pool(self, pool_execut assert after.exit_code == 0 assert "after_rebuild" in after.text + def test_shell_exit_beside_a_busy_pane_rebuilds_pool(self, pool_executor): + # The busy pane keeps the tmux server up, so the exited pane is gone + # while the server still answers. + busy = threading.Thread( + target=lambda: pool_executor(TerminalAction(command="sleep 3", timeout=10)) + ) + busy.start() + time.sleep(0.5) + obs = pool_executor(TerminalAction(command="exit 7", timeout=5.0)) + busy.join() + + assert obs.is_error + assert "rebuilt the terminal pool" in obs.text + after = pool_executor(TerminalAction(command="echo after_rebuild", timeout=5.0)) + assert "after_rebuild" in after.text + def test_reset_after_shell_exit_uses_rebuilt_pool(self, pool_executor): obs = pool_executor(TerminalAction(command="exit 0", timeout=1.0)) assert obs.is_error diff --git a/tests/tools/terminal/test_tmux_command_batching.py b/tests/tools/terminal/test_tmux_command_batching.py index f78522b5be4..c9873698a81 100644 --- a/tests/tools/terminal/test_tmux_command_batching.py +++ b/tests/tools/terminal/test_tmux_command_batching.py @@ -122,21 +122,27 @@ def executor(tmp_path): def test_command_runs_in_few_tmux_processes(executor, monkeypatch): executor(TerminalAction(command="echo warm")) - calls = 0 + commands: list[str] = [] real_init = libtmux.common.tmux_cmd.__init__ - def counting_init(self, *args, **kwargs): - nonlocal calls - calls += 1 + def recording_init(self, *args, **kwargs): + commands.append(next(a for a in args if not str(a).startswith("-"))) real_init(self, *args, **kwargs) - monkeypatch.setattr(libtmux.common.tmux_cmd, "__init__", counting_init) + monkeypatch.setattr(libtmux.common.tmux_cmd, "__init__", recording_init) observation = executor(TerminalAction(command="echo counted")) assert "counted" in observation.text # Each tmux process stalls the server's other threads while it is spawned; - # a command used to take 11. - assert calls <= 6 + # a command used to take 11. How often the screen is polled depends on + # timing, so only the rest is counted. + assert len([c for c in commands if c != "capture-pane"]) <= 3 + + +def test_command_starting_with_a_dash_runs(executor): + observation = executor(TerminalAction(command="-not-a-command", timeout=10)) + + assert observation.exit_code == 127 def test_command_ending_in_semicolon_runs(executor):