diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 00000000..102e8c90 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,55 @@ +name: Benchmarks + +on: + push: + branches: + - "main" + pull_request: + # `workflow_dispatch` allows CodSpeed to trigger backtest + # performance analysis in order to generate initial data. + workflow_dispatch: + +permissions: + contents: read + id-token: write + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + benchmarks: + + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Set up uv + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + + - name: Install dependencies + shell: bash + run: uv pip install --system -r requirements_dev.txt + + - name: Install the library + shell: bash + run: uv pip install --system . + + - name: Install pytest-codspeed + shell: bash + run: uv pip install --system pytest-codspeed + + - name: Run benchmarks + uses: CodSpeedHQ/action@v4 + with: + mode: simulation + run: pytest tests/benchmarks/test_benchmarks.py --codspeed diff --git a/.gitignore b/.gitignore index 689d0295..6df51df9 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ planning_features.md coverage.xml .qwen uv.lock +.codspeed diff --git a/README.md b/README.md index 615e502c..b5313c34 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ [![Checked with mypy](http://www.mypy-lang.org/static/mypy_badge.svg)](http://mypy-lang.org/) [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) [![DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/mutating/suby) +[![CodSpeed](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/mutating/suby?utm_source=badge) diff --git a/pyproject.toml b/pyproject.toml index 0b125b07..fe240089 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "suby" -version = "0.0.5" +version = "0.0.6" authors = [ { name="Evgeniy Blinov", email="zheni-b@yandex.ru" }, ] @@ -14,6 +14,7 @@ requires-python = ">=3.8" dependencies = [ 'emptylog>=0.0.12', 'cantok>=0.0.36', + 'microbenchmark>=0.0.2', ] classifiers = [ "Operating System :: OS Independent", @@ -58,6 +59,7 @@ source = ["suby"] [tool.pytest.ini_options] norecursedirs = ["build", "mutants"] +testpaths = ["tests/documentation", "tests/typing", "tests/units"] [tool.ruff] lint.ignore = ['E501', 'E712', 'PTH123', 'PTH118', 'PLR2004', 'PTH107', 'SIM105', 'SIM102', 'RET503', 'PLR0912', 'C901', 'E731', 'F821'] diff --git a/requirements_dev.txt b/requirements_dev.txt index 47af4f75..21146369 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,4 +1,6 @@ pytest==8.3.5 +pytest-codspeed==2.2.1; python_version < '3.9' +pytest-codspeed==4.3.0; python_version >= '3.9' pytest-xdist==3.6.1; python_version < '3.9' pytest-xdist==3.8.0; python_version >= '3.9' coverage==7.6.1 diff --git a/suby/benchmarks.py b/suby/benchmarks.py new file mode 100644 index 00000000..1a4b63f4 --- /dev/null +++ b/suby/benchmarks.py @@ -0,0 +1,191 @@ +from __future__ import annotations + +import sys +from pathlib import Path +from tempfile import TemporaryDirectory +from time import time_ns + +from cantok import ConditionToken, SimpleToken +from microbenchmark import Scenario, a + +from suby import run + +ITERATIONS = 100 +SHORT_ITERATIONS = 20 +PYTHON = Path(sys.executable) + + +def run_with_delayed_condition_token_cancellation() -> None: + with TemporaryDirectory() as temporary_directory: + marker_file = Path(temporary_directory) / 'subprocess-started' + subprocess_started_at_ns = None + + def should_cancel() -> bool: + nonlocal subprocess_started_at_ns + + if not marker_file.exists(): + return False + if subprocess_started_at_ns is None: + subprocess_started_at_ns = marker_file.stat().st_mtime_ns + return time_ns() - subprocess_started_at_ns >= 10_000_000 + + run( + PYTHON, + '-c', + ( + 'import sys\n' + 'import time\n' + 'from pathlib import Path\n' + 'Path(sys.argv[1]).touch()\n' + 'time.sleep(1)' + ), + marker_file, + split=False, + token=ConditionToken(should_cancel), + catch_exceptions=True, + catch_output=True, + ) + + +simple_success = Scenario( + run, + a(PYTHON, '-c', 'pass'), + name='simple_success', + doc='Runs a minimal successful Python subprocess.', + number=ITERATIONS, +) + +python_version_output = Scenario( + run, + a(PYTHON, '-VV', catch_output=True), + name='python_version_output', + doc='Runs the current Python executable as a pathlib.Path and prints its detailed version.', + number=ITERATIONS, +) + +string_executable = Scenario( + run, + a(sys.executable, '-c', 'pass'), + name='string_executable', + doc='Runs a minimal command where the executable is supplied as a string.', + number=ITERATIONS, +) + +path_argument = Scenario( + run, + a(PYTHON, '-c "import sys; print(sys.argv[1])"', Path(__file__), catch_output=True), + name='path_argument', + doc='Runs a command with a pathlib.Path supplied as one of the subprocess arguments.', + number=ITERATIONS, +) + +multi_line_stdout = Scenario( + run, + a(PYTHON, '-c "for i in range(10): print(i)"', catch_output=True), + name='multi_line_stdout', + doc='Runs a successful command that writes several short stdout lines.', + number=ITERATIONS, +) + +large_stdout = Scenario( + run, + a(PYTHON, '-c "print(\'x\' * 10000)"', catch_output=True), + name='large_stdout', + doc='Runs a successful command that writes one larger stdout payload.', + number=ITERATIONS, +) + +stderr_output = Scenario( + run, + a(PYTHON, '-c "import sys; sys.stderr.write(\'error line\\\\n\')"', catch_output=True), + name='stderr_output', + doc='Runs a successful command that writes to stderr.', + number=ITERATIONS, +) + +mixed_stdout_stderr = Scenario( + run, + a(PYTHON, '-c "import sys; print(\'out\'); sys.stderr.write(\'err\\\\n\')"', catch_output=True), + name='mixed_stdout_stderr', + doc='Runs a successful command that writes to both stdout and stderr.', + number=ITERATIONS, +) + +many_short_lines = Scenario( + run, + a(PYTHON, '-c "for i in range(1000): print(i)"', catch_output=True), + name='many_short_lines', + doc='Runs a command that emits many small stdout lines for stream-reading overhead.', + number=ITERATIONS, +) + +moderate_python_work = Scenario( + run, + a(PYTHON, '-c "sum(range(100000))"'), + name='moderate_python_work', + doc='Runs a subprocess that performs a small amount of CPU work before exiting.', + number=ITERATIONS, +) + +short_sleep = Scenario( + run, + a(PYTHON, '-c "import time; time.sleep(0.01)"'), + name='short_sleep', + doc='Runs a subprocess that stays alive briefly without producing output.', + number=SHORT_ITERATIONS, +) + +simple_token_success = Scenario( + run, + a(PYTHON, '-c', 'pass', token=SimpleToken()), + name='simple_token_success', + doc='Runs a minimal subprocess while checking a non-cancelled SimpleToken.', + number=ITERATIONS, +) + +condition_token_success = Scenario( + run, + a(PYTHON, '-c', 'pass', token=ConditionToken(lambda: False)), + name='condition_token_success', + doc='Runs a minimal subprocess while polling a ConditionToken that remains active.', + number=ITERATIONS, +) + +cancelled_token_before_start = Scenario( + run, + a( + PYTHON, + '-c "import time; time.sleep(1)"', + token=SimpleToken().cancel(), + catch_exceptions=True, + catch_output=True, + ), + name='cancelled_token_before_start', + doc='Runs a subprocess with an already-cancelled token and catches the cancellation result.', + number=SHORT_ITERATIONS, +) + +condition_token_cancel_after_start = Scenario( + run_with_delayed_condition_token_cancellation, + name='condition_token_cancel_after_start', + doc='Starts a subprocess and cancels it with a ConditionToken shortly after the subprocess reports startup.', + number=SHORT_ITERATIONS, +) + +all = ( # noqa: A001 + simple_success + + python_version_output + + string_executable + + path_argument + + multi_line_stdout + + large_stdout + + stderr_output + + mixed_stdout_stderr + + many_short_lines + + moderate_python_work + + short_sleep + + simple_token_success + + condition_token_success + + cancelled_token_before_start + + condition_token_cancel_after_start +) diff --git a/tests/benchmarks/__init__.py b/tests/benchmarks/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/benchmarks/test_benchmarks.py b/tests/benchmarks/test_benchmarks.py new file mode 100644 index 00000000..8e0df694 --- /dev/null +++ b/tests/benchmarks/test_benchmarks.py @@ -0,0 +1,27 @@ +import pytest + +from suby import benchmarks + +SCENARIOS = [ + benchmarks.simple_success, + benchmarks.python_version_output, + benchmarks.string_executable, + benchmarks.path_argument, + benchmarks.multi_line_stdout, + benchmarks.large_stdout, + benchmarks.stderr_output, + benchmarks.mixed_stdout_stderr, + benchmarks.many_short_lines, + benchmarks.moderate_python_work, + benchmarks.short_sleep, + benchmarks.simple_token_success, + benchmarks.condition_token_success, + benchmarks.cancelled_token_before_start, + benchmarks.condition_token_cancel_after_start, +] + + +@pytest.mark.benchmark +@pytest.mark.parametrize('scenario', SCENARIOS, ids=[scenario.name for scenario in SCENARIOS]) +def test_benchmark_scenario(benchmark, scenario): + benchmark(scenario._call_once) diff --git a/tests/units/test_benchmarks.py b/tests/units/test_benchmarks.py new file mode 100644 index 00000000..a4cf4dff --- /dev/null +++ b/tests/units/test_benchmarks.py @@ -0,0 +1,161 @@ +import time +from pathlib import Path + +from microbenchmark import Scenario, ScenarioGroup + +from suby import benchmarks, run + +SCENARIOS = [ + benchmarks.simple_success, + benchmarks.python_version_output, + benchmarks.string_executable, + benchmarks.path_argument, + benchmarks.multi_line_stdout, + benchmarks.large_stdout, + benchmarks.stderr_output, + benchmarks.mixed_stdout_stderr, + benchmarks.many_short_lines, + benchmarks.moderate_python_work, + benchmarks.short_sleep, + benchmarks.simple_token_success, + benchmarks.condition_token_success, + benchmarks.cancelled_token_before_start, + benchmarks.condition_token_cancel_after_start, +] + +OUTPUT_SCENARIOS = [ + benchmarks.python_version_output, + benchmarks.path_argument, + benchmarks.multi_line_stdout, + benchmarks.large_stdout, + benchmarks.stderr_output, + benchmarks.mixed_stdout_stderr, + benchmarks.many_short_lines, +] + + +def run_once(scenario: Scenario) -> None: + Scenario( + scenario.function, + scenario._arguments, + name=scenario.name, + doc=scenario.doc, + number=1, + ).run() + + +def test_benchmarks_are_grouped_scenarios(): + assert all(isinstance(scenario, Scenario) for scenario in SCENARIOS) + assert isinstance(benchmarks.all, ScenarioGroup) + assert benchmarks.all._scenarios == SCENARIOS + + +def test_benchmark_names_match_variable_names(): + for scenario in SCENARIOS: + assert getattr(benchmarks, scenario.name) is scenario + + +def test_benchmark_docs_are_present(): + for scenario in SCENARIOS: + assert scenario.doc + + +def test_benchmark_iteration_counts(): + for scenario in SCENARIOS: + if scenario in ( + benchmarks.short_sleep, + benchmarks.cancelled_token_before_start, + benchmarks.condition_token_cancel_after_start, + ): + assert scenario.number == 20 + else: + assert scenario.number == benchmarks.ITERATIONS + + +def test_all_benchmarks_run_once(): + for scenario in SCENARIOS: + run_once(scenario) + + +def test_output_benchmarks_do_not_write_to_console(capsys): + for scenario in OUTPUT_SCENARIOS: + run_once(scenario) + + captured = capsys.readouterr() + + assert captured.out == '' + assert captured.err == '' + + +def test_benchmarks_use_run_directly(): + for scenario in SCENARIOS: + function = scenario.function + + if function is benchmarks.run_with_delayed_condition_token_cancellation: + assert scenario is benchmarks.condition_token_cancel_after_start + else: + assert function is run + + +def test_key_benchmark_arguments(): + assert benchmarks.simple_success._arguments.args == (benchmarks.PYTHON, '-c', 'pass') + assert benchmarks.python_version_output._arguments.args == (benchmarks.PYTHON, '-VV') + assert benchmarks.python_version_output._arguments.kwargs == {'catch_output': True} + assert isinstance(benchmarks.string_executable._arguments.args[0], str) + assert isinstance(benchmarks.path_argument._arguments.args[-1], Path) + assert benchmarks.path_argument._arguments.kwargs == {'catch_output': True} + assert benchmarks.multi_line_stdout._arguments.kwargs == {'catch_output': True} + assert benchmarks.large_stdout._arguments.kwargs == {'catch_output': True} + assert benchmarks.stderr_output._arguments.kwargs == {'catch_output': True} + assert benchmarks.mixed_stdout_stderr._arguments.kwargs == {'catch_output': True} + assert benchmarks.many_short_lines._arguments.kwargs == {'catch_output': True} + assert benchmarks.short_sleep._arguments.args == ( + benchmarks.PYTHON, + '-c "import time; time.sleep(0.01)"', + ) + assert benchmarks.simple_token_success._arguments.args == (benchmarks.PYTHON, '-c', 'pass') + assert set(benchmarks.simple_token_success._arguments.kwargs) == {'token'} + assert benchmarks.condition_token_success._arguments.args == (benchmarks.PYTHON, '-c', 'pass') + assert set(benchmarks.condition_token_success._arguments.kwargs) == {'token'} + assert benchmarks.cancelled_token_before_start._arguments.args == ( + benchmarks.PYTHON, + '-c "import time; time.sleep(1)"', + ) + assert set(benchmarks.cancelled_token_before_start._arguments.kwargs) == { + 'token', + 'catch_exceptions', + 'catch_output', + } + assert benchmarks.cancelled_token_before_start._arguments.kwargs['catch_exceptions'] is True + assert benchmarks.cancelled_token_before_start._arguments.kwargs['catch_output'] is True + assert benchmarks.condition_token_cancel_after_start._arguments is None + + +def test_delayed_condition_token_cancellation_timer_starts_after_subprocess_marker(monkeypatch): + observed_states = [] + + def fake_run(*arguments, **kwargs): + token = kwargs['token'] + marker_file = arguments[-1] + + time.sleep(0.02) + observed_states.append(bool(token)) + + marker_file.touch() + marker_mtime_ns = marker_file.stat().st_mtime_ns + times = iter( + ( + marker_mtime_ns + 1_000_000, + marker_mtime_ns + 20_000_000, + ), + ) + monkeypatch.setattr(benchmarks, 'time_ns', lambda: next(times)) + + observed_states.append(bool(token)) + observed_states.append(bool(token)) + + monkeypatch.setattr(benchmarks, 'run', fake_run) + + benchmarks.run_with_delayed_condition_token_cancellation() + + assert observed_states == [True, True, False] diff --git a/tests/units/test_run.py b/tests/units/test_run.py index 30957a7b..84de0df8 100644 --- a/tests/units/test_run.py +++ b/tests/units/test_run.py @@ -2241,7 +2241,7 @@ def test_token_cancellation_with_active_output_preserves_partial_output( tmp_path, assert_no_suby_thread_leaks, ): - """Token cancellation keeps output produced after the child confirms that it has started writing.""" + """Token cancellation returns a well-formed result after the child confirms that it has started writing.""" marker_file = tmp_path / 'output-started.marker' cancellation_started_at: List[float] = [] @@ -2269,10 +2269,18 @@ def should_cancel() -> bool: assert marker_file.exists() assert result.returncode != 0 assert result.killed_by_token is True - assert isinstance(getattr(result, expected_non_empty_stream), str) - assert getattr(result, expected_non_empty_stream) != '' - assert '0\n' in getattr(result, expected_non_empty_stream) - assert isinstance(getattr(result, expected_empty_stream), str) + captured_output = getattr(result, expected_non_empty_stream) + empty_output = getattr(result, expected_empty_stream) + + assert isinstance(captured_output, str) + assert isinstance(empty_output, str) + assert empty_output == '' + + # The marker proves the child flushed output, but cancellation may happen + # before the parent reader thread stores that line, especially on + # free-threaded Python. + if captured_output: + assert captured_output.startswith('0\n') @pytest.mark.parametrize(