From 7936f6a70108e9c1fd98a6ab0d21cab009af16ac Mon Sep 17 00:00:00 2001 From: AenEnlil Date: Fri, 12 Sep 2025 18:38:53 +0300 Subject: [PATCH 1/9] implemented test steps reporting through decorator and context manager --- pytestomatio/main.py | 19 ++++++ pytestomatio/utils/steps.py | 126 ++++++++++++++++++++++++++++++++++++ 2 files changed, 145 insertions(+) create mode 100644 pytestomatio/utils/steps.py diff --git a/pytestomatio/main.py b/pytestomatio/main.py index 927e73d..5382ee3 100644 --- a/pytestomatio/main.py +++ b/pytestomatio/main.py @@ -8,6 +8,7 @@ from pytestomatio.utils.helper import add_and_enrich_tests, get_test_mapping, collect_tests, read_env_s3_keys from pytestomatio.utils.parser_setup import parser_options +from pytestomatio.utils.steps import _step_managers from pytestomatio.utils import validations from pytestomatio.testomatio.testRunConfig import TestRunConfig @@ -27,6 +28,20 @@ def pytest_addoption(parser: Parser) -> None: parser_options(parser, testomatio) +def pytest_runtest_setup(item): + """Assign current item for test steps handling, clear step manager for current item if exists""" + pytest._current_item = item + if item.nodeid in _step_managers: + _step_managers.pop(item.nodeid) + + +def pytest_runtest_teardown(item, nextitem): + """Clear current item and current item step manager""" + if item.nodeid in _step_managers: + _step_managers.pop(item.nodeid) + pytest._current_item = None + + def pytest_collection(session): """Capture original collected items before any filters are applied.""" # This hook is called after initial test collection, before other filters. @@ -178,6 +193,10 @@ def pytest_runtest_makereport(item: Item, call: CallInfo): if hasattr(item, 'callspec'): request['example'] = test_item.safe_params(item.callspec.params) + step_manager = _step_managers.get(item.nodeid) + if step_manager: + request['steps'] = step_manager.get_steps() + if item.nodeid not in pytest.testomatio.test_run_config.status_request: pytest.testomatio.test_run_config.status_request[item.nodeid] = request else: diff --git a/pytestomatio/utils/steps.py b/pytestomatio/utils/steps.py new file mode 100644 index 0000000..76a4d2b --- /dev/null +++ b/pytestomatio/utils/steps.py @@ -0,0 +1,126 @@ +import functools +import time +import pytest + +from typing import Dict, List + + +_step_managers = {} + + +class Step: + def __init__(self, title: str): + self.title = title + self.start_time = None + self.end_time = None + self.status = None + self.error = None + self.children = [] + + @property + def duration(self): + if self.start_time and self.end_time: + return self.end_time - self.start_time + + def to_dict(self) -> Dict: + """Returns dict representation of Step""" + return { + "title": self.title, + "status": self.status, + "duration": self.duration, + "error": self.error, + "steps": [child.to_dict() for child in self.children] + } + + +class StepManager: + def __init__(self): + self.root_steps = [] + self._step_stack = [] + + def _get_current_step(self) -> Step: + return self._step_stack[-1] if self._step_stack else None + + def start_step(self, step: Step) -> Step: + """Handles start of test step""" + step.start_time = time.time() + current_step = self._get_current_step() + if current_step: + current_step.children.append(step) + else: + self.root_steps.append(step) + + self._step_stack.append(step) + return step + + def finish_step(self, step: Step, exc_type=None, exc_val=None, exc_tb=None): + """Handles end of test step. Updates step end time, status and errors if step or nested steps are failed""" + step.end_time = time.time() + failed_children = [child for child in step.children if child.status == "failed"] + if exc_type or failed_children: + step.status = "failed" + if exc_type: + step.error = str(exc_val) + elif failed_children: + step.error = f'Child step failed: {failed_children[0].name}' + else: + step.status = "passed" + + if self._step_stack and self._step_stack[-1] == step: + self._step_stack.pop() + + def get_steps(self) -> List[Dict]: + """Returns list of steps in manager""" + return [i.to_dict() for i in self.root_steps] + + +def get_step_manager() -> StepManager: + """Returns StepManager for current Pytest Test Item""" + item = getattr(pytest, '_current_item', None) + item_id = item.nodeid if item else None + + if item_id not in _step_managers: + _step_managers[item_id] = StepManager() + return _step_managers[item_id] + + +class StepContext: + def __init__(self, title: str): + self.step = Step(title) + self.manager = get_step_manager() + + def __enter__(self): + self.manager.start_step(self.step) + return self.step + + def __exit__(self, exc_type, exc_val, exc_tb): + self.manager.finish_step(self.step, exc_type, exc_val, exc_tb) + return False + + +def step(title: str): + """Context manager for test step""" + return StepContext(title) + + +def step_decorator(title: str): + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + manager = get_step_manager() + step_obj = Step(title) + manager.start_step(step_obj) + try: + result = func(*args, **kwargs) + manager.finish_step(step_obj) + return result + except Exception as e: + manager.finish_step(step_obj, type(e), e) + raise + return wrapper + return decorator + + +def step_function(name: str): + """Decorator for test step""" + return step_decorator(name) From fc98d681839ee7d4b2447e4a323d7bbcb285e5a7 Mon Sep 17 00:00:00 2001 From: AenEnlil Date: Fri, 12 Sep 2025 19:54:46 +0300 Subject: [PATCH 2/9] added unit tests --- pytestomatio/utils/steps.py | 2 +- tests/test_utils/test_steps.py | 286 +++++++++++++++++++++++++++++++++ 2 files changed, 287 insertions(+), 1 deletion(-) create mode 100644 tests/test_utils/test_steps.py diff --git a/pytestomatio/utils/steps.py b/pytestomatio/utils/steps.py index 76a4d2b..c47b815 100644 --- a/pytestomatio/utils/steps.py +++ b/pytestomatio/utils/steps.py @@ -62,7 +62,7 @@ def finish_step(self, step: Step, exc_type=None, exc_val=None, exc_tb=None): if exc_type: step.error = str(exc_val) elif failed_children: - step.error = f'Child step failed: {failed_children[0].name}' + step.error = f'Child step failed: {failed_children[0].title}' else: step.status = "passed" diff --git a/tests/test_utils/test_steps.py b/tests/test_utils/test_steps.py new file mode 100644 index 0000000..9a1e821 --- /dev/null +++ b/tests/test_utils/test_steps.py @@ -0,0 +1,286 @@ +from unittest.mock import patch, Mock + +import pytest + +from pytestomatio.utils.steps import Step, StepManager, _step_managers, StepContext, step, step_decorator, \ + get_step_manager + + +class TestStep: + """Test for Step class""" + + def test_step_init(self): + step = Step("Test Step") + assert step.title == 'Test Step' + assert step.start_time is None + assert step.end_time is None + assert step.status is None + assert step.error is None + assert step.children == [] + + def test_step_duration_calculation(self): + step = Step("Test Step") + assert step.duration is None + step.start_time = 1.0 + assert step.duration is None + step.end_time = 2.5 + assert step.duration == 1.5 + + def test_step_to_dict(self): + step = Step("Test Step") + step.status = "passed" + step.start_time = 1.0 + step.end_time = 2.0 + + child = Step('Nested step') + child.status = 'failed' + child.error = 'Test error' + step.children.append(child) + + result = step.to_dict() + expected = { + "title": "Test Step", + "status": "passed", + "duration": 1.0, + "error": None, + "steps": [ + { + "title": "Nested step", + "status": "failed", + "duration": None, + "error": "Test error", + "steps": [] + } + ] + } + assert result == expected + + +class TestStepManager: + + @pytest.fixture + def step_manager(self): + return StepManager() + + def test_initial_state(self, step_manager): + assert step_manager.root_steps == [] + assert step_manager._step_stack == [] + + def test_start_root_step(self, step_manager): + step = Step('Test Step') + result = step_manager.start_step(step) + + assert result == step + assert step.start_time is not None + assert step in step_manager.root_steps + assert step in step_manager._step_stack + + def test_start_nested_step(self, step_manager): + root_step = Step('Root Step') + nested_step = Step('Nested Step') + step_manager.start_step(root_step) + step_manager.start_step(nested_step) + + assert nested_step in root_step.children + assert nested_step not in step_manager.root_steps + assert nested_step in step_manager._step_stack + + def test_finish_step_success(self, step_manager): + step = Step("Test step") + step_manager.start_step(step) + step_manager.finish_step(step) + + assert step.end_time is not None + assert step.status == "passed" + assert step.error is None + assert step not in step_manager._step_stack + + def test_finish_step_with_exception(self, step_manager): + step = Step("Test step") + step_manager.start_step(step) + + exc = ValueError("Test error") + step_manager.finish_step(step, ValueError, exc) + + assert step.status == "failed" + assert step.error == "Test error" + + def test_finish_step_with_failed_children(self, step_manager): + parent = Step("Parent step") + child = Step("Child step") + + step_manager.start_step(parent) + step_manager.start_step(child) + + step_manager.finish_step(child, ValueError, ValueError("Child error")) + step_manager.finish_step(parent) + + assert parent.status == "failed" + assert "Child step failed: Child step" in parent.error + + def test_get_current_step(self, step_manager): + assert step_manager._get_current_step() is None + + step1 = Step("Step 1") + step2 = Step("Step 2") + + step_manager.start_step(step1) + assert step_manager._get_current_step() == step1 + + step_manager.start_step(step2) + assert step_manager._get_current_step() == step2 + + step_manager.finish_step(step2) + assert step_manager._get_current_step() == step1 + + step_manager.finish_step(step1) + assert step_manager._get_current_step() is None + + def test_get_steps_without_nested_steps(self, step_manager): + step = Step("Step") + step_manager.start_step(step) + step_manager.finish_step(step) + + steps = step_manager.get_steps() + assert steps + assert len(steps) == 1 + assert steps[0].get('title') == step.title + + def test_get_steps_with_nested_steps(self, step_manager): + root_step = Step("Root Step") + step_manager.start_step(root_step) + nested_step = Step("Nested Step") + step_manager.start_step(nested_step) + step_manager.finish_step(nested_step) + step_manager.finish_step(root_step) + + steps = step_manager.get_steps() + assert steps + assert len(steps) == 1 + assert steps[0].get('title') == root_step.title + + nested_steps = steps[0].get('steps') + assert nested_steps + assert nested_steps[0].get('title') == nested_step.title + + +class TestStepContext: + + # def setup_method(self): + # # Очищаем глобальный state + # _managers.clear() + + @patch('pytestomatio.utils.steps.get_step_manager') + def test_context_manager_success(self, mock_get_manager): + mock_manager = Mock(spec=StepManager) + mock_get_manager.return_value = mock_manager + + with StepContext("Test step"): + pass + + assert mock_manager.start_step.call_count == 1 + assert mock_manager.finish_step.call_count == 1 + + start_args = mock_manager.start_step.call_args[0] + step = start_args[0] + assert step.title == "Test step" + + finish_args = mock_manager.finish_step.call_args[0] + assert finish_args[0] == step + assert finish_args[1] is None # exc_type + assert finish_args[2] is None # exc_val + + @patch('pytestomatio.utils.steps.get_step_manager') + def test_context_manager_with_exception(self, mock_get_manager): + mock_manager = Mock(spec=StepManager) + mock_get_manager.return_value = mock_manager + + with pytest.raises(ValueError, match="Test error"): + with StepContext("Test step"): + raise ValueError("Test error") + + finish_args = mock_manager.finish_step.call_args[0] + assert finish_args[1] == ValueError # exc_type + assert str(finish_args[2]) == "Test error" # exc_val + + class TestStepFunction: + + @patch('pytestomatio.utils.steps.get_step_manager') + def test_step_returns_context(self, mock_get_manager): + mock_manager = Mock(spec=StepManager) + mock_get_manager.return_value = mock_manager + + result = step("Test step") + assert isinstance(result, StepContext) + assert result.step.title == "Test step" + + +class TestStepDecorator: + + @patch('pytestomatio.utils.steps.get_step_manager') + def test_decorator_success(self, mock_get_manager): + mock_manager = Mock(spec=StepManager) + mock_get_manager.return_value = mock_manager + + @step_decorator("Decorated step") + def test_function(): + return "result" + + result = test_function() + + assert result == "result" + assert mock_manager.start_step.call_count == 1 + assert mock_manager.finish_step.call_count == 1 + + step_obj = mock_manager.start_step.call_args[0][0] + assert step_obj.title == "Decorated step" + + @patch('pytestomatio.utils.steps.get_step_manager') + def test_decorator_with_exception(self, mock_get_manager): + mock_manager = Mock(spec=StepManager) + mock_get_manager.return_value = mock_manager + + @step_decorator("Failing step") + def failing_function(): + raise RuntimeError("Function failed") + + with pytest.raises(RuntimeError, match="Function failed"): + failing_function() + + finish_call = mock_manager.finish_step.call_args[0] + assert finish_call[1] == RuntimeError + assert str(finish_call[2]) == "Function failed" + + def test_decorator_preserves_metadata(self): + @step_decorator("Test step") + def original_function(): + """Original docstring""" + pass + + assert original_function.__name__ == "original_function" + assert original_function.__doc__ == "Original docstring" + + + class TestGetStepManager: + + def setup_method(self): + _step_managers.clear() + + @patch('pytest._current_item') + def test_get_manager_with_pytest_item(self, mock_item): + mock_item.nodeid = "test_file.py::test_function" + + manager1 = get_step_manager() + manager2 = get_step_manager() + + assert isinstance(manager1, StepManager) + assert manager1 is manager2 + assert "test_file.py::test_function" in _step_managers + + def test_get_manager_without_pytest_item(self): + with patch('pytest._current_item') as item: + item.return_value = None + manager = get_step_manager() + assert isinstance(manager, StepManager) + + From 7632e0ec88b5ac3a5f76cd08d3e4b2dba0001cc6 Mon Sep 17 00:00:00 2001 From: AenEnlil Date: Mon, 15 Sep 2025 11:59:12 +0300 Subject: [PATCH 3/9] added category and options fields --- pytestomatio/utils/steps.py | 32 ++++++++++++++++++-------- tests/test_utils/test_steps.py | 42 +++++++++++++++++++++++++++++++++- 2 files changed, 64 insertions(+), 10 deletions(-) diff --git a/pytestomatio/utils/steps.py b/pytestomatio/utils/steps.py index c47b815..a78e8c5 100644 --- a/pytestomatio/utils/steps.py +++ b/pytestomatio/utils/steps.py @@ -8,14 +8,26 @@ _step_managers = {} +class StepOptions: + def __init__(self, box: bool = False): + self.box = box + + def to_dict(self): + return { + "box": self.box + } + + class Step: - def __init__(self, title: str): + def __init__(self, title: str, category: str = None, options: StepOptions = None): self.title = title + self.category = category self.start_time = None self.end_time = None self.status = None self.error = None self.children = [] + self.options = options.to_dict() if options else options @property def duration(self): @@ -26,9 +38,11 @@ def to_dict(self) -> Dict: """Returns dict representation of Step""" return { "title": self.title, + "category": self.category, "status": self.status, "duration": self.duration, "error": self.error, + "options": self.options, "steps": [child.to_dict() for child in self.children] } @@ -85,8 +99,8 @@ def get_step_manager() -> StepManager: class StepContext: - def __init__(self, title: str): - self.step = Step(title) + def __init__(self, title: str, category: str = None, options: StepOptions = None): + self.step = Step(title, category, options) self.manager = get_step_manager() def __enter__(self): @@ -98,17 +112,17 @@ def __exit__(self, exc_type, exc_val, exc_tb): return False -def step(title: str): +def step(title: str, category: str = None, options: StepOptions = None): """Context manager for test step""" - return StepContext(title) + return StepContext(title, category, options) -def step_decorator(title: str): +def step_decorator(title: str, category: str = None, options: StepOptions = None): def decorator(func): @functools.wraps(func) def wrapper(*args, **kwargs): manager = get_step_manager() - step_obj = Step(title) + step_obj = Step(title, category, options) manager.start_step(step_obj) try: result = func(*args, **kwargs) @@ -121,6 +135,6 @@ def wrapper(*args, **kwargs): return decorator -def step_function(name: str): +def step_function(name: str, category: str = None, options: StepOptions = None): """Decorator for test step""" - return step_decorator(name) + return step_decorator(name, category, options) diff --git a/tests/test_utils/test_steps.py b/tests/test_utils/test_steps.py index 9a1e821..412b038 100644 --- a/tests/test_utils/test_steps.py +++ b/tests/test_utils/test_steps.py @@ -3,7 +3,19 @@ import pytest from pytestomatio.utils.steps import Step, StepManager, _step_managers, StepContext, step, step_decorator, \ - get_step_manager + get_step_manager, StepOptions + + +class TestStepOptions: + + def test_options_init(self): + options = StepOptions(box=True) + assert options.box is True + + def test_to_dict(self): + options = StepOptions(box=True) + result = options.to_dict() + assert result['box'] == options.box class TestStep: @@ -12,10 +24,34 @@ class TestStep: def test_step_init(self): step = Step("Test Step") assert step.title == 'Test Step' + assert step.category is None + assert step.start_time is None + assert step.end_time is None + assert step.status is None + assert step.error is None + assert step.options is None + assert step.children == [] + + def test_step_init_with_category(self): + step = Step("Test Step", "New") + assert step.title == 'Test Step' + assert step.category == 'New' + assert step.start_time is None + assert step.end_time is None + assert step.status is None + assert step.error is None + assert step.options is None + assert step.children == [] + + def test_step_init_with_options(self): + step = Step("Test Step", options=StepOptions(box=True)) + assert step.title == 'Test Step' + assert step.category is None assert step.start_time is None assert step.end_time is None assert step.status is None assert step.error is None + assert step.options == {'box': True} assert step.children == [] def test_step_duration_calculation(self): @@ -40,15 +76,19 @@ def test_step_to_dict(self): result = step.to_dict() expected = { "title": "Test Step", + "category": None, "status": "passed", "duration": 1.0, "error": None, + "options": None, "steps": [ { "title": "Nested step", + "category": None, "status": "failed", "duration": None, "error": "Test error", + "options": None, "steps": [] } ] From c15cce1c59b6757a426217ebe759fa77cfc7815a Mon Sep 17 00:00:00 2001 From: AenEnlil Date: Mon, 15 Sep 2025 14:43:08 +0300 Subject: [PATCH 4/9] documentation updated --- README.md | 29 +++++++++++++++++++++++++++++ pytestomatio/utils/steps.py | 4 ++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b8592e0..8e551e4 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,35 @@ pytest --testomatio report --testRunEnv "windows11,chrome,1920x1080" Environment values are comma separated, please use double quotation. +### Submitting Test Steps + +Plugin supports structural division of the test into separate steps. When reporting to +testomat.io, along with general information about the test, data for each step is displayed(execution status, time, error). + +Test steps implemented as decorator and as Context Manager. + +```python +import pytest +from pytestomatio.utils.steps import step, step_function + +class Book: + def __init__(self, author): + self.author = author + +# decorator +@step_function(title='Check author step', category='check', options=None) +def check_author(author_name, expected_name): + assert author_name == expected_name + +def test_book_create(): + author_name = 'David Ket' + # context manager + with step(title='Book create', category='create'): + book = Book(author_name) + assert book + check_author(book.author, author_name) +``` + ### Submitting Test Artifacts diff --git a/pytestomatio/utils/steps.py b/pytestomatio/utils/steps.py index a78e8c5..9949b88 100644 --- a/pytestomatio/utils/steps.py +++ b/pytestomatio/utils/steps.py @@ -135,6 +135,6 @@ def wrapper(*args, **kwargs): return decorator -def step_function(name: str, category: str = None, options: StepOptions = None): +def step_function(title: str, category: str = None, options: StepOptions = None): """Decorator for test step""" - return step_decorator(name, category, options) + return step_decorator(title, category, options) From 50def69ed986805f4751aefe64ec1dc79e094eeb Mon Sep 17 00:00:00 2001 From: AenEnlil Date: Mon, 15 Sep 2025 16:07:09 +0300 Subject: [PATCH 5/9] removed StepOptions, updated category field --- pytestomatio/utils/steps.py | 46 ++++++++++++++++------------------ tests/test_utils/test_steps.py | 33 ++++++------------------ 2 files changed, 30 insertions(+), 49 deletions(-) diff --git a/pytestomatio/utils/steps.py b/pytestomatio/utils/steps.py index 9949b88..799af4e 100644 --- a/pytestomatio/utils/steps.py +++ b/pytestomatio/utils/steps.py @@ -8,26 +8,17 @@ _step_managers = {} -class StepOptions: - def __init__(self, box: bool = False): - self.box = box - - def to_dict(self): - return { - "box": self.box - } - - class Step: - def __init__(self, title: str, category: str = None, options: StepOptions = None): + step_categories = {'user', 'system', 'framework'} + + def __init__(self, title: str, category: str = None): self.title = title - self.category = category + self.category = category if category in self.step_categories else None self.start_time = None self.end_time = None self.status = None self.error = None self.children = [] - self.options = options.to_dict() if options else options @property def duration(self): @@ -42,7 +33,6 @@ def to_dict(self) -> Dict: "status": self.status, "duration": self.duration, "error": self.error, - "options": self.options, "steps": [child.to_dict() for child in self.children] } @@ -99,8 +89,8 @@ def get_step_manager() -> StepManager: class StepContext: - def __init__(self, title: str, category: str = None, options: StepOptions = None): - self.step = Step(title, category, options) + def __init__(self, title: str, category: str = None): + self.step = Step(title, category) self.manager = get_step_manager() def __enter__(self): @@ -112,17 +102,21 @@ def __exit__(self, exc_type, exc_val, exc_tb): return False -def step(title: str, category: str = None, options: StepOptions = None): - """Context manager for test step""" - return StepContext(title, category, options) +def step(title: str, category: str = None): + """Context manager for test step + :param title: name of test step + :param category: category of test step(user|framework|system) + """ + return StepContext(title, category) -def step_decorator(title: str, category: str = None, options: StepOptions = None): + +def step_decorator(title: str, category: str = None): def decorator(func): @functools.wraps(func) def wrapper(*args, **kwargs): manager = get_step_manager() - step_obj = Step(title, category, options) + step_obj = Step(title, category) manager.start_step(step_obj) try: result = func(*args, **kwargs) @@ -135,6 +129,10 @@ def wrapper(*args, **kwargs): return decorator -def step_function(title: str, category: str = None, options: StepOptions = None): - """Decorator for test step""" - return step_decorator(title, category, options) +def step_function(title: str, category: str = None): + """Decorator for test step + + :param title: name of test step + :param category: category of test step(user|framework|system) + """ + return step_decorator(title, category) diff --git a/tests/test_utils/test_steps.py b/tests/test_utils/test_steps.py index 412b038..461f3e6 100644 --- a/tests/test_utils/test_steps.py +++ b/tests/test_utils/test_steps.py @@ -3,19 +3,7 @@ import pytest from pytestomatio.utils.steps import Step, StepManager, _step_managers, StepContext, step, step_decorator, \ - get_step_manager, StepOptions - - -class TestStepOptions: - - def test_options_init(self): - options = StepOptions(box=True) - assert options.box is True - - def test_to_dict(self): - options = StepOptions(box=True) - result = options.to_dict() - assert result['box'] == options.box + get_step_manager class TestStep: @@ -29,29 +17,26 @@ def test_step_init(self): assert step.end_time is None assert step.status is None assert step.error is None - assert step.options is None assert step.children == [] - def test_step_init_with_category(self): - step = Step("Test Step", "New") + def test_step_init_with_allowed_category(self): + step = Step("Test Step", "system") assert step.title == 'Test Step' - assert step.category == 'New' + assert step.category == 'system' assert step.start_time is None assert step.end_time is None assert step.status is None assert step.error is None - assert step.options is None assert step.children == [] - def test_step_init_with_options(self): - step = Step("Test Step", options=StepOptions(box=True)) + def test_step_init_with_not_allowed_category(self): + step = Step("Test Step", "New") assert step.title == 'Test Step' assert step.category is None assert step.start_time is None assert step.end_time is None assert step.status is None assert step.error is None - assert step.options == {'box': True} assert step.children == [] def test_step_duration_calculation(self): @@ -63,7 +48,7 @@ def test_step_duration_calculation(self): assert step.duration == 1.5 def test_step_to_dict(self): - step = Step("Test Step") + step = Step("Test Step", 'system') step.status = "passed" step.start_time = 1.0 step.end_time = 2.0 @@ -76,11 +61,10 @@ def test_step_to_dict(self): result = step.to_dict() expected = { "title": "Test Step", - "category": None, + "category": 'system', "status": "passed", "duration": 1.0, "error": None, - "options": None, "steps": [ { "title": "Nested step", @@ -88,7 +72,6 @@ def test_step_to_dict(self): "status": "failed", "duration": None, "error": "Test error", - "options": None, "steps": [] } ] From 88a48a23357b1a370f4387263d62b635fcb97d55 Mon Sep 17 00:00:00 2001 From: AenEnlil Date: Mon, 15 Sep 2025 16:31:32 +0300 Subject: [PATCH 6/9] readme updated --- README.md | 47 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 8e551e4..083097f 100644 --- a/README.md +++ b/README.md @@ -83,31 +83,64 @@ Environment values are comma separated, please use double quotation. ### Submitting Test Steps -Plugin supports structural division of the test into separate steps. When reporting to -testomat.io, along with general information about the test, data for each step is displayed(execution status, time, error). +The plugin supports dividing tests into separate, trackable steps. When reporting to testomat.io, you can view detailed information for each step including execution status, duration, and any errors that occurred. -Test steps implemented as decorator and as Context Manager. +**Important**: This plugin only supports **reporting** test steps to testomat.io during test execution. Test steps cannot be imported to testomat.io using **sync** option. +Test steps can be implemented using either decorators or context managers, giving you flexibility in how you structure your tests. + +**Error Handling**: If a step fails, the error is captured and reported to testomat.io while the test execution continues with remaining steps. + +**Parameters**: +``` +@step_function( + title = "Step1", # Step name displayed in testomat.io + category = "user" # Optional: categorize steps(user, system, framework) +) +with step( + title = "Step1", # Step name displayed in testomat.io + category = "user" # Optional: categorize steps(user, system, framework) +) + +``` + +**Example**: ```python import pytest from pytestomatio.utils.steps import step, step_function class Book: - def __init__(self, author): + def __init__(self, author, text): self.author = author + self.text = text + + def read(self): + return self.text # decorator -@step_function(title='Check author step', category='check', options=None) +@step_function(title='Check author step', category='user') def check_author(author_name, expected_name): assert author_name == expected_name def test_book_create(): author_name = 'David Ket' # context manager - with step(title='Book create', category='create'): - book = Book(author_name) + with step(title='Book create', category='user'): + book = Book(author_name, 'text') assert book check_author(book.author, author_name) + +# nested steps also supported +def test_book_read(): + text = 'book text' + author_name = 'David Ket' + + with step(title='Read book'): + with step(title='Book create', category='user'): + book = Book(author_name, 'text') + assert book + check_author(book.author, author_name) + assert book.read() == text ``` From d8d26704274083c53df257a0f779861cfbcd4030 Mon Sep 17 00:00:00 2001 From: AenEnlil Date: Mon, 15 Sep 2025 16:39:16 +0300 Subject: [PATCH 7/9] bump: version 2.10.0 -> 2.10.2b0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index bacde0a..fdfb5ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ version_provider = "pep621" update_changelog_on_bump = false [project] name = "pytestomatio" -version = "2.10.0" +version = "2.10.2b0" dependencies = [ "requests>=2.29.0", From cc80b730946ac0232443d15fc9a1d52d188f7577 Mon Sep 17 00:00:00 2001 From: AenEnlil Date: Fri, 26 Sep 2025 17:11:40 +0300 Subject: [PATCH 8/9] implemented steps reporting configuration. Now steps of skipped or failed tests reported by default. For reporting passed test steps added separate option. Also added option to disable steps reporting for all tests. --- README.md | 49 +++++++++++++++++---- pytestomatio/main.py | 9 ++-- pytestomatio/testomatio/testRunConfig.py | 4 ++ tests/test_testomatio/test_testRunConfig.py | 40 ++++++++++++++++- 4 files changed, 90 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index d2e0a8b..b987530 100644 --- a/README.md +++ b/README.md @@ -170,13 +170,15 @@ You can use environment variable to control certain features of testomat.io #### Test Run configuration -| Env variable | What it does | Examples | -|--------------------------|----------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------| -| TESTOMATIO_TITLE | Name of a test run to create on testomat.io | TESTOMATIO_TITLE="Nightly Smoke Tests" pytest --testomatio report | -| TESTOMATIO_RUN_ID | Id of existing test run to use for sending test results to | TESTOMATIO_RUN_ID=98dfas0 pytest --testomatio report | -| TESTOMATIO_RUNGROUP_TITLE | Create a group (folder) for a test run. If group already exists, attach test run to it | TESTOMATIO_RUNGROUP_TITLE="Release 2.0" pytest --testomatio report | -| TESTOMATIO_ENV | Assign environment to a test run, env variant of **testRunEnv** option. Has a lower precedence than **testRunEnv** option. | TESTOMATIO_ENV="linux,chrome,1920x1080" pytest --testomatio report | -| TESTOMATIO_LABEL | Assign labels to a test run. Labels must exist in project and their scope must be enabled for runs | TESTOMATIO_LABEL="smoke,regression" pytest --testomatio report | +| Env variable | What it does | Examples | +|---------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------| +| TESTOMATIO_TITLE | Name of a test run to create on testomat.io | TESTOMATIO_TITLE="Nightly Smoke Tests" pytest --testomatio report | +| TESTOMATIO_RUN_ID | Id of existing test run to use for sending test results to | TESTOMATIO_RUN_ID=98dfas0 pytest --testomatio report | +| TESTOMATIO_RUNGROUP_TITLE | Create a group (folder) for a test run. If group already exists, attach test run to it | TESTOMATIO_RUNGROUP_TITLE="Release 2.0" pytest --testomatio report | +| TESTOMATIO_ENV | Assign environment to a test run, env variant of **testRunEnv** option. Has a lower precedence than **testRunEnv** option. | TESTOMATIO_ENV="linux,chrome,1920x1080" pytest --testomatio report | +| TESTOMATIO_LABEL | Assign labels to a test run. Labels must exist in project and their scope must be enabled for runs | TESTOMATIO_LABEL="smoke,regression" pytest --testomatio report | +| TESTOMATIO_NO_STEPS | Disable reporting of all steps completely. When enabled, no steps will be included in the test report regardless of test status | TESTOMATIO_NO_STEPS=True pytest --testomatio report | +| TESTOMATIO_STEPS_PASSED | Enable steps reporting for passed tests(disabled by default). When disabled, only failed and skipped tests will include step details to reduce noise | TESTOMATIO_STEPS_PASSED=True pytest --testomatio report | #### S3 Bucket configuration | Env variable | Description | @@ -214,7 +216,7 @@ Environment values are comma separated, please use double quotation. The plugin supports dividing tests into separate, trackable steps. When reporting to testomat.io, you can view detailed information for each step including execution status, duration, and any errors that occurred. -**Important**: This plugin only supports **reporting** test steps to testomat.io during test execution. Test steps cannot be imported to testomat.io using **sync** option. +**Important**: This plugin only supports **reporting** test steps to testomat.io during test execution. Test steps cannot be imported to testomat.io using **sync** option. Steps reported for skipped and failed test by default. To enable steps reporting for passed tests use **TESTOMATIO_STEPS_PASSED** env variable. Test steps can be implemented using either decorators or context managers, giving you flexibility in how you structure your tests. @@ -272,6 +274,37 @@ def test_book_read(): assert book.read() == text ``` +**Note:** Step is registered when the step code is executed. Therefore, if test mark as skipped(not executed at all) or test code execution stops before step code is executed, step will not be attached to test: +```python +import pytest +from pytestomatio.utils.steps import step + +# Step will not be added in report +@pytest.mark.skip +def test_skipped(): + with step('Step1', 'user'): + assert True + +# Step will not be added in report +def test_exception_raised(): + raise ValueError() + with step('Step1', 'user'): + assert True + +# Step will not be added in report +def test_early_skip(): + pytest.skip() + with step('Step1', 'user'): + assert True + +# Step1 will be added in report, Step2 will not be +def test_nested_step_skip_or_exception(): + with step('Step1', 'user'): + with step('Step2', 'user'): + pytest.skip() # or AttributeError() +``` + + ### Submitting Test Artifacts diff --git a/pytestomatio/main.py b/pytestomatio/main.py index 5382ee3..8aa241b 100644 --- a/pytestomatio/main.py +++ b/pytestomatio/main.py @@ -193,9 +193,12 @@ def pytest_runtest_makereport(item: Item, call: CallInfo): if hasattr(item, 'callspec'): request['example'] = test_item.safe_params(item.callspec.params) - step_manager = _step_managers.get(item.nodeid) - if step_manager: - request['steps'] = step_manager.get_steps() + if not pytest.testomatio.test_run_config.disable_steps: + step_manager = _step_managers.get(item.nodeid) + if step_manager: + if call.excinfo is not None or \ + (call.excinfo is None and pytest.testomatio.test_run_config.enable_steps_for_passed_test): + request['steps'] = step_manager.get_steps() if item.nodeid not in pytest.testomatio.test_run_config.status_request: pytest.testomatio.test_run_config.status_request[item.nodeid] = request diff --git a/pytestomatio/testomatio/testRunConfig.py b/pytestomatio/testomatio/testRunConfig.py index b58421f..ac569c4 100644 --- a/pytestomatio/testomatio/testRunConfig.py +++ b/pytestomatio/testomatio/testRunConfig.py @@ -11,8 +11,12 @@ def __init__(self): run_id = os.environ.get('TESTOMATIO_RUN_ID') or os.environ.get('TESTOMATIO_RUN') title = os.environ.get('TESTOMATIO_TITLE') if os.environ.get('TESTOMATIO_TITLE') else 'test run at ' + dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S") shared_run = os.environ.get('TESTOMATIO_SHARED_RUN') in ['True', 'true', '1'] + disable_steps = os.environ.get('TESTOMATIO_NO_STEPS') in ['True', 'true', '1'] + enable_steps_for_passed_test = os.environ.get('TESTOMATIO_STEPS_PASSED') in ['True', 'true', '1'] self.test_run_id = run_id self.title = title + self.enable_steps_for_passed_test = enable_steps_for_passed_test + self.disable_steps = disable_steps self.environment = safe_string_list(os.environ.get('TESTOMATIO_ENV')) self.label = safe_string_list(os.environ.get('TESTOMATIO_LABEL')) self.group_title = os.environ.get('TESTOMATIO_RUNGROUP_TITLE') diff --git a/tests/test_testomatio/test_testRunConfig.py b/tests/test_testomatio/test_testRunConfig.py index 3acad4b..8c608fe 100644 --- a/tests/test_testomatio/test_testRunConfig.py +++ b/tests/test_testomatio/test_testRunConfig.py @@ -19,6 +19,8 @@ def test_init_default_values(self): assert config.test_run_id is None assert config.title == "test run at 2024-01-15 10:30:45" assert config.environment is None + assert config.enable_steps_for_passed_test is False + assert config.disable_steps is False assert config.label is None assert config.group_title is None assert config.parallel is True @@ -32,7 +34,9 @@ def test_init_with_env_variables(self): 'TESTOMATIO_TITLE': 'Custom Test Run', 'TESTOMATIO_ENV': 'linux,chrome,1920x1080', 'TESTOMATIO_LABEL': 'smoke,regression', - 'TESTOMATIO_RUNGROUP_TITLE': 'Release 2.0' + 'TESTOMATIO_RUNGROUP_TITLE': 'Release 2.0', + 'TESTOMATIO_NO_STEPS': '1', + 'TESTOMATIO_STEPS_PASSED': '1' } with patch.dict(os.environ, env_vars, clear=True): @@ -41,6 +45,8 @@ def test_init_with_env_variables(self): assert config.test_run_id == 'run_12345' assert config.title == 'Custom Test Run' assert config.environment == 'linux,chrome,1920x1080' + assert config.enable_steps_for_passed_test is True + assert config.disable_steps is True assert config.label == 'smoke,regression' assert config.group_title == 'Release 2.0' assert config.parallel is True @@ -64,6 +70,38 @@ def test_init_shared_run_false_variations(self, value): assert config.shared_run is False assert config.parallel is True + @pytest.mark.parametrize('value', ['True', 'true', '1']) + def test_init_disable_steps_true_variations(self, value): + """Test different true values for TESTOMATIO_NO_STEPS""" + with patch.dict(os.environ, {'TESTOMATIO_NO_STEPS': value}, clear=True): + config = TestRunConfig() + + assert config.disable_steps is True + + @pytest.mark.parametrize('value', ['False', 'false', '0', 'anything']) + def test_init_disable_steps_false_variations(self, value): + """Test different false values TESTOMATIO_NO_STEPS""" + with patch.dict(os.environ, {'TESTOMATIO_NO_STEPS': value}, clear=True): + config = TestRunConfig() + + assert config.disable_steps is False + + @pytest.mark.parametrize('value', ['True', 'true', '1']) + def test_enable_passed_steps_run_true_variations(self, value): + """Test different true values for TESTOMATIO_STEPS_PASSED""" + with patch.dict(os.environ, {'TESTOMATIO_STEPS_PASSED': value}, clear=True): + config = TestRunConfig() + + assert config.enable_steps_for_passed_test is True + + @pytest.mark.parametrize('value', ['False', 'false', '0', 'anything']) + def test_enable_passed_steps_false_variations(self, value): + """Test different false values TESTOMATIO_STEPS_PASSED""" + with patch.dict(os.environ, {'TESTOMATIO_STEPS_PASSED': value}, clear=True): + config = TestRunConfig() + + assert config.enable_steps_for_passed_test is False + def test_to_dict_full_data(self): """Test to_dict with full data""" env_vars = { From 15d5778043b6ca8604d5f4875e4073aebf68c179 Mon Sep 17 00:00:00 2001 From: AenEnlil Date: Fri, 26 Sep 2025 18:28:53 +0300 Subject: [PATCH 9/9] bump: version 2.10.2b0 -> 2.10.2b9 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e7ba53b..346faf3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ version_provider = "pep621" update_changelog_on_bump = false [project] name = "pytestomatio" -version = "2.10.2b0" +version = "2.10.2b9" dependencies = [ "requests>=2.32.4",