diff --git a/README.md b/README.md index f066689..1a984e2 100644 --- a/README.md +++ b/README.md @@ -171,6 +171,68 @@ pytest --testomatio report --test-id "Tc0880217|Tfd1c595c" Note: Test id should be started from letter "T" +### Submitting Test Steps + +The plugin supports dividing tests into separate, trackable steps. When reporting to testomat.io, you can view detailed information for each step including execution status, duration, and any errors that occurred. + +**Important**: This plugin only supports **reporting** test steps to testomat.io during test execution. Test steps cannot be imported to testomat.io using **sync** option. + +Test steps can be implemented using either decorators or context managers, giving you flexibility in how you structure your tests. + +**Error Handling**: If a step fails, the error is captured and reported to testomat.io while the test execution continues with remaining steps. + +**Parameters**: +``` +@step_function( + title = "Step1", # Step name displayed in testomat.io + category = "user" # Optional: categorize steps(user, system, framework) +) +with step( + title = "Step1", # Step name displayed in testomat.io + category = "user" # Optional: categorize steps(user, system, framework) +) + +``` + +**Example**: +```python +import pytest +from pytestomatio.utils.steps import step, step_function + +class Book: + def __init__(self, author, text): + self.author = author + self.text = text + + def read(self): + return self.text + +# decorator +@step_function(title='Check author step', category='user') +def check_author(author_name, expected_name): + assert author_name == expected_name + +def test_book_create(): + author_name = 'David Ket' + # context manager + with step(title='Book create', category='user'): + book = Book(author_name, 'text') + assert book + check_author(book.author, author_name) + +# nested steps also supported +def test_book_read(): + text = 'book text' + author_name = 'David Ket' + + with step(title='Read book'): + with step(title='Book create', category='user'): + book = Book(author_name, 'text') + assert book + check_author(book.author, author_name) + assert book.read() == text +``` + ### Configuration with environment variables You can use environment variable to control certain features of testomat.io diff --git a/pyproject.toml b/pyproject.toml index a9aa962..4836f13 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,8 +13,7 @@ version_provider = "pep621" update_changelog_on_bump = false [project] name = "pytestomatio" -version = "2.10.3b1" - +version = "2.11.0" dependencies = [ "requests>=2.32.4", diff --git a/pytestomatio/main.py b/pytestomatio/main.py index fafa036..eb58d3d 100644 --- a/pytestomatio/main.py +++ b/pytestomatio/main.py @@ -10,6 +10,7 @@ from pytestomatio.utils.helper import add_and_enrich_tests, get_test_mapping, collect_tests, read_env_s3_keys from pytestomatio.utils.parser_setup import parser_options +from pytestomatio.utils.steps import _step_managers from pytestomatio.utils import validations from pytestomatio.testomatio.testRunConfig import TestRunConfig @@ -29,6 +30,20 @@ def pytest_addoption(parser: Parser) -> None: parser_options(parser, testomatio) +def pytest_runtest_setup(item): + """Assign current item for test steps handling, clear step manager for current item if exists""" + pytest._current_item = item + if item.nodeid in _step_managers: + _step_managers.pop(item.nodeid) + + +def pytest_runtest_teardown(item, nextitem): + """Clear current item and current item step manager""" + if item.nodeid in _step_managers: + _step_managers.pop(item.nodeid) + pytest._current_item = None + + def pytest_collection(session): """Capture original collected items before any filters are applied.""" # This hook is called after initial test collection, before other filters. @@ -209,6 +224,10 @@ def pytest_runtest_makereport(item: Item, call: CallInfo): if hasattr(item, 'callspec'): request['example'] = test_item.safe_params(item.callspec.params) + step_manager = _step_managers.get(item.nodeid) + if step_manager: + request['steps'] = step_manager.get_steps() + if item.nodeid not in pytest.testomatio.test_run_config.status_request: pytest.testomatio.test_run_config.status_request[item.nodeid] = request else: diff --git a/pytestomatio/utils/steps.py b/pytestomatio/utils/steps.py new file mode 100644 index 0000000..799af4e --- /dev/null +++ b/pytestomatio/utils/steps.py @@ -0,0 +1,138 @@ +import functools +import time +import pytest + +from typing import Dict, List + + +_step_managers = {} + + +class Step: + step_categories = {'user', 'system', 'framework'} + + def __init__(self, title: str, category: str = None): + self.title = title + self.category = category if category in self.step_categories else None + self.start_time = None + self.end_time = None + self.status = None + self.error = None + self.children = [] + + @property + def duration(self): + if self.start_time and self.end_time: + return self.end_time - self.start_time + + def to_dict(self) -> Dict: + """Returns dict representation of Step""" + return { + "title": self.title, + "category": self.category, + "status": self.status, + "duration": self.duration, + "error": self.error, + "steps": [child.to_dict() for child in self.children] + } + + +class StepManager: + def __init__(self): + self.root_steps = [] + self._step_stack = [] + + def _get_current_step(self) -> Step: + return self._step_stack[-1] if self._step_stack else None + + def start_step(self, step: Step) -> Step: + """Handles start of test step""" + step.start_time = time.time() + current_step = self._get_current_step() + if current_step: + current_step.children.append(step) + else: + self.root_steps.append(step) + + self._step_stack.append(step) + return step + + def finish_step(self, step: Step, exc_type=None, exc_val=None, exc_tb=None): + """Handles end of test step. Updates step end time, status and errors if step or nested steps are failed""" + step.end_time = time.time() + failed_children = [child for child in step.children if child.status == "failed"] + if exc_type or failed_children: + step.status = "failed" + if exc_type: + step.error = str(exc_val) + elif failed_children: + step.error = f'Child step failed: {failed_children[0].title}' + else: + step.status = "passed" + + if self._step_stack and self._step_stack[-1] == step: + self._step_stack.pop() + + def get_steps(self) -> List[Dict]: + """Returns list of steps in manager""" + return [i.to_dict() for i in self.root_steps] + + +def get_step_manager() -> StepManager: + """Returns StepManager for current Pytest Test Item""" + item = getattr(pytest, '_current_item', None) + item_id = item.nodeid if item else None + + if item_id not in _step_managers: + _step_managers[item_id] = StepManager() + return _step_managers[item_id] + + +class StepContext: + def __init__(self, title: str, category: str = None): + self.step = Step(title, category) + self.manager = get_step_manager() + + def __enter__(self): + self.manager.start_step(self.step) + return self.step + + def __exit__(self, exc_type, exc_val, exc_tb): + self.manager.finish_step(self.step, exc_type, exc_val, exc_tb) + return False + + +def step(title: str, category: str = None): + """Context manager for test step + + :param title: name of test step + :param category: category of test step(user|framework|system) + """ + return StepContext(title, category) + + +def step_decorator(title: str, category: str = None): + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + manager = get_step_manager() + step_obj = Step(title, category) + manager.start_step(step_obj) + try: + result = func(*args, **kwargs) + manager.finish_step(step_obj) + return result + except Exception as e: + manager.finish_step(step_obj, type(e), e) + raise + return wrapper + return decorator + + +def step_function(title: str, category: str = None): + """Decorator for test step + + :param title: name of test step + :param category: category of test step(user|framework|system) + """ + return step_decorator(title, category) diff --git a/tests/test_utils/test_steps.py b/tests/test_utils/test_steps.py new file mode 100644 index 0000000..461f3e6 --- /dev/null +++ b/tests/test_utils/test_steps.py @@ -0,0 +1,309 @@ +from unittest.mock import patch, Mock + +import pytest + +from pytestomatio.utils.steps import Step, StepManager, _step_managers, StepContext, step, step_decorator, \ + get_step_manager + + +class TestStep: + """Test for Step class""" + + def test_step_init(self): + step = Step("Test Step") + assert step.title == 'Test Step' + assert step.category is None + assert step.start_time is None + assert step.end_time is None + assert step.status is None + assert step.error is None + assert step.children == [] + + def test_step_init_with_allowed_category(self): + step = Step("Test Step", "system") + assert step.title == 'Test Step' + assert step.category == 'system' + assert step.start_time is None + assert step.end_time is None + assert step.status is None + assert step.error is None + assert step.children == [] + + def test_step_init_with_not_allowed_category(self): + step = Step("Test Step", "New") + assert step.title == 'Test Step' + assert step.category is None + assert step.start_time is None + assert step.end_time is None + assert step.status is None + assert step.error is None + assert step.children == [] + + def test_step_duration_calculation(self): + step = Step("Test Step") + assert step.duration is None + step.start_time = 1.0 + assert step.duration is None + step.end_time = 2.5 + assert step.duration == 1.5 + + def test_step_to_dict(self): + step = Step("Test Step", 'system') + step.status = "passed" + step.start_time = 1.0 + step.end_time = 2.0 + + child = Step('Nested step') + child.status = 'failed' + child.error = 'Test error' + step.children.append(child) + + result = step.to_dict() + expected = { + "title": "Test Step", + "category": 'system', + "status": "passed", + "duration": 1.0, + "error": None, + "steps": [ + { + "title": "Nested step", + "category": None, + "status": "failed", + "duration": None, + "error": "Test error", + "steps": [] + } + ] + } + assert result == expected + + +class TestStepManager: + + @pytest.fixture + def step_manager(self): + return StepManager() + + def test_initial_state(self, step_manager): + assert step_manager.root_steps == [] + assert step_manager._step_stack == [] + + def test_start_root_step(self, step_manager): + step = Step('Test Step') + result = step_manager.start_step(step) + + assert result == step + assert step.start_time is not None + assert step in step_manager.root_steps + assert step in step_manager._step_stack + + def test_start_nested_step(self, step_manager): + root_step = Step('Root Step') + nested_step = Step('Nested Step') + step_manager.start_step(root_step) + step_manager.start_step(nested_step) + + assert nested_step in root_step.children + assert nested_step not in step_manager.root_steps + assert nested_step in step_manager._step_stack + + def test_finish_step_success(self, step_manager): + step = Step("Test step") + step_manager.start_step(step) + step_manager.finish_step(step) + + assert step.end_time is not None + assert step.status == "passed" + assert step.error is None + assert step not in step_manager._step_stack + + def test_finish_step_with_exception(self, step_manager): + step = Step("Test step") + step_manager.start_step(step) + + exc = ValueError("Test error") + step_manager.finish_step(step, ValueError, exc) + + assert step.status == "failed" + assert step.error == "Test error" + + def test_finish_step_with_failed_children(self, step_manager): + parent = Step("Parent step") + child = Step("Child step") + + step_manager.start_step(parent) + step_manager.start_step(child) + + step_manager.finish_step(child, ValueError, ValueError("Child error")) + step_manager.finish_step(parent) + + assert parent.status == "failed" + assert "Child step failed: Child step" in parent.error + + def test_get_current_step(self, step_manager): + assert step_manager._get_current_step() is None + + step1 = Step("Step 1") + step2 = Step("Step 2") + + step_manager.start_step(step1) + assert step_manager._get_current_step() == step1 + + step_manager.start_step(step2) + assert step_manager._get_current_step() == step2 + + step_manager.finish_step(step2) + assert step_manager._get_current_step() == step1 + + step_manager.finish_step(step1) + assert step_manager._get_current_step() is None + + def test_get_steps_without_nested_steps(self, step_manager): + step = Step("Step") + step_manager.start_step(step) + step_manager.finish_step(step) + + steps = step_manager.get_steps() + assert steps + assert len(steps) == 1 + assert steps[0].get('title') == step.title + + def test_get_steps_with_nested_steps(self, step_manager): + root_step = Step("Root Step") + step_manager.start_step(root_step) + nested_step = Step("Nested Step") + step_manager.start_step(nested_step) + step_manager.finish_step(nested_step) + step_manager.finish_step(root_step) + + steps = step_manager.get_steps() + assert steps + assert len(steps) == 1 + assert steps[0].get('title') == root_step.title + + nested_steps = steps[0].get('steps') + assert nested_steps + assert nested_steps[0].get('title') == nested_step.title + + +class TestStepContext: + + # def setup_method(self): + # # Очищаем глобальный state + # _managers.clear() + + @patch('pytestomatio.utils.steps.get_step_manager') + def test_context_manager_success(self, mock_get_manager): + mock_manager = Mock(spec=StepManager) + mock_get_manager.return_value = mock_manager + + with StepContext("Test step"): + pass + + assert mock_manager.start_step.call_count == 1 + assert mock_manager.finish_step.call_count == 1 + + start_args = mock_manager.start_step.call_args[0] + step = start_args[0] + assert step.title == "Test step" + + finish_args = mock_manager.finish_step.call_args[0] + assert finish_args[0] == step + assert finish_args[1] is None # exc_type + assert finish_args[2] is None # exc_val + + @patch('pytestomatio.utils.steps.get_step_manager') + def test_context_manager_with_exception(self, mock_get_manager): + mock_manager = Mock(spec=StepManager) + mock_get_manager.return_value = mock_manager + + with pytest.raises(ValueError, match="Test error"): + with StepContext("Test step"): + raise ValueError("Test error") + + finish_args = mock_manager.finish_step.call_args[0] + assert finish_args[1] == ValueError # exc_type + assert str(finish_args[2]) == "Test error" # exc_val + + class TestStepFunction: + + @patch('pytestomatio.utils.steps.get_step_manager') + def test_step_returns_context(self, mock_get_manager): + mock_manager = Mock(spec=StepManager) + mock_get_manager.return_value = mock_manager + + result = step("Test step") + assert isinstance(result, StepContext) + assert result.step.title == "Test step" + + +class TestStepDecorator: + + @patch('pytestomatio.utils.steps.get_step_manager') + def test_decorator_success(self, mock_get_manager): + mock_manager = Mock(spec=StepManager) + mock_get_manager.return_value = mock_manager + + @step_decorator("Decorated step") + def test_function(): + return "result" + + result = test_function() + + assert result == "result" + assert mock_manager.start_step.call_count == 1 + assert mock_manager.finish_step.call_count == 1 + + step_obj = mock_manager.start_step.call_args[0][0] + assert step_obj.title == "Decorated step" + + @patch('pytestomatio.utils.steps.get_step_manager') + def test_decorator_with_exception(self, mock_get_manager): + mock_manager = Mock(spec=StepManager) + mock_get_manager.return_value = mock_manager + + @step_decorator("Failing step") + def failing_function(): + raise RuntimeError("Function failed") + + with pytest.raises(RuntimeError, match="Function failed"): + failing_function() + + finish_call = mock_manager.finish_step.call_args[0] + assert finish_call[1] == RuntimeError + assert str(finish_call[2]) == "Function failed" + + def test_decorator_preserves_metadata(self): + @step_decorator("Test step") + def original_function(): + """Original docstring""" + pass + + assert original_function.__name__ == "original_function" + assert original_function.__doc__ == "Original docstring" + + + class TestGetStepManager: + + def setup_method(self): + _step_managers.clear() + + @patch('pytest._current_item') + def test_get_manager_with_pytest_item(self, mock_item): + mock_item.nodeid = "test_file.py::test_function" + + manager1 = get_step_manager() + manager2 = get_step_manager() + + assert isinstance(manager1, StepManager) + assert manager1 is manager2 + assert "test_file.py::test_function" in _step_managers + + def test_get_manager_without_pytest_item(self): + with patch('pytest._current_item') as item: + item.return_value = None + manager = get_step_manager() + assert isinstance(manager, StepManager) + +