diff --git a/README.md b/README.md index 555696b..b987530 100644 --- a/README.md +++ b/README.md @@ -170,13 +170,15 @@ You can use environment variable to control certain features of testomat.io #### Test Run configuration -| Env variable | What it does | Examples | -|--------------------------|----------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------| -| TESTOMATIO_TITLE | Name of a test run to create on testomat.io | TESTOMATIO_TITLE="Nightly Smoke Tests" pytest --testomatio report | -| TESTOMATIO_RUN_ID | Id of existing test run to use for sending test results to | TESTOMATIO_RUN_ID=98dfas0 pytest --testomatio report | -| TESTOMATIO_RUNGROUP_TITLE | Create a group (folder) for a test run. If group already exists, attach test run to it | TESTOMATIO_RUNGROUP_TITLE="Release 2.0" pytest --testomatio report | -| TESTOMATIO_ENV | Assign environment to a test run, env variant of **testRunEnv** option. Has a lower precedence than **testRunEnv** option. | TESTOMATIO_ENV="linux,chrome,1920x1080" pytest --testomatio report | -| TESTOMATIO_LABEL | Assign labels to a test run. Labels must exist in project and their scope must be enabled for runs | TESTOMATIO_LABEL="smoke,regression" pytest --testomatio report | +| Env variable | What it does | Examples | +|---------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------| +| TESTOMATIO_TITLE | Name of a test run to create on testomat.io | TESTOMATIO_TITLE="Nightly Smoke Tests" pytest --testomatio report | +| TESTOMATIO_RUN_ID | Id of existing test run to use for sending test results to | TESTOMATIO_RUN_ID=98dfas0 pytest --testomatio report | +| TESTOMATIO_RUNGROUP_TITLE | Create a group (folder) for a test run. If group already exists, attach test run to it | TESTOMATIO_RUNGROUP_TITLE="Release 2.0" pytest --testomatio report | +| TESTOMATIO_ENV | Assign environment to a test run, env variant of **testRunEnv** option. Has a lower precedence than **testRunEnv** option. | TESTOMATIO_ENV="linux,chrome,1920x1080" pytest --testomatio report | +| TESTOMATIO_LABEL | Assign labels to a test run. Labels must exist in project and their scope must be enabled for runs | TESTOMATIO_LABEL="smoke,regression" pytest --testomatio report | +| TESTOMATIO_NO_STEPS | Disable reporting of all steps completely. When enabled, no steps will be included in the test report regardless of test status | TESTOMATIO_NO_STEPS=True pytest --testomatio report | +| TESTOMATIO_STEPS_PASSED | Enable steps reporting for passed tests(disabled by default). When disabled, only failed and skipped tests will include step details to reduce noise | TESTOMATIO_STEPS_PASSED=True pytest --testomatio report | #### S3 Bucket configuration | Env variable | Description | @@ -200,6 +202,110 @@ testomatio_url = https://app.testomat.io ``` +### Submitting Test Run Environment + +to configure test environment, you can use additional option: + +```bash +pytest --testomatio report --testRunEnv "windows11,chrome,1920x1080" +``` + +Environment values are comma separated, please use double quotation. + +### Submitting Test Steps + +The plugin supports dividing tests into separate, trackable steps. When reporting to testomat.io, you can view detailed information for each step including execution status, duration, and any errors that occurred. + +**Important**: This plugin only supports **reporting** test steps to testomat.io during test execution. Test steps cannot be imported to testomat.io using **sync** option. Steps reported for skipped and failed test by default. To enable steps reporting for passed tests use **TESTOMATIO_STEPS_PASSED** env variable. + +Test steps can be implemented using either decorators or context managers, giving you flexibility in how you structure your tests. + +**Error Handling**: If a step fails, the error is captured and reported to testomat.io while the test execution continues with remaining steps. + +**Parameters**: +``` +@step_function( + title = "Step1", # Step name displayed in testomat.io + category = "user" # Optional: categorize steps(user, system, framework) +) +with step( + title = "Step1", # Step name displayed in testomat.io + category = "user" # Optional: categorize steps(user, system, framework) +) + +``` + +**Example**: +```python +import pytest +from pytestomatio.utils.steps import step, step_function + +class Book: + def __init__(self, author, text): + self.author = author + self.text = text + + def read(self): + return self.text + +# decorator +@step_function(title='Check author step', category='user') +def check_author(author_name, expected_name): + assert author_name == expected_name + +def test_book_create(): + author_name = 'David Ket' + # context manager + with step(title='Book create', category='user'): + book = Book(author_name, 'text') + assert book + check_author(book.author, author_name) + +# nested steps also supported +def test_book_read(): + text = 'book text' + author_name = 'David Ket' + + with step(title='Read book'): + with step(title='Book create', category='user'): + book = Book(author_name, 'text') + assert book + check_author(book.author, author_name) + assert book.read() == text +``` + +**Note:** Step is registered when the step code is executed. Therefore, if test mark as skipped(not executed at all) or test code execution stops before step code is executed, step will not be attached to test: +```python +import pytest +from pytestomatio.utils.steps import step + +# Step will not be added in report +@pytest.mark.skip +def test_skipped(): + with step('Step1', 'user'): + assert True + +# Step will not be added in report +def test_exception_raised(): + raise ValueError() + with step('Step1', 'user'): + assert True + +# Step will not be added in report +def test_early_skip(): + pytest.skip() + with step('Step1', 'user'): + assert True + +# Step1 will be added in report, Step2 will not be +def test_nested_step_skip_or_exception(): + with step('Step1', 'user'): + with step('Step2', 'user'): + pytest.skip() # or AttributeError() +``` + + + ### Submitting Test Artifacts testomat.io does not store any screenshots, logs or other artifacts. diff --git a/pyproject.toml b/pyproject.toml index 472690c..346faf3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,8 +13,7 @@ version_provider = "pep621" update_changelog_on_bump = false [project] name = "pytestomatio" -version = "2.10.1" - +version = "2.10.2b9" dependencies = [ "requests>=2.32.4", diff --git a/pytestomatio/main.py b/pytestomatio/main.py index 927e73d..8aa241b 100644 --- a/pytestomatio/main.py +++ b/pytestomatio/main.py @@ -8,6 +8,7 @@ from pytestomatio.utils.helper import add_and_enrich_tests, get_test_mapping, collect_tests, read_env_s3_keys from pytestomatio.utils.parser_setup import parser_options +from pytestomatio.utils.steps import _step_managers from pytestomatio.utils import validations from pytestomatio.testomatio.testRunConfig import TestRunConfig @@ -27,6 +28,20 @@ def pytest_addoption(parser: Parser) -> None: parser_options(parser, testomatio) +def pytest_runtest_setup(item): + """Assign current item for test steps handling, clear step manager for current item if exists""" + pytest._current_item = item + if item.nodeid in _step_managers: + _step_managers.pop(item.nodeid) + + +def pytest_runtest_teardown(item, nextitem): + """Clear current item and current item step manager""" + if item.nodeid in _step_managers: + _step_managers.pop(item.nodeid) + pytest._current_item = None + + def pytest_collection(session): """Capture original collected items before any filters are applied.""" # This hook is called after initial test collection, before other filters. @@ -178,6 +193,13 @@ def pytest_runtest_makereport(item: Item, call: CallInfo): if hasattr(item, 'callspec'): request['example'] = test_item.safe_params(item.callspec.params) + if not pytest.testomatio.test_run_config.disable_steps: + step_manager = _step_managers.get(item.nodeid) + if step_manager: + if call.excinfo is not None or \ + (call.excinfo is None and pytest.testomatio.test_run_config.enable_steps_for_passed_test): + request['steps'] = step_manager.get_steps() + if item.nodeid not in pytest.testomatio.test_run_config.status_request: pytest.testomatio.test_run_config.status_request[item.nodeid] = request else: diff --git a/pytestomatio/testomatio/testRunConfig.py b/pytestomatio/testomatio/testRunConfig.py index b58421f..ac569c4 100644 --- a/pytestomatio/testomatio/testRunConfig.py +++ b/pytestomatio/testomatio/testRunConfig.py @@ -11,8 +11,12 @@ def __init__(self): run_id = os.environ.get('TESTOMATIO_RUN_ID') or os.environ.get('TESTOMATIO_RUN') title = os.environ.get('TESTOMATIO_TITLE') if os.environ.get('TESTOMATIO_TITLE') else 'test run at ' + dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S") shared_run = os.environ.get('TESTOMATIO_SHARED_RUN') in ['True', 'true', '1'] + disable_steps = os.environ.get('TESTOMATIO_NO_STEPS') in ['True', 'true', '1'] + enable_steps_for_passed_test = os.environ.get('TESTOMATIO_STEPS_PASSED') in ['True', 'true', '1'] self.test_run_id = run_id self.title = title + self.enable_steps_for_passed_test = enable_steps_for_passed_test + self.disable_steps = disable_steps self.environment = safe_string_list(os.environ.get('TESTOMATIO_ENV')) self.label = safe_string_list(os.environ.get('TESTOMATIO_LABEL')) self.group_title = os.environ.get('TESTOMATIO_RUNGROUP_TITLE') diff --git a/pytestomatio/utils/steps.py b/pytestomatio/utils/steps.py new file mode 100644 index 0000000..799af4e --- /dev/null +++ b/pytestomatio/utils/steps.py @@ -0,0 +1,138 @@ +import functools +import time +import pytest + +from typing import Dict, List + + +_step_managers = {} + + +class Step: + step_categories = {'user', 'system', 'framework'} + + def __init__(self, title: str, category: str = None): + self.title = title + self.category = category if category in self.step_categories else None + self.start_time = None + self.end_time = None + self.status = None + self.error = None + self.children = [] + + @property + def duration(self): + if self.start_time and self.end_time: + return self.end_time - self.start_time + + def to_dict(self) -> Dict: + """Returns dict representation of Step""" + return { + "title": self.title, + "category": self.category, + "status": self.status, + "duration": self.duration, + "error": self.error, + "steps": [child.to_dict() for child in self.children] + } + + +class StepManager: + def __init__(self): + self.root_steps = [] + self._step_stack = [] + + def _get_current_step(self) -> Step: + return self._step_stack[-1] if self._step_stack else None + + def start_step(self, step: Step) -> Step: + """Handles start of test step""" + step.start_time = time.time() + current_step = self._get_current_step() + if current_step: + current_step.children.append(step) + else: + self.root_steps.append(step) + + self._step_stack.append(step) + return step + + def finish_step(self, step: Step, exc_type=None, exc_val=None, exc_tb=None): + """Handles end of test step. Updates step end time, status and errors if step or nested steps are failed""" + step.end_time = time.time() + failed_children = [child for child in step.children if child.status == "failed"] + if exc_type or failed_children: + step.status = "failed" + if exc_type: + step.error = str(exc_val) + elif failed_children: + step.error = f'Child step failed: {failed_children[0].title}' + else: + step.status = "passed" + + if self._step_stack and self._step_stack[-1] == step: + self._step_stack.pop() + + def get_steps(self) -> List[Dict]: + """Returns list of steps in manager""" + return [i.to_dict() for i in self.root_steps] + + +def get_step_manager() -> StepManager: + """Returns StepManager for current Pytest Test Item""" + item = getattr(pytest, '_current_item', None) + item_id = item.nodeid if item else None + + if item_id not in _step_managers: + _step_managers[item_id] = StepManager() + return _step_managers[item_id] + + +class StepContext: + def __init__(self, title: str, category: str = None): + self.step = Step(title, category) + self.manager = get_step_manager() + + def __enter__(self): + self.manager.start_step(self.step) + return self.step + + def __exit__(self, exc_type, exc_val, exc_tb): + self.manager.finish_step(self.step, exc_type, exc_val, exc_tb) + return False + + +def step(title: str, category: str = None): + """Context manager for test step + + :param title: name of test step + :param category: category of test step(user|framework|system) + """ + return StepContext(title, category) + + +def step_decorator(title: str, category: str = None): + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + manager = get_step_manager() + step_obj = Step(title, category) + manager.start_step(step_obj) + try: + result = func(*args, **kwargs) + manager.finish_step(step_obj) + return result + except Exception as e: + manager.finish_step(step_obj, type(e), e) + raise + return wrapper + return decorator + + +def step_function(title: str, category: str = None): + """Decorator for test step + + :param title: name of test step + :param category: category of test step(user|framework|system) + """ + return step_decorator(title, category) diff --git a/tests/test_testomatio/test_testRunConfig.py b/tests/test_testomatio/test_testRunConfig.py index 3acad4b..8c608fe 100644 --- a/tests/test_testomatio/test_testRunConfig.py +++ b/tests/test_testomatio/test_testRunConfig.py @@ -19,6 +19,8 @@ def test_init_default_values(self): assert config.test_run_id is None assert config.title == "test run at 2024-01-15 10:30:45" assert config.environment is None + assert config.enable_steps_for_passed_test is False + assert config.disable_steps is False assert config.label is None assert config.group_title is None assert config.parallel is True @@ -32,7 +34,9 @@ def test_init_with_env_variables(self): 'TESTOMATIO_TITLE': 'Custom Test Run', 'TESTOMATIO_ENV': 'linux,chrome,1920x1080', 'TESTOMATIO_LABEL': 'smoke,regression', - 'TESTOMATIO_RUNGROUP_TITLE': 'Release 2.0' + 'TESTOMATIO_RUNGROUP_TITLE': 'Release 2.0', + 'TESTOMATIO_NO_STEPS': '1', + 'TESTOMATIO_STEPS_PASSED': '1' } with patch.dict(os.environ, env_vars, clear=True): @@ -41,6 +45,8 @@ def test_init_with_env_variables(self): assert config.test_run_id == 'run_12345' assert config.title == 'Custom Test Run' assert config.environment == 'linux,chrome,1920x1080' + assert config.enable_steps_for_passed_test is True + assert config.disable_steps is True assert config.label == 'smoke,regression' assert config.group_title == 'Release 2.0' assert config.parallel is True @@ -64,6 +70,38 @@ def test_init_shared_run_false_variations(self, value): assert config.shared_run is False assert config.parallel is True + @pytest.mark.parametrize('value', ['True', 'true', '1']) + def test_init_disable_steps_true_variations(self, value): + """Test different true values for TESTOMATIO_NO_STEPS""" + with patch.dict(os.environ, {'TESTOMATIO_NO_STEPS': value}, clear=True): + config = TestRunConfig() + + assert config.disable_steps is True + + @pytest.mark.parametrize('value', ['False', 'false', '0', 'anything']) + def test_init_disable_steps_false_variations(self, value): + """Test different false values TESTOMATIO_NO_STEPS""" + with patch.dict(os.environ, {'TESTOMATIO_NO_STEPS': value}, clear=True): + config = TestRunConfig() + + assert config.disable_steps is False + + @pytest.mark.parametrize('value', ['True', 'true', '1']) + def test_enable_passed_steps_run_true_variations(self, value): + """Test different true values for TESTOMATIO_STEPS_PASSED""" + with patch.dict(os.environ, {'TESTOMATIO_STEPS_PASSED': value}, clear=True): + config = TestRunConfig() + + assert config.enable_steps_for_passed_test is True + + @pytest.mark.parametrize('value', ['False', 'false', '0', 'anything']) + def test_enable_passed_steps_false_variations(self, value): + """Test different false values TESTOMATIO_STEPS_PASSED""" + with patch.dict(os.environ, {'TESTOMATIO_STEPS_PASSED': value}, clear=True): + config = TestRunConfig() + + assert config.enable_steps_for_passed_test is False + def test_to_dict_full_data(self): """Test to_dict with full data""" env_vars = { diff --git a/tests/test_utils/test_steps.py b/tests/test_utils/test_steps.py new file mode 100644 index 0000000..461f3e6 --- /dev/null +++ b/tests/test_utils/test_steps.py @@ -0,0 +1,309 @@ +from unittest.mock import patch, Mock + +import pytest + +from pytestomatio.utils.steps import Step, StepManager, _step_managers, StepContext, step, step_decorator, \ + get_step_manager + + +class TestStep: + """Test for Step class""" + + def test_step_init(self): + step = Step("Test Step") + assert step.title == 'Test Step' + assert step.category is None + assert step.start_time is None + assert step.end_time is None + assert step.status is None + assert step.error is None + assert step.children == [] + + def test_step_init_with_allowed_category(self): + step = Step("Test Step", "system") + assert step.title == 'Test Step' + assert step.category == 'system' + assert step.start_time is None + assert step.end_time is None + assert step.status is None + assert step.error is None + assert step.children == [] + + def test_step_init_with_not_allowed_category(self): + step = Step("Test Step", "New") + assert step.title == 'Test Step' + assert step.category is None + assert step.start_time is None + assert step.end_time is None + assert step.status is None + assert step.error is None + assert step.children == [] + + def test_step_duration_calculation(self): + step = Step("Test Step") + assert step.duration is None + step.start_time = 1.0 + assert step.duration is None + step.end_time = 2.5 + assert step.duration == 1.5 + + def test_step_to_dict(self): + step = Step("Test Step", 'system') + step.status = "passed" + step.start_time = 1.0 + step.end_time = 2.0 + + child = Step('Nested step') + child.status = 'failed' + child.error = 'Test error' + step.children.append(child) + + result = step.to_dict() + expected = { + "title": "Test Step", + "category": 'system', + "status": "passed", + "duration": 1.0, + "error": None, + "steps": [ + { + "title": "Nested step", + "category": None, + "status": "failed", + "duration": None, + "error": "Test error", + "steps": [] + } + ] + } + assert result == expected + + +class TestStepManager: + + @pytest.fixture + def step_manager(self): + return StepManager() + + def test_initial_state(self, step_manager): + assert step_manager.root_steps == [] + assert step_manager._step_stack == [] + + def test_start_root_step(self, step_manager): + step = Step('Test Step') + result = step_manager.start_step(step) + + assert result == step + assert step.start_time is not None + assert step in step_manager.root_steps + assert step in step_manager._step_stack + + def test_start_nested_step(self, step_manager): + root_step = Step('Root Step') + nested_step = Step('Nested Step') + step_manager.start_step(root_step) + step_manager.start_step(nested_step) + + assert nested_step in root_step.children + assert nested_step not in step_manager.root_steps + assert nested_step in step_manager._step_stack + + def test_finish_step_success(self, step_manager): + step = Step("Test step") + step_manager.start_step(step) + step_manager.finish_step(step) + + assert step.end_time is not None + assert step.status == "passed" + assert step.error is None + assert step not in step_manager._step_stack + + def test_finish_step_with_exception(self, step_manager): + step = Step("Test step") + step_manager.start_step(step) + + exc = ValueError("Test error") + step_manager.finish_step(step, ValueError, exc) + + assert step.status == "failed" + assert step.error == "Test error" + + def test_finish_step_with_failed_children(self, step_manager): + parent = Step("Parent step") + child = Step("Child step") + + step_manager.start_step(parent) + step_manager.start_step(child) + + step_manager.finish_step(child, ValueError, ValueError("Child error")) + step_manager.finish_step(parent) + + assert parent.status == "failed" + assert "Child step failed: Child step" in parent.error + + def test_get_current_step(self, step_manager): + assert step_manager._get_current_step() is None + + step1 = Step("Step 1") + step2 = Step("Step 2") + + step_manager.start_step(step1) + assert step_manager._get_current_step() == step1 + + step_manager.start_step(step2) + assert step_manager._get_current_step() == step2 + + step_manager.finish_step(step2) + assert step_manager._get_current_step() == step1 + + step_manager.finish_step(step1) + assert step_manager._get_current_step() is None + + def test_get_steps_without_nested_steps(self, step_manager): + step = Step("Step") + step_manager.start_step(step) + step_manager.finish_step(step) + + steps = step_manager.get_steps() + assert steps + assert len(steps) == 1 + assert steps[0].get('title') == step.title + + def test_get_steps_with_nested_steps(self, step_manager): + root_step = Step("Root Step") + step_manager.start_step(root_step) + nested_step = Step("Nested Step") + step_manager.start_step(nested_step) + step_manager.finish_step(nested_step) + step_manager.finish_step(root_step) + + steps = step_manager.get_steps() + assert steps + assert len(steps) == 1 + assert steps[0].get('title') == root_step.title + + nested_steps = steps[0].get('steps') + assert nested_steps + assert nested_steps[0].get('title') == nested_step.title + + +class TestStepContext: + + # def setup_method(self): + # # Очищаем глобальный state + # _managers.clear() + + @patch('pytestomatio.utils.steps.get_step_manager') + def test_context_manager_success(self, mock_get_manager): + mock_manager = Mock(spec=StepManager) + mock_get_manager.return_value = mock_manager + + with StepContext("Test step"): + pass + + assert mock_manager.start_step.call_count == 1 + assert mock_manager.finish_step.call_count == 1 + + start_args = mock_manager.start_step.call_args[0] + step = start_args[0] + assert step.title == "Test step" + + finish_args = mock_manager.finish_step.call_args[0] + assert finish_args[0] == step + assert finish_args[1] is None # exc_type + assert finish_args[2] is None # exc_val + + @patch('pytestomatio.utils.steps.get_step_manager') + def test_context_manager_with_exception(self, mock_get_manager): + mock_manager = Mock(spec=StepManager) + mock_get_manager.return_value = mock_manager + + with pytest.raises(ValueError, match="Test error"): + with StepContext("Test step"): + raise ValueError("Test error") + + finish_args = mock_manager.finish_step.call_args[0] + assert finish_args[1] == ValueError # exc_type + assert str(finish_args[2]) == "Test error" # exc_val + + class TestStepFunction: + + @patch('pytestomatio.utils.steps.get_step_manager') + def test_step_returns_context(self, mock_get_manager): + mock_manager = Mock(spec=StepManager) + mock_get_manager.return_value = mock_manager + + result = step("Test step") + assert isinstance(result, StepContext) + assert result.step.title == "Test step" + + +class TestStepDecorator: + + @patch('pytestomatio.utils.steps.get_step_manager') + def test_decorator_success(self, mock_get_manager): + mock_manager = Mock(spec=StepManager) + mock_get_manager.return_value = mock_manager + + @step_decorator("Decorated step") + def test_function(): + return "result" + + result = test_function() + + assert result == "result" + assert mock_manager.start_step.call_count == 1 + assert mock_manager.finish_step.call_count == 1 + + step_obj = mock_manager.start_step.call_args[0][0] + assert step_obj.title == "Decorated step" + + @patch('pytestomatio.utils.steps.get_step_manager') + def test_decorator_with_exception(self, mock_get_manager): + mock_manager = Mock(spec=StepManager) + mock_get_manager.return_value = mock_manager + + @step_decorator("Failing step") + def failing_function(): + raise RuntimeError("Function failed") + + with pytest.raises(RuntimeError, match="Function failed"): + failing_function() + + finish_call = mock_manager.finish_step.call_args[0] + assert finish_call[1] == RuntimeError + assert str(finish_call[2]) == "Function failed" + + def test_decorator_preserves_metadata(self): + @step_decorator("Test step") + def original_function(): + """Original docstring""" + pass + + assert original_function.__name__ == "original_function" + assert original_function.__doc__ == "Original docstring" + + + class TestGetStepManager: + + def setup_method(self): + _step_managers.clear() + + @patch('pytest._current_item') + def test_get_manager_with_pytest_item(self, mock_item): + mock_item.nodeid = "test_file.py::test_function" + + manager1 = get_step_manager() + manager2 = get_step_manager() + + assert isinstance(manager1, StepManager) + assert manager1 is manager2 + assert "test_file.py::test_function" in _step_managers + + def test_get_manager_without_pytest_item(self): + with patch('pytest._current_item') as item: + item.return_value = None + manager = get_step_manager() + assert isinstance(manager, StepManager) + +