From 93d457d15a0109a5f8ee97d571b19cc216a51316 Mon Sep 17 00:00:00 2001 From: ed Date: Fri, 1 Jul 2022 17:24:51 +0200 Subject: [PATCH 1/6] add pytest driver --- setup.cfg | 1 + xtesting/core/pytest.py | 33 +++++++++++++++++++++++++++++++++ xtesting/utils/pytesthooks.py | 25 +++++++++++++++++++++++++ 3 files changed, 59 insertions(+) create mode 100755 xtesting/core/pytest.py create mode 100644 xtesting/utils/pytesthooks.py diff --git a/setup.cfg b/setup.cfg index 944e2d0f..06ff9950 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,6 +31,7 @@ xtesting.testcase = second = xtesting.samples.second:Test mts = xtesting.core.mts:MTSLauncher ansible = xtesting.core.ansible:Ansible + pytest = xtesting.core.pytest:Pytest [build_sphinx] all_files = 1 diff --git a/xtesting/core/pytest.py b/xtesting/core/pytest.py new file mode 100755 index 00000000..9a0ed1af --- /dev/null +++ b/xtesting/core/pytest.py @@ -0,0 +1,33 @@ +#! /usr/bin/env python3 + +import logging +import time + +import pytest +import xtesting + + +class Pytest(xtesting.core.testcase.TestCase): + """pytest runner.""" + + __logger = logging.getLogger(__name__) + + def run(self, **kwargs): + try: + dir = kwargs.pop('dir') + options = kwargs.pop('options', []) + except KeyError as err: + self.__logger.exception(f"Missing args: {err.args[0]!r}") + return self.EX_RUN_ERROR + if kwargs: + self.__logger.exception(f"Unexpected args {kwargs}") + return self.EX_RUN_ERROR + + self.start_time = time.time() + + pytest.main(args=['--tb=no', '-p', 'xtesting.utils.pytesthooks'] + options + [dir]) + + self.stop_time = time.time() + self.details = xtesting.utils.pytesthooks.details + self.result = 100 + return self.EX_OK diff --git a/xtesting/utils/pytesthooks.py b/xtesting/utils/pytesthooks.py new file mode 100644 index 00000000..1d880f14 --- /dev/null +++ b/xtesting/utils/pytesthooks.py @@ -0,0 +1,25 @@ +import pytest + +details = None + + +def pytest_sessionstart(session): + global details + details = dict(tests=[]) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + global details + outcome = yield + result = outcome.get_result() + if result.when == 'call': + if result.outcome == 'passed': + test = dict(status='PASSED', result=call.result) + elif result.outcome == 'failed': + test = dict(status='FAILED', result=call.excinfo) + elif result.outcome == 'skipped': + test = dict(status='SKIPPED', result=call.excinfo) + else: + test = {} + details['tests'].append(test) From 6c622f7463249c1d11a230a9d3c0995bd0ff9c54 Mon Sep 17 00:00:00 2001 From: ed Date: Mon, 4 Jul 2022 09:09:56 +0200 Subject: [PATCH 2/6] tests result --- xtesting/core/pytest.py | 53 +++++++++++++++++++++++++++-------- xtesting/utils/pytesthooks.py | 27 +++++++++++++----- 2 files changed, 61 insertions(+), 19 deletions(-) diff --git a/xtesting/core/pytest.py b/xtesting/core/pytest.py index 9a0ed1af..1e73b917 100755 --- a/xtesting/core/pytest.py +++ b/xtesting/core/pytest.py @@ -8,26 +8,55 @@ class Pytest(xtesting.core.testcase.TestCase): - """pytest runner.""" + """pytest driver - __logger = logging.getLogger(__name__) + The 'pytest driver' can be used on every test set written for pytest. + Given a pytest package that is launched with the command: - def run(self, **kwargs): + `pytest --opt1 arg1 --opt2 testdir` + + it can be executed by xtesting with the following testcase.yaml: + + ```yaml + run: + name: pytest + args: + dir: testdir + options: + opt1: arg1 + opt2: null + ``` + + options can be written as a list of strings `['--opt1', 'arg1', '--opt2']` + """ + + logger = logging.getLogger('pytest') + + def run(self, **args): + # parsing args + # - 'dir' is mandatory + # - 'options' is an optional list or a dict flatten to a list try: - dir = kwargs.pop('dir') - options = kwargs.pop('options', []) + dir = args.pop('dir') + options = args.pop('options', []) + if isinstance(options, dict): + options = [v for o in + zip([f'--{k}' if len(str(k)) > 1 else f'-{k}' for k in options], options.values()) + for v in o if v is not None] except KeyError as err: - self.__logger.exception(f"Missing args: {err.args[0]!r}") + self.logger.exception(f"Missing args: {err.args[0]!r}") return self.EX_RUN_ERROR - if kwargs: - self.__logger.exception(f"Unexpected args {kwargs}") + if args: + self.logger.exception(f"Unexpected args {args}") return self.EX_RUN_ERROR + # running pytest with 'options' in 'dir' + # the pytesthooks initiates an empty 'details' and populates it with individual test results self.start_time = time.time() - pytest.main(args=['--tb=no', '-p', 'xtesting.utils.pytesthooks'] + options + [dir]) - self.stop_time = time.time() + + # fectching results in pytesthooks self.details = xtesting.utils.pytesthooks.details - self.result = 100 - return self.EX_OK + self.result = xtesting.utils.pytesthooks.result + return self.EX_OK if xtesting.utils.pytesthooks.ok else self.EX_NOK diff --git a/xtesting/utils/pytesthooks.py b/xtesting/utils/pytesthooks.py index 1d880f14..c14688dc 100644 --- a/xtesting/utils/pytesthooks.py +++ b/xtesting/utils/pytesthooks.py @@ -1,25 +1,38 @@ import pytest details = None +passed = None +failed = None +result = None +ok = None def pytest_sessionstart(session): - global details + global details, passed, failed, result, ok details = dict(tests=[]) + passed = 0 + failed = 0 + result = 0 + ok = True @pytest.hookimpl(hookwrapper=True) def pytest_runtest_makereport(item, call): - global details + global details, passed, failed, result, ok outcome = yield - result = outcome.get_result() - if result.when == 'call': - if result.outcome == 'passed': + outcome = outcome.get_result() + if call.when == 'call': + if outcome == 'passed': + passed += 1 test = dict(status='PASSED', result=call.result) - elif result.outcome == 'failed': + elif outcome == 'failed': + failed += 1 + ok = False test = dict(status='FAILED', result=call.excinfo) - elif result.outcome == 'skipped': + elif outcome == 'skipped': test = dict(status='SKIPPED', result=call.excinfo) else: test = {} + if passed + failed: + result = passed / (passed + failed) details['tests'].append(test) From 4871753b8c285b022cc4b8a8cb5bdb97792f7dc8 Mon Sep 17 00:00:00 2001 From: ed Date: Mon, 4 Jul 2022 17:18:42 +0200 Subject: [PATCH 3/6] reorder pytest arguments --- xtesting/core/pytest.py | 4 ++-- xtesting/utils/pytesthooks.py | 7 ++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/xtesting/core/pytest.py b/xtesting/core/pytest.py index 1e73b917..a3eebdf1 100755 --- a/xtesting/core/pytest.py +++ b/xtesting/core/pytest.py @@ -53,10 +53,10 @@ def run(self, **args): # running pytest with 'options' in 'dir' # the pytesthooks initiates an empty 'details' and populates it with individual test results self.start_time = time.time() - pytest.main(args=['--tb=no', '-p', 'xtesting.utils.pytesthooks'] + options + [dir]) + pytest.main(args=[dir] + ['--tb=no', '-p', 'xtesting.utils.pytesthooks'] + options) self.stop_time = time.time() # fectching results in pytesthooks self.details = xtesting.utils.pytesthooks.details self.result = xtesting.utils.pytesthooks.result - return self.EX_OK if xtesting.utils.pytesthooks.ok else self.EX_NOK + return self.EX_OK diff --git a/xtesting/utils/pytesthooks.py b/xtesting/utils/pytesthooks.py index c14688dc..9afe0b3f 100644 --- a/xtesting/utils/pytesthooks.py +++ b/xtesting/utils/pytesthooks.py @@ -4,21 +4,19 @@ passed = None failed = None result = None -ok = None def pytest_sessionstart(session): - global details, passed, failed, result, ok + global details, passed, failed, result details = dict(tests=[]) passed = 0 failed = 0 result = 0 - ok = True @pytest.hookimpl(hookwrapper=True) def pytest_runtest_makereport(item, call): - global details, passed, failed, result, ok + global details, passed, failed, result outcome = yield outcome = outcome.get_result() if call.when == 'call': @@ -27,7 +25,6 @@ def pytest_runtest_makereport(item, call): test = dict(status='PASSED', result=call.result) elif outcome == 'failed': failed += 1 - ok = False test = dict(status='FAILED', result=call.excinfo) elif outcome == 'skipped': test = dict(status='SKIPPED', result=call.excinfo) From a61ae2007e2ad1a1f8d2ae4e010cdd3ae0b96a0a Mon Sep 17 00:00:00 2001 From: ed Date: Mon, 25 Jul 2022 12:33:00 +0200 Subject: [PATCH 4/6] result in [0;100] --- xtesting/utils/pytesthooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xtesting/utils/pytesthooks.py b/xtesting/utils/pytesthooks.py index 9afe0b3f..9fcc5902 100644 --- a/xtesting/utils/pytesthooks.py +++ b/xtesting/utils/pytesthooks.py @@ -31,5 +31,5 @@ def pytest_runtest_makereport(item, call): else: test = {} if passed + failed: - result = passed / (passed + failed) + result = passed / (passed + failed) * 100 details['tests'].append(test) From a70e973a0f961622f5759bcc824353092cd47adc Mon Sep 17 00:00:00 2001 From: ed Date: Tue, 31 Jan 2023 10:09:10 +0100 Subject: [PATCH 5/6] populate details --- xtesting/utils/pytesthooks.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/xtesting/utils/pytesthooks.py b/xtesting/utils/pytesthooks.py index 9fcc5902..b87a0320 100644 --- a/xtesting/utils/pytesthooks.py +++ b/xtesting/utils/pytesthooks.py @@ -17,19 +17,15 @@ def pytest_sessionstart(session): @pytest.hookimpl(hookwrapper=True) def pytest_runtest_makereport(item, call): global details, passed, failed, result - outcome = yield - outcome = outcome.get_result() - if call.when == 'call': - if outcome == 'passed': + yreport = yield + report = yreport.get_result() + if report.when == 'call': + test = dict(name=report.nodeid, status=report.outcome.upper()) + if report.passed: passed += 1 - test = dict(status='PASSED', result=call.result) - elif outcome == 'failed': + elif report.failed: failed += 1 - test = dict(status='FAILED', result=call.excinfo) - elif outcome == 'skipped': - test = dict(status='SKIPPED', result=call.excinfo) - else: - test = {} + test['failure'] = report.longreprtext if passed + failed: result = passed / (passed + failed) * 100 details['tests'].append(test) From 1a0719b41f5e017952f1e1dff264198e62464553 Mon Sep 17 00:00:00 2001 From: ed Date: Tue, 7 Mar 2023 12:18:03 +0100 Subject: [PATCH 6/6] convert options to str --- xtesting/core/pytest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xtesting/core/pytest.py b/xtesting/core/pytest.py index a3eebdf1..0fc582c7 100755 --- a/xtesting/core/pytest.py +++ b/xtesting/core/pytest.py @@ -40,9 +40,9 @@ def run(self, **args): dir = args.pop('dir') options = args.pop('options', []) if isinstance(options, dict): - options = [v for o in - zip([f'--{k}' if len(str(k)) > 1 else f'-{k}' for k in options], options.values()) - for v in o if v is not None] + options = [str(item) for opt in + zip([f'--{k}' if len(str(k)) > 1 else f'-{k}' for k in options.keys()], options.values()) + for item in opt if item is not None] except KeyError as err: self.logger.exception(f"Missing args: {err.args[0]!r}") return self.EX_RUN_ERROR