diff --git a/testbase/management.py b/testbase/management.py index a9de0a3..8039f4d 100644 --- a/testbase/management.py +++ b/testbase/management.py @@ -24,6 +24,7 @@ import traceback import sys import shlex +from xmlrpc.client import Boolean, boolean import six from testbase import logger @@ -186,11 +187,11 @@ class RunTest(Command): parser.add_argument("--share-data", help="share data", default="") parser.add_argument("--global-parameters", help="global parameters", default="") - + parser.add_argument("--stop-on-failure", help="when the testcase execute fail, the test task will stop", default=False) parser.add_argument("--config-file", help="runtime config file path") def run_args_parser(self, runner_args): - """兼容参数传入concurrency=5,retries=1，支持subprocess shell=False""" + """兼容参数传入concurrency=5,retries=1, 支持subprocess shell=False""" regex_c = re.compile(r'(\w+=\w+)+') regex = regex_c.search(runner_args) if regex: @@ -246,6 +247,9 @@ def execute(self, args): if args.global_parameters and isinstance(args.global_parameters, six.string_types): args.global_parameters = json.loads(args.global_parameters) + + # if args.stop_on_failure and isinstance(args.stop_on_failure, six.string_types): + # args.stop_on_failure = json.loads(args.stop_on_failure) test_conf = TestCaseSettings(names=args.tests, excluded_names=args.excluded_names, diff --git a/testbase/report.py b/testbase/report.py index 362fc1b..912da18 100644 --- a/testbase/report.py +++ b/testbase/report.py @@ -14,7 +14,6 @@ # """测试报告 """ - import sys import socket import os @@ -109,6 +108,17 @@ def log_filtered_test(self, loader, testcase, reason): """ pass + def log_ignored_test(self, testcase, reason): + """ + 记录一个忽略的测试用例 + + :param testcase: 测试用例 + :type testcase: TestCase实例 + :param reason: 忽略原因 + :type reason: str + """ + pass + def log_load_error(self, loader, name, error): """记录一个加载失败的用例或用例集 @@ -513,6 +523,19 @@ def log_filtered_test(self, loader, testcase, reason): self._write( "filtered test case: %s (reason: %s)\n" % (testcase.test_name, reason) ) + + def log_ignored_test(self, testcase, reason): + """ + 记录一个被忽略的测试用例 + :param testcase: 测试用例 + :type testcase: TestCase + :param reason: 忽略原因 + :type reason: str + """ + + self._write( + "ignored test case: %s (rease %s)\n" % (testcase.test_name, reason) + ) def log_load_error(self, loader, name, error): """记录一个加载失败的用例或用例集 @@ -693,6 +716,24 @@ def log_filtered_test(self, loader, testcase, reason): doc2 = dom.parseString(nodestr) filterNode = doc2.childNodes[0] self._runrstnode.appendChild(filterNode) + + def log_ignored_test(self, testcase, reason): + """ + 记录一个被忽略的测试用例 + :param testcase: 测试用例 + :type testcase: TestCase + :param reason: 忽略原因 + :type reason: str + """ + + nodestr = """ + """ % ( + smart_text(saxutils.escape(testcase.test_name)), + smart_text(saxutils.escape(reason)), + ) + doc2 = dom.parseString(nodestr) + ignoreNode = doc2.childNodes[0] + self._runrstnode.appendChild(ignoreNode) def log_load_error(self, loader, name, error): """记录一个加载失败的用例或用例集 @@ -764,6 +805,7 @@ def __init__(self, title="调试测试"): super(JSONTestReportBase, self).__init__() self._logs = [] self._filtered_tests = [] + self._ignored_tests = [] self._load_errors = [] self._passed_tests = {} self._failed_tests = {} @@ -869,6 +911,16 @@ def log_filtered_test(self, loader, testcase, reason): :type reason: str """ self._filtered_tests.append({"name": testcase.test_name, "reason": reason}) + + def log_ignored_test(self, testcase, reason): + """ + 记录一个被忽略的测试用例 + :param testcase: 测试用例 + :type testcase: TestCase + :param reason: 忽略原因 + :type reason: str + """ + self._ignored_tests.append({"name": testcase.test_name, "reason": reason}) def log_load_error(self, loader, name, error): """记录一个加载失败的用例或用例集 diff --git a/testbase/runner.py b/testbase/runner.py index f5bd8be..e067210 100644 --- a/testbase/runner.py +++ b/testbase/runner.py @@ -48,7 +48,7 @@ from testbase.loader import TestLoader from testbase import serialization -from testbase.testcase import TestCase, TestCaseRunner, TestSuite +from testbase.testcase import TestCase, TestCaseRunner, TestCaseType, TestSuite from testbase.report import TestReportBase from testbase.testresult import TestResultCollection from testbase.resource import TestResourceManager, LocalResourceManagerBackend @@ -63,7 +63,7 @@ class TestCaseSettings(object): ''' def __init__(self, names=None, excluded_names=None, priorities=None, status=None, owners=None, - tags=None, excluded_tags=None, share_data={}, global_parameters={}): + tags=None, excluded_tags=None, share_data={}, global_parameters={}, stop_on_failure=False): '''构造函数 :param names: 测试用例名 @@ -80,6 +80,8 @@ def __init__(self, names=None, excluded_names=None, priorities=None, status=None :type tags: list :param excluded_tags: 指定标签排除用例 :type tags: list + :param stop_on_failure: 失败停止用例执行 + :type stop_on_failure: bool ''' if names is None: self.names = [] @@ -135,6 +137,7 @@ def __init__(self, names=None, excluded_names=None, priorities=None, status=None self.tags = set(tags) if tags else None self.excluded_tags = set(excluded_tags) if excluded_tags else None self.share_data = share_data + self.stop_on_failure = stop_on_failure def _generate_testcase_names(self, testcase): if testcase.endswith(".py"): @@ -245,6 +248,7 @@ def load(self, target): tests = loader.load(target.names) self.__report.log_loaded_tests(loader, tests) filtered_tests = loader.get_filtered_tests_with_reason().items() + self._stop_on_failure = target.stop_on_failure for test, reason in filtered_tests: self.__report.log_filtered_test(loader, test, reason) for testname, error in loader.get_last_errors().items(): @@ -404,6 +408,10 @@ def run_all_tests(self, tests): test = tests_queue.popleft() passed = self.run_test(test) if not passed: + if self._stop_on_failure: + reason = "The execution of the previous test case failed, setting current testcase ignored." + for case in tests_queue: + self.__report.log_ignored_test(case, reason) tests_retry_dict.setdefault(test, 0) if tests_retry_dict[test] < self._retries: tests_retry_dict[test] += 1 @@ -494,6 +502,17 @@ def log_filtered_test(self, loader, testcase, reason): ''' with self._lock: return self._report.log_filtered_test(loader, testcase, reason) + + def log_ignored_test(self, testcase, reason): + """ + 记录一个被忽略的测试用例 + :param testcase: 测试用例 + :type testcase: TestCase + :param reason: 忽略原因 + :type reason: str + """ + with self._lock: + return self._report.log_ignored_test(testcase, reason) def log_load_error(self, loader, name, error): '''记录一个加载失败的用例或用例集 @@ -586,10 +605,16 @@ def _run_test_from_queue(self, tests_queue, tests_retry_dict): passed = self.run_test(test) with self._lock: if not passed: - tests_retry_dict.setdefault(test, 0) - if tests_retry_dict[test] < self._retries: - tests_retry_dict[test] += 1 - tests_queue.append(test) + if self._stop_on_failure: + reason = "The execution of the previous test case failed, setting current testcase ignored." + for case in tests_queue: + self.__report.log_ignored_test(case, reason) + tests_queue.clear() + else: + tests_retry_dict.setdefault(test, 0) + if tests_retry_dict[test] < self._retries: + tests_retry_dict[test] += 1 + tests_queue.append(test) @classmethod def get_parser(cls): @@ -1120,10 +1145,16 @@ def run_all_tests(self, tests): test = serialization.loads(msg[2]) passed = msg[3] if not passed: - tests_retry_dict.setdefault(test.test_name, 0) - if tests_retry_dict[test.test_name] < self._retries: - tests_retry_dict[test.test_name] += 1 - tests_queue.append(test) + if self._stop_on_failure: + for case in tests_queue: + reason = "The execution of the previous test case failed, setting current testcase ignored." + self.__report.log_ignored_test(case, reason) + tests_queue.clear() + else: + tests_retry_dict.setdefault(test.test_name, 0) + if tests_retry_dict[test.test_name] < self._retries: + tests_retry_dict[test.test_name] += 1 + tests_queue.append(test) if len(tests_queue) > 0: worker.run_testcase(tests_queue.popleft()) diff --git a/testbase/testcase.py b/testbase/testcase.py index 0a58e64..2a5c9d1 100644 --- a/testbase/testcase.py +++ b/testbase/testcase.py @@ -164,7 +164,7 @@ class EnumStatus(object): TestCaseStatus.Implement, TestCaseStatus.Review, TestCaseStatus.Ready, - TestCaseStatus.Suspend) + TestCaseStatus.Suspend,) class EnumPriority(object): '''测试用例优先级枚举类 diff --git a/tests/sampletest/stoponfailuretest.py b/tests/sampletest/stoponfailuretest.py new file mode 100644 index 0000000..1a4b921 --- /dev/null +++ b/tests/sampletest/stoponfailuretest.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +''' +参数测试用例 +''' + +import testbase +import time + + +class FirstFailureTest(testbase.TestCase): + """ + 参数测试用例 + """ + owner = "foo" + status = testbase.TestCase.EnumStatus.Ready + priority = testbase.TestCase.EnumPriority.Normal + timeout = 1 + + def run_test(self): + self.assertEqual("1", "2") + +class SecondFailureTest(testbase.TestCase): + """ + 参数测试用例 + """ + owner = "foo" + status = testbase.TestCase.EnumStatus.Ready + priority = testbase.TestCase.EnumPriority.Normal + timeout = 1 + + + def run_test(self): + self.assertEqual("hello", "world") + diff --git a/tests/test_testbase/test_runner.py b/tests/test_testbase/test_runner.py index 877081c..78372e7 100644 --- a/tests/test_testbase/test_runner.py +++ b/tests/test_testbase/test_runner.py @@ -86,6 +86,9 @@ def log_loaded_tests(self, loader, testcases): def log_filtered_test(self, loader, testcase, reason): self.logs.append(["log_filtered_test", loader, testcase, reason]) + + def log_ignored_test(self, testcase, reason): + self.logs.append(["log_ignored_test", testcase, reason]) def log_load_error(self, loader, name, error): self.logs.append(["log_load_error", loader, name, error]) @@ -314,7 +317,23 @@ def test_execute_type(self): if runner_type == runner.MultiProcessTestRunner: sys.modules["__main__"] = old_main sys.modules["__main__"].__file__ = old_main_file + + def test_stop_on_failure(self): + runner_types = [runner.TestRunner, runner.ThreadingTestRunner, runner.MultiProcessTestRunner] + for runner_type in runner_types: + if runner_type == runner.MultiProcessTestRunner: + import sys + old_main = sys.modules["__main__"] + old_main_file = sys.modules["__main__"].__file__ + sys.modules["__main__"] = sys.modules[__name__] + sys.modules["__main__"].__file__ = sys.modules[__name__].__file__ + + report = TestReport() + r = runner_type(report, execute_type="random") + r.run(runner.TestCaseSettings(["tests.sampletest.stoponfailuretest.FirstFailureTest", "tests.sampletest.stoponfailuretest.SecondFailureTest"])) + for it in report.logs: + print(it[0]) if __name__ == "__main__": unittest.main()