From 1fb8556203c3444fba279e05a991410a3be176fa Mon Sep 17 00:00:00 2001 From: blakeyzhang Date: Tue, 21 Jun 2022 18:15:05 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E5=A2=9E=E5=8A=A0stop=5Fon=5Ffailure?= =?UTF-8?q?=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- testbase/management.py | 11 ++++++++--- testbase/runner.py | 8 ++++++-- testbase/testcase.py | 7 ++++--- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/testbase/management.py b/testbase/management.py index a9de0a3..8bf11c1 100644 --- a/testbase/management.py +++ b/testbase/management.py @@ -24,6 +24,7 @@ import traceback import sys import shlex +from xmlrpc.client import Boolean, boolean import six from testbase import logger @@ -186,11 +187,11 @@ class RunTest(Command): parser.add_argument("--share-data", help="share data", default="") parser.add_argument("--global-parameters", help="global parameters", default="") - + parser.add_argument("--stop-on-failure", help="when the testcase execute fail, the test task will stop", default="true") parser.add_argument("--config-file", help="runtime config file path") def run_args_parser(self, runner_args): - """兼容参数传入concurrency=5,retries=1，支持subprocess shell=False""" + """兼容参数传入concurrency=5,retries=1, 支持subprocess shell=False""" regex_c = re.compile(r'(\w+=\w+)+') regex = regex_c.search(runner_args) if regex: @@ -246,6 +247,9 @@ def execute(self, args): if args.global_parameters and isinstance(args.global_parameters, six.string_types): args.global_parameters = json.loads(args.global_parameters) + + if args.stop_on_failure and isinstance(args.stop_on_failure, six.string_types): + args.stop_on_failure = json.loads(args.stop_on_failure) test_conf = TestCaseSettings(names=args.tests, excluded_names=args.excluded_names, @@ -254,7 +258,8 @@ def execute(self, args): owners=args.owners, tags=args.tags, excluded_tags=args.excluded_tags, - global_parameters=args.global_parameters) + global_parameters=args.global_parameters, + stop_on_failure=args.stop_on_failure) report_type = report_types[args.report_type] if args.report_type == 'xml': diff --git a/testbase/runner.py b/testbase/runner.py index f5bd8be..e9ffa2c 100644 --- a/testbase/runner.py +++ b/testbase/runner.py @@ -48,7 +48,7 @@ from testbase.loader import TestLoader from testbase import serialization -from testbase.testcase import TestCase, TestCaseRunner, TestSuite +from testbase.testcase import TestCase, TestCaseRunner, TestCaseStatus, TestSuite from testbase.report import TestReportBase from testbase.testresult import TestResultCollection from testbase.resource import TestResourceManager, LocalResourceManagerBackend @@ -63,7 +63,7 @@ class TestCaseSettings(object): ''' def __init__(self, names=None, excluded_names=None, priorities=None, status=None, owners=None, - tags=None, excluded_tags=None, share_data={}, global_parameters={}): + tags=None, excluded_tags=None, share_data={}, global_parameters={}, stop_on_failure=None): '''构造函数 :param names: 测试用例名 @@ -80,6 +80,8 @@ def __init__(self, names=None, excluded_names=None, priorities=None, status=None :type tags: list :param excluded_tags: 指定标签排除用例 :type tags: list + :param stop_on_failure: 失败停止用例执行 + :type stop_on_failure: string(true or false) ''' if names is None: self.names = [] @@ -135,6 +137,7 @@ def __init__(self, names=None, excluded_names=None, priorities=None, status=None self.tags = set(tags) if tags else None self.excluded_tags = set(excluded_tags) if excluded_tags else None self.share_data = share_data + self.stop_on_failure = stop_on_failure def _generate_testcase_names(self, testcase): if testcase.endswith(".py"): @@ -245,6 +248,7 @@ def load(self, target): tests = loader.load(target.names) self.__report.log_loaded_tests(loader, tests) filtered_tests = loader.get_filtered_tests_with_reason().items() + self._stop_on_failure = target.stop_on_failure for test, reason in filtered_tests: self.__report.log_filtered_test(loader, test, reason) for testname, error in loader.get_last_errors().items(): diff --git a/testbase/testcase.py b/testbase/testcase.py index 0a58e64..5f21a27 100644 --- a/testbase/testcase.py +++ b/testbase/testcase.py @@ -44,7 +44,7 @@ class TestCaseStatus(object): :attention: 此类将会被移除，请使用TestCase.EnumStatus ''' - Design, Implement, Review, Ready, Suspend = ('Design', 'Implement', 'Review', 'Ready', 'Suspend') + Design, Implement, Review, Ready, Suspend, Ignored = ('Design', 'Implement', 'Review', 'Ready', 'Suspend', 'Ignored') class TestCasePriority(object): @@ -160,11 +160,12 @@ class EnumStatus(object): 则可以先置为该字段为Suspend,等到可用的时候再将该字段置为Ready ''' - Design, Implement, Review, Ready, Suspend = (TestCaseStatus.Design, + Design, Implement, Review, Ready, Suspend, Ignored = (TestCaseStatus.Design, TestCaseStatus.Implement, TestCaseStatus.Review, TestCaseStatus.Ready, - TestCaseStatus.Suspend) + TestCaseStatus.Suspend, + TestCaseStatus.Ignored) class EnumPriority(object): '''测试用例优先级枚举类 From 69c09efccdd5acc92fb0dd44cf7f7887bde793c6 Mon Sep 17 00:00:00 2001 From: blakeyzhang Date: Wed, 22 Jun 2022 17:41:23 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=E4=B8=B2=E8=A1=8C=E6=89=A7=E8=A1=8C?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E7=94=A8=E4=BE=8B=E4=BF=AE=E6=94=B9=E5=AE=8C?= =?UTF-8?q?=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- testbase/runner.py | 18 ++++++++++++------ tests/test_testbase/test_runner.py | 14 ++++++++++++++ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/testbase/runner.py b/testbase/runner.py index e9ffa2c..ffbc914 100644 --- a/testbase/runner.py +++ b/testbase/runner.py @@ -63,7 +63,7 @@ class TestCaseSettings(object): ''' def __init__(self, names=None, excluded_names=None, priorities=None, status=None, owners=None, - tags=None, excluded_tags=None, share_data={}, global_parameters={}, stop_on_failure=None): + tags=None, excluded_tags=None, share_data={}, global_parameters={}, stop_on_failure=False): '''构造函数 :param names: 测试用例名 @@ -81,7 +81,7 @@ def __init__(self, names=None, excluded_names=None, priorities=None, status=None :param excluded_tags: 指定标签排除用例 :type tags: list :param stop_on_failure: 失败停止用例执行 - :type stop_on_failure: string(true or false) + :type stop_on_failure: bool ''' if names is None: self.names = [] @@ -408,10 +408,16 @@ def run_all_tests(self, tests): test = tests_queue.popleft() passed = self.run_test(test) if not passed: - tests_retry_dict.setdefault(test, 0) - if tests_retry_dict[test] < self._retries: - tests_retry_dict[test] += 1 - tests_queue.append(test) + if self._stop_on_failure: + test.status = TestCase.EnumStatus.Ignored + for case in tests_queue: + case.status = TestCase.EnumStatus.Ignored + tests_queue.clear() + else: + tests_retry_dict.setdefault(test, 0) + if tests_retry_dict[test] < self._retries: + tests_retry_dict[test] += 1 + tests_queue.append(test) @classmethod def get_parser(cls): diff --git a/tests/test_testbase/test_runner.py b/tests/test_testbase/test_runner.py index 877081c..333cfc6 100644 --- a/tests/test_testbase/test_runner.py +++ b/tests/test_testbase/test_runner.py @@ -314,7 +314,21 @@ def test_execute_type(self): if runner_type == runner.MultiProcessTestRunner: sys.modules["__main__"] = old_main sys.modules["__main__"].__file__ = old_main_file + + def test_stop_on_failure(self): + runner_types = [runner.TestRunner, runner.ThreadingTestRunner, runner.MultiProcessTestRunner] + for runner_type in runner_types: + if runner_type == runner.MultiProcessTestRunner: + import sys + old_main = sys.modules["__main__"] + old_main_file = sys.modules["__main__"].__file__ + sys.modules["__main__"] = sys.modules[__name__] + sys.modules["__main__"].__file__ = sys.modules[__name__].__file__ + + report = TestReport() + r = runner_type(report, execute_type="random") + # TODO：完善测试用例 if __name__ == "__main__": unittest.main() From 92ed1c37d177d3b020760810e14744e64678e611 Mon Sep 17 00:00:00 2001 From: blakeyzhang Date: Thu, 23 Jun 2022 20:21:45 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- testbase/report.py | 54 ++++++++++++++++++++++++++ testbase/runner.py | 55 ++++++++++++++++++--------- testbase/testcase.py | 7 ++-- tests/sampletest/stoponfailuretest.py | 34 +++++++++++++++++ tests/test_testbase/test_runner.py | 7 +++- 5 files changed, 135 insertions(+), 22 deletions(-) create mode 100644 tests/sampletest/stoponfailuretest.py diff --git a/testbase/report.py b/testbase/report.py index 362fc1b..98ba7bd 100644 --- a/testbase/report.py +++ b/testbase/report.py @@ -15,6 +15,7 @@ """测试报告 """ +from pydoc import doc import sys import socket import os @@ -109,6 +110,17 @@ def log_filtered_test(self, loader, testcase, reason): """ pass + def log_ignored_test(self, testcase, reason): + """ + 记录一个忽略的测试用例 + + :param testcase: 测试用例 + :type testcase: TestCase实例 + :param reason: 忽略原因 + :type reason: str + """ + pass + def log_load_error(self, loader, name, error): """记录一个加载失败的用例或用例集 @@ -513,6 +525,19 @@ def log_filtered_test(self, loader, testcase, reason): self._write( "filtered test case: %s (reason: %s)\n" % (testcase.test_name, reason) ) + + def log_ignored_test(self, testcase, reason): + """ + 记录一个被忽略的测试用例 + :param testcase: 测试用例 + :type testcase: TestCase + :param reason: 忽略原因 + :type reason: str + """ + + self._write( + "ignored test case: %s (rease %s)\n" % (testcase.test_name, reason) + ) def log_load_error(self, loader, name, error): """记录一个加载失败的用例或用例集 @@ -693,6 +718,24 @@ def log_filtered_test(self, loader, testcase, reason): doc2 = dom.parseString(nodestr) filterNode = doc2.childNodes[0] self._runrstnode.appendChild(filterNode) + + def log_ignored_test(self, testcase, reason): + """ + 记录一个被忽略的测试用例 + :param testcase: 测试用例 + :type testcase: TestCase + :param reason: 忽略原因 + :type reason: str + """ + + nodestr = """ + """ % ( + smart_text(saxutils.escape(testcase.test_name)), + smart_text(saxutils.escape(reason)), + ) + doc2 = dom.parseString(nodestr) + ignoreNode = doc2.childNodes[0] + self._runrstnode.appendChild(ignoreNode) def log_load_error(self, loader, name, error): """记录一个加载失败的用例或用例集 @@ -764,6 +807,7 @@ def __init__(self, title="调试测试"): super(JSONTestReportBase, self).__init__() self._logs = [] self._filtered_tests = [] + self._ignored_tests = [] self._load_errors = [] self._passed_tests = {} self._failed_tests = {} @@ -869,6 +913,16 @@ def log_filtered_test(self, loader, testcase, reason): :type reason: str """ self._filtered_tests.append({"name": testcase.test_name, "reason": reason}) + + def log_ignored_test(self, testcase, reason): + """ + 记录一个被忽略的测试用例 + :param testcase: 测试用例 + :type testcase: TestCase + :param reason: 忽略原因 + :type reason: str + """ + self._ignored_tests.append({"name": testcase.test_name, "reason": reason}) def log_load_error(self, loader, name, error): """记录一个加载失败的用例或用例集 diff --git a/testbase/runner.py b/testbase/runner.py index ffbc914..e067210 100644 --- a/testbase/runner.py +++ b/testbase/runner.py @@ -48,7 +48,7 @@ from testbase.loader import TestLoader from testbase import serialization -from testbase.testcase import TestCase, TestCaseRunner, TestCaseStatus, TestSuite +from testbase.testcase import TestCase, TestCaseRunner, TestCaseType, TestSuite from testbase.report import TestReportBase from testbase.testresult import TestResultCollection from testbase.resource import TestResourceManager, LocalResourceManagerBackend @@ -409,15 +409,13 @@ def run_all_tests(self, tests): passed = self.run_test(test) if not passed: if self._stop_on_failure: - test.status = TestCase.EnumStatus.Ignored + reason = "The execution of the previous test case failed, setting current testcase ignored." for case in tests_queue: - case.status = TestCase.EnumStatus.Ignored - tests_queue.clear() - else: - tests_retry_dict.setdefault(test, 0) - if tests_retry_dict[test] < self._retries: - tests_retry_dict[test] += 1 - tests_queue.append(test) + self.__report.log_ignored_test(case, reason) + tests_retry_dict.setdefault(test, 0) + if tests_retry_dict[test] < self._retries: + tests_retry_dict[test] += 1 + tests_queue.append(test) @classmethod def get_parser(cls): @@ -504,6 +502,17 @@ def log_filtered_test(self, loader, testcase, reason): ''' with self._lock: return self._report.log_filtered_test(loader, testcase, reason) + + def log_ignored_test(self, testcase, reason): + """ + 记录一个被忽略的测试用例 + :param testcase: 测试用例 + :type testcase: TestCase + :param reason: 忽略原因 + :type reason: str + """ + with self._lock: + return self._report.log_ignored_test(testcase, reason) def log_load_error(self, loader, name, error): '''记录一个加载失败的用例或用例集 @@ -596,10 +605,16 @@ def _run_test_from_queue(self, tests_queue, tests_retry_dict): passed = self.run_test(test) with self._lock: if not passed: - tests_retry_dict.setdefault(test, 0) - if tests_retry_dict[test] < self._retries: - tests_retry_dict[test] += 1 - tests_queue.append(test) + if self._stop_on_failure: + reason = "The execution of the previous test case failed, setting current testcase ignored." + for case in tests_queue: + self.__report.log_ignored_test(case, reason) + tests_queue.clear() + else: + tests_retry_dict.setdefault(test, 0) + if tests_retry_dict[test] < self._retries: + tests_retry_dict[test] += 1 + tests_queue.append(test) @classmethod def get_parser(cls): @@ -1130,10 +1145,16 @@ def run_all_tests(self, tests): test = serialization.loads(msg[2]) passed = msg[3] if not passed: - tests_retry_dict.setdefault(test.test_name, 0) - if tests_retry_dict[test.test_name] < self._retries: - tests_retry_dict[test.test_name] += 1 - tests_queue.append(test) + if self._stop_on_failure: + for case in tests_queue: + reason = "The execution of the previous test case failed, setting current testcase ignored." + self.__report.log_ignored_test(case, reason) + tests_queue.clear() + else: + tests_retry_dict.setdefault(test.test_name, 0) + if tests_retry_dict[test.test_name] < self._retries: + tests_retry_dict[test.test_name] += 1 + tests_queue.append(test) if len(tests_queue) > 0: worker.run_testcase(tests_queue.popleft()) diff --git a/testbase/testcase.py b/testbase/testcase.py index 5f21a27..2a5c9d1 100644 --- a/testbase/testcase.py +++ b/testbase/testcase.py @@ -44,7 +44,7 @@ class TestCaseStatus(object): :attention: 此类将会被移除，请使用TestCase.EnumStatus ''' - Design, Implement, Review, Ready, Suspend, Ignored = ('Design', 'Implement', 'Review', 'Ready', 'Suspend', 'Ignored') + Design, Implement, Review, Ready, Suspend = ('Design', 'Implement', 'Review', 'Ready', 'Suspend') class TestCasePriority(object): @@ -160,12 +160,11 @@ class EnumStatus(object): 则可以先置为该字段为Suspend,等到可用的时候再将该字段置为Ready ''' - Design, Implement, Review, Ready, Suspend, Ignored = (TestCaseStatus.Design, + Design, Implement, Review, Ready, Suspend = (TestCaseStatus.Design, TestCaseStatus.Implement, TestCaseStatus.Review, TestCaseStatus.Ready, - TestCaseStatus.Suspend, - TestCaseStatus.Ignored) + TestCaseStatus.Suspend,) class EnumPriority(object): '''测试用例优先级枚举类 diff --git a/tests/sampletest/stoponfailuretest.py b/tests/sampletest/stoponfailuretest.py new file mode 100644 index 0000000..1a4b921 --- /dev/null +++ b/tests/sampletest/stoponfailuretest.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +''' +参数测试用例 +''' + +import testbase +import time + + +class FirstFailureTest(testbase.TestCase): + """ + 参数测试用例 + """ + owner = "foo" + status = testbase.TestCase.EnumStatus.Ready + priority = testbase.TestCase.EnumPriority.Normal + timeout = 1 + + def run_test(self): + self.assertEqual("1", "2") + +class SecondFailureTest(testbase.TestCase): + """ + 参数测试用例 + """ + owner = "foo" + status = testbase.TestCase.EnumStatus.Ready + priority = testbase.TestCase.EnumPriority.Normal + timeout = 1 + + + def run_test(self): + self.assertEqual("hello", "world") + diff --git a/tests/test_testbase/test_runner.py b/tests/test_testbase/test_runner.py index 333cfc6..78372e7 100644 --- a/tests/test_testbase/test_runner.py +++ b/tests/test_testbase/test_runner.py @@ -86,6 +86,9 @@ def log_loaded_tests(self, loader, testcases): def log_filtered_test(self, loader, testcase, reason): self.logs.append(["log_filtered_test", loader, testcase, reason]) + + def log_ignored_test(self, testcase, reason): + self.logs.append(["log_ignored_test", testcase, reason]) def log_load_error(self, loader, name, error): self.logs.append(["log_load_error", loader, name, error]) @@ -328,7 +331,9 @@ def test_stop_on_failure(self): report = TestReport() r = runner_type(report, execute_type="random") - # TODO：完善测试用例 + r.run(runner.TestCaseSettings(["tests.sampletest.stoponfailuretest.FirstFailureTest", "tests.sampletest.stoponfailuretest.SecondFailureTest"])) + for it in report.logs: + print(it[0]) if __name__ == "__main__": unittest.main() From 9e5b0b099e79684056c3320d2f20b9f17f821fdb Mon Sep 17 00:00:00 2001 From: blakeyzhang Date: Fri, 24 Jun 2022 10:24:08 +0800 Subject: [PATCH 4/4] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E5=8F=82=E6=95=B0=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- testbase/management.py | 9 ++++----- testbase/report.py | 2 -- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/testbase/management.py b/testbase/management.py index 8bf11c1..8039f4d 100644 --- a/testbase/management.py +++ b/testbase/management.py @@ -187,7 +187,7 @@ class RunTest(Command): parser.add_argument("--share-data", help="share data", default="") parser.add_argument("--global-parameters", help="global parameters", default="") - parser.add_argument("--stop-on-failure", help="when the testcase execute fail, the test task will stop", default="true") + parser.add_argument("--stop-on-failure", help="when the testcase execute fail, the test task will stop", default=False) parser.add_argument("--config-file", help="runtime config file path") def run_args_parser(self, runner_args): @@ -248,8 +248,8 @@ def execute(self, args): if args.global_parameters and isinstance(args.global_parameters, six.string_types): args.global_parameters = json.loads(args.global_parameters) - if args.stop_on_failure and isinstance(args.stop_on_failure, six.string_types): - args.stop_on_failure = json.loads(args.stop_on_failure) + # if args.stop_on_failure and isinstance(args.stop_on_failure, six.string_types): + # args.stop_on_failure = json.loads(args.stop_on_failure) test_conf = TestCaseSettings(names=args.tests, excluded_names=args.excluded_names, @@ -258,8 +258,7 @@ def execute(self, args): owners=args.owners, tags=args.tags, excluded_tags=args.excluded_tags, - global_parameters=args.global_parameters, - stop_on_failure=args.stop_on_failure) + global_parameters=args.global_parameters) report_type = report_types[args.report_type] if args.report_type == 'xml': diff --git a/testbase/report.py b/testbase/report.py index 98ba7bd..912da18 100644 --- a/testbase/report.py +++ b/testbase/report.py @@ -14,8 +14,6 @@ # """测试报告 """ - -from pydoc import doc import sys import socket import os