From 9287e01c32aae4415274723eb6d59cf1c7f8f66f Mon Sep 17 00:00:00 2001 From: Ewoud Kohl van Wijngaarden Date: Fri, 15 May 2026 12:50:32 +0200 Subject: [PATCH 1/2] Introduce feature directories in tests This sets up a directory structure where an entire directory gets a feature mark. This makes it easy to skip entire feature tests. --- docs/developer/testing.md | 6 ++++-- tests/conftest.py | 13 +++++++++++++ tests/feature/__init__.py | 0 tests/{ => feature}/iop/__init__.py | 0 tests/{ => feature}/iop/images_test.py | 2 -- tests/{ => feature}/iop/test_advisor.py | 5 ----- tests/{ => feature}/iop/test_advisor_frontend.py | 5 ----- tests/{ => feature}/iop/test_cvemap_downloader.py | 5 ----- tests/{ => feature}/iop/test_engine.py | 5 ----- tests/{ => feature}/iop/test_gateway.py | 5 ----- tests/{ => feature}/iop/test_ingress.py | 5 ----- tests/{ => feature}/iop/test_integration.py | 5 ----- tests/{ => feature}/iop/test_inventory.py | 5 ----- tests/{ => feature}/iop/test_inventory_frontend.py | 5 ----- tests/{ => feature}/iop/test_kafka.py | 5 ----- tests/{ => feature}/iop/test_puptoo.py | 5 ----- tests/{ => feature}/iop/test_remediation.py | 5 ----- tests/{ => feature}/iop/test_vmaas.py | 5 ----- tests/{ => feature}/iop/test_vulnerability.py | 5 ----- .../iop/test_vulnerability_frontend.py | 5 ----- tests/{ => feature}/iop/test_yuptoo.py | 5 ----- 21 files changed, 17 insertions(+), 84 deletions(-) create mode 100644 tests/feature/__init__.py rename tests/{ => feature}/iop/__init__.py (100%) rename tests/{ => feature}/iop/images_test.py (95%) rename tests/{ => feature}/iop/test_advisor.py (99%) rename tests/{ => feature}/iop/test_advisor_frontend.py (94%) rename tests/{ => feature}/iop/test_cvemap_downloader.py (96%) rename tests/{ => feature}/iop/test_engine.py (95%) rename tests/{ => feature}/iop/test_gateway.py (93%) rename tests/{ => feature}/iop/test_ingress.py (87%) rename tests/{ => feature}/iop/test_integration.py (98%) rename tests/{ => feature}/iop/test_inventory.py (97%) rename tests/{ => feature}/iop/test_inventory_frontend.py (94%) rename tests/{ => feature}/iop/test_kafka.py (97%) rename tests/{ => feature}/iop/test_puptoo.py (71%) rename tests/{ => feature}/iop/test_remediation.py (94%) rename tests/{ => feature}/iop/test_vmaas.py (96%) rename tests/{ => feature}/iop/test_vulnerability.py (99%) rename tests/{ => feature}/iop/test_vulnerability_frontend.py (94%) rename tests/{ => feature}/iop/test_yuptoo.py (71%) diff --git a/docs/developer/testing.md b/docs/developer/testing.md index 22c9122b8..5f1b78a47 100644 --- a/docs/developer/testing.md +++ b/docs/developer/testing.md @@ -225,10 +225,12 @@ def test_dynflow_service_instances(server, instance): ## Where to add new tests +The directory `tests/feature` is special. Any subdirectory automatically applies the feature marker. `tests/feature/foreman/api_test.py` is equivalent to `tests/foreman_api_test.py` with `pytestmark = pytest.mark.feature("foreman")`. + | What you're testing | File | | --- | --- | -| A new service (container or systemd unit) | `tests/_test.py` | -| Foreman API behavior | `tests/foreman_api_test.py` (or a new file for larger areas) | +| A new service (container or systemd unit) | `tests/feature//_test.py` | +| Foreman API behavior | `tests/feature/foreman/api_test.py` (or a new file for larger areas) | | Client registration or content workflows | `tests/client_test.py` | | CLI flags, playbooks, or feature management | `tests/features_test.py` or `tests/playbooks_test.py` | | A standalone script (no deployment needed) | `tests/unit/_test.py` | diff --git a/tests/conftest.py b/tests/conftest.py index 9bf66ec3b..31ac39073 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -242,6 +242,19 @@ def pytest_configure(config): config.user_parameters = UserParameters(config) +def pytest_collection_modifyitems(config, items): + feature_dir = config.rootdir / 'tests' / 'feature' + for item in items: + try: + rel_path = item.path.relative_to(feature_dir) + except ValueError: + # Not in the features directory + pass + else: + feature = rel_path.parts[0] + item.add_marker(pytest.mark.feature(feature)) + + def pytest_runtest_setup(item): feature_markers = set(mark.args[0] for mark in item.iter_markers(name="feature")) if feature_markers: diff --git a/tests/feature/__init__.py b/tests/feature/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/iop/__init__.py b/tests/feature/iop/__init__.py similarity index 100% rename from tests/iop/__init__.py rename to tests/feature/iop/__init__.py diff --git a/tests/iop/images_test.py b/tests/feature/iop/images_test.py similarity index 95% rename from tests/iop/images_test.py rename to tests/feature/iop/images_test.py index 0fb6d2680..eb84367d6 100644 --- a/tests/iop/images_test.py +++ b/tests/feature/iop/images_test.py @@ -1,7 +1,5 @@ import pytest -pytestmark = pytest.mark.feature("iop") - IOP_IMAGES = [ "iop-kafka", "iop-ingress", diff --git a/tests/iop/test_advisor.py b/tests/feature/iop/test_advisor.py similarity index 99% rename from tests/iop/test_advisor.py rename to tests/feature/iop/test_advisor.py index 06bfa776f..1d0e440bf 100644 --- a/tests/iop/test_advisor.py +++ b/tests/feature/iop/test_advisor.py @@ -1,8 +1,3 @@ -import pytest - -pytestmark = pytest.mark.feature("iop") - - def test_advisor_backend_api_service(server): service = server.service("iop-service-advisor-backend-api") assert service.is_running diff --git a/tests/iop/test_advisor_frontend.py b/tests/feature/iop/test_advisor_frontend.py similarity index 94% rename from tests/iop/test_advisor_frontend.py rename to tests/feature/iop/test_advisor_frontend.py index 904b6b82b..c260bbaa4 100644 --- a/tests/iop/test_advisor_frontend.py +++ b/tests/feature/iop/test_advisor_frontend.py @@ -1,8 +1,3 @@ -import pytest - -pytestmark = pytest.mark.feature("iop") - - def test_advisor_frontend_assets_directory(server): assets_dir = server.file("/var/www/iop/assets/apps/advisor") assert assets_dir.exists diff --git a/tests/iop/test_cvemap_downloader.py b/tests/feature/iop/test_cvemap_downloader.py similarity index 96% rename from tests/iop/test_cvemap_downloader.py rename to tests/feature/iop/test_cvemap_downloader.py index 5f4fa12f5..d16cd272d 100644 --- a/tests/iop/test_cvemap_downloader.py +++ b/tests/feature/iop/test_cvemap_downloader.py @@ -1,8 +1,3 @@ -import pytest - -pytestmark = pytest.mark.feature("iop") - - def test_cvemap_download_script(server): script = server.file("/usr/local/bin/iop-cvemap-download.sh") assert script.exists diff --git a/tests/iop/test_engine.py b/tests/feature/iop/test_engine.py similarity index 95% rename from tests/iop/test_engine.py rename to tests/feature/iop/test_engine.py index c768ab4d2..99f8fd7eb 100644 --- a/tests/iop/test_engine.py +++ b/tests/feature/iop/test_engine.py @@ -1,8 +1,3 @@ -import pytest - -pytestmark = pytest.mark.feature("iop") - - def test_engine_service(server): service = server.service("iop-core-engine") assert service.is_running diff --git a/tests/iop/test_gateway.py b/tests/feature/iop/test_gateway.py similarity index 93% rename from tests/iop/test_gateway.py rename to tests/feature/iop/test_gateway.py index c1ca06d82..4fc37d064 100644 --- a/tests/iop/test_gateway.py +++ b/tests/feature/iop/test_gateway.py @@ -1,8 +1,3 @@ -import pytest - -pytestmark = pytest.mark.feature("iop") - - def test_gateway_service(server): service = server.service("iop-core-gateway") assert service.is_running diff --git a/tests/iop/test_ingress.py b/tests/feature/iop/test_ingress.py similarity index 87% rename from tests/iop/test_ingress.py rename to tests/feature/iop/test_ingress.py index 84765c688..95c791d76 100644 --- a/tests/iop/test_ingress.py +++ b/tests/feature/iop/test_ingress.py @@ -1,8 +1,3 @@ -import pytest - -pytestmark = pytest.mark.feature("iop") - - def test_ingress_service(server): service = server.service("iop-core-ingress") assert service.is_running diff --git a/tests/iop/test_integration.py b/tests/feature/iop/test_integration.py similarity index 98% rename from tests/iop/test_integration.py rename to tests/feature/iop/test_integration.py index 0dd10ac6d..6b096d8cf 100644 --- a/tests/iop/test_integration.py +++ b/tests/feature/iop/test_integration.py @@ -1,8 +1,3 @@ -import pytest - -pytestmark = pytest.mark.feature("iop") - - def test_iop_core_kafka_service(server): service = server.service("iop-core-kafka") assert service.is_running diff --git a/tests/iop/test_inventory.py b/tests/feature/iop/test_inventory.py similarity index 97% rename from tests/iop/test_inventory.py rename to tests/feature/iop/test_inventory.py index bf2d1260a..21a2bc28e 100644 --- a/tests/iop/test_inventory.py +++ b/tests/feature/iop/test_inventory.py @@ -1,8 +1,3 @@ -import pytest - -pytestmark = pytest.mark.feature("iop") - - def test_inventory_migrate_service(server): service = server.service("iop-core-host-inventory-migrate") assert service.is_enabled diff --git a/tests/iop/test_inventory_frontend.py b/tests/feature/iop/test_inventory_frontend.py similarity index 94% rename from tests/iop/test_inventory_frontend.py rename to tests/feature/iop/test_inventory_frontend.py index 9c16fa34d..5e4a96979 100644 --- a/tests/iop/test_inventory_frontend.py +++ b/tests/feature/iop/test_inventory_frontend.py @@ -1,8 +1,3 @@ -import pytest - -pytestmark = pytest.mark.feature("iop") - - def test_inventory_frontend_assets_directory(server): assets_dir = server.file("/var/www/iop/assets/apps/inventory") assert assets_dir.exists diff --git a/tests/iop/test_kafka.py b/tests/feature/iop/test_kafka.py similarity index 97% rename from tests/iop/test_kafka.py rename to tests/feature/iop/test_kafka.py index ff94d930b..b52d76c38 100644 --- a/tests/iop/test_kafka.py +++ b/tests/feature/iop/test_kafka.py @@ -1,8 +1,3 @@ -import pytest - -pytestmark = pytest.mark.feature("iop") - - def test_kafka_service(server): service = server.service("iop-core-kafka") assert service.is_running diff --git a/tests/iop/test_puptoo.py b/tests/feature/iop/test_puptoo.py similarity index 71% rename from tests/iop/test_puptoo.py rename to tests/feature/iop/test_puptoo.py index 4dbc9958e..27ce57717 100644 --- a/tests/iop/test_puptoo.py +++ b/tests/feature/iop/test_puptoo.py @@ -1,8 +1,3 @@ -import pytest - -pytestmark = pytest.mark.feature("iop") - - def test_puptoo_service(server): service = server.service("iop-core-puptoo") assert service.is_running diff --git a/tests/iop/test_remediation.py b/tests/feature/iop/test_remediation.py similarity index 94% rename from tests/iop/test_remediation.py rename to tests/feature/iop/test_remediation.py index f29a2afdc..202ffb4d0 100644 --- a/tests/iop/test_remediation.py +++ b/tests/feature/iop/test_remediation.py @@ -1,8 +1,3 @@ -import pytest - -pytestmark = pytest.mark.feature("iop") - - def test_remediation_api_service(server): service = server.service("iop-service-remediations-api") assert service.is_running diff --git a/tests/iop/test_vmaas.py b/tests/feature/iop/test_vmaas.py similarity index 96% rename from tests/iop/test_vmaas.py rename to tests/feature/iop/test_vmaas.py index e8751b265..225bf9306 100644 --- a/tests/iop/test_vmaas.py +++ b/tests/feature/iop/test_vmaas.py @@ -1,8 +1,3 @@ -import pytest - -pytestmark = pytest.mark.feature("iop") - - def test_vmaas_reposcan_service(server): service = server.service("iop-service-vmaas-reposcan") assert service.is_running diff --git a/tests/iop/test_vulnerability.py b/tests/feature/iop/test_vulnerability.py similarity index 99% rename from tests/iop/test_vulnerability.py rename to tests/feature/iop/test_vulnerability.py index dffaa7e40..a409acc17 100644 --- a/tests/iop/test_vulnerability.py +++ b/tests/feature/iop/test_vulnerability.py @@ -1,8 +1,3 @@ -import pytest - -pytestmark = pytest.mark.feature("iop") - - def test_vulnerability_manager_service(server): service = server.service("iop-service-vuln-manager") assert service.is_running diff --git a/tests/iop/test_vulnerability_frontend.py b/tests/feature/iop/test_vulnerability_frontend.py similarity index 94% rename from tests/iop/test_vulnerability_frontend.py rename to tests/feature/iop/test_vulnerability_frontend.py index b5c7e910e..8f6172bb7 100644 --- a/tests/iop/test_vulnerability_frontend.py +++ b/tests/feature/iop/test_vulnerability_frontend.py @@ -1,8 +1,3 @@ -import pytest - -pytestmark = pytest.mark.feature("iop") - - def test_vulnerability_frontend_assets_directory(server): assets_dir = server.file("/var/www/iop/assets/apps/vulnerability") assert assets_dir.exists diff --git a/tests/iop/test_yuptoo.py b/tests/feature/iop/test_yuptoo.py similarity index 71% rename from tests/iop/test_yuptoo.py rename to tests/feature/iop/test_yuptoo.py index 8af0ed85a..3acc0ace3 100644 --- a/tests/iop/test_yuptoo.py +++ b/tests/feature/iop/test_yuptoo.py @@ -1,8 +1,3 @@ -import pytest - -pytestmark = pytest.mark.feature("iop") - - def test_yuptoo_service(server): service = server.service("iop-core-yuptoo") assert service.is_running From 1b235ab35e4be46c0b47af4bbfcd6dfd933198d0 Mon Sep 17 00:00:00 2001 From: Ewoud Kohl van Wijngaarden Date: Fri, 15 May 2026 13:11:20 +0200 Subject: [PATCH 2/2] Move foreman, foreman-proxy and hammer feature tests into tests/feature This guards it behind the feature flag, allowing the tests to be skipped if the feature is disabled. For example, on a standalone foreman-proxy. --- tests/feature/foreman-proxy/__init__.py | 0 .../{foreman_proxy_test.py => feature/foreman-proxy/base_test.py} | 0 tests/feature/foreman/__init__.py | 0 tests/{foreman_api_test.py => feature/foreman/api_test.py} | 0 tests/{foreman_test.py => feature/foreman/base_test.py} | 0 .../foreman/compute_resources_test.py} | 0 .../{foreman_plugins_test.py => feature/foreman/plugins_test.py} | 0 tests/feature/hammer/__init__.py | 0 tests/{hammer_test.py => feature/hammer/base_test.py} | 0 9 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/feature/foreman-proxy/__init__.py rename tests/{foreman_proxy_test.py => feature/foreman-proxy/base_test.py} (100%) create mode 100644 tests/feature/foreman/__init__.py rename tests/{foreman_api_test.py => feature/foreman/api_test.py} (100%) rename tests/{foreman_test.py => feature/foreman/base_test.py} (100%) rename tests/{foreman_compute_resources_test.py => feature/foreman/compute_resources_test.py} (100%) rename tests/{foreman_plugins_test.py => feature/foreman/plugins_test.py} (100%) create mode 100644 tests/feature/hammer/__init__.py rename tests/{hammer_test.py => feature/hammer/base_test.py} (100%) diff --git a/tests/feature/foreman-proxy/__init__.py b/tests/feature/foreman-proxy/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/foreman_proxy_test.py b/tests/feature/foreman-proxy/base_test.py similarity index 100% rename from tests/foreman_proxy_test.py rename to tests/feature/foreman-proxy/base_test.py diff --git a/tests/feature/foreman/__init__.py b/tests/feature/foreman/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/foreman_api_test.py b/tests/feature/foreman/api_test.py similarity index 100% rename from tests/foreman_api_test.py rename to tests/feature/foreman/api_test.py diff --git a/tests/foreman_test.py b/tests/feature/foreman/base_test.py similarity index 100% rename from tests/foreman_test.py rename to tests/feature/foreman/base_test.py diff --git a/tests/foreman_compute_resources_test.py b/tests/feature/foreman/compute_resources_test.py similarity index 100% rename from tests/foreman_compute_resources_test.py rename to tests/feature/foreman/compute_resources_test.py diff --git a/tests/foreman_plugins_test.py b/tests/feature/foreman/plugins_test.py similarity index 100% rename from tests/foreman_plugins_test.py rename to tests/feature/foreman/plugins_test.py diff --git a/tests/feature/hammer/__init__.py b/tests/feature/hammer/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/hammer_test.py b/tests/feature/hammer/base_test.py similarity index 100% rename from tests/hammer_test.py rename to tests/feature/hammer/base_test.py