Skip tests for features not enabled and show pytest results

development/playbooks/test/test.yaml

@@ -47,3 +47,14 @@
       args:
         chdir: "{{ inventory_dir }}/../"
       changed_when: false
+      register: pytest_result
+      ignore_errors: true
+
+    - name: Display pytest results
+      ansible.builtin.debug:
+        msg: "{{ pytest_result.stdout }}"
+
+    - name: Fail if tests did not pass
+      ansible.builtin.fail:
+        msg: "Tests failed. See output above for details."
+      when: pytest_result.rc != 0

tests/client_test.py

@@ -1,3 +1,6 @@
+import pytest
+
+
 def test_foreman_content_view(client_environment, activation_key, organization, foremanapi, client):
     client.run('dnf install -y subscription-manager')
     rcmd = foremanapi.create('registration_commands', {'organization_id': organization['id'], 'insecure': True, 'activation_keys': [activation_key['name']], 'force': True})
@@ -9,6 +12,8 @@ def test_foreman_content_view(client_environment, activation_key, organization,
     client.run('subscription-manager unregister')
     client.run('subscription-manager clean')
 
+@pytest.mark.feature("remote-execution")
+@pytest.mark.feature("foreman-proxy")
 def test_foreman_rex(client_environment, activation_key, organization, foremanapi, client, client_fqdn):
     client.run('dnf install -y subscription-manager')
     rcmd = foremanapi.create('registration_commands', {'organization_id': organization['id'], 'insecure': True, 'activation_keys': [activation_key['name']], 'force': True})

tests/conftest.py

@@ -1,3 +1,5 @@
+import os
+import pathlib
 import uuid
 
 import apypie
@@ -6,7 +8,6 @@
 import pytest
 import testinfra
 import yaml
-import os
 
 from jinja2 import Environment, FileSystemLoader, select_autoescape
 
@@ -83,11 +84,20 @@ def ssh_config(server_hostname):
 
 
 @pytest.fixture(scope="module")
-def foremanapi(ssh_config, server_fqdn):
+def foreman_admin_password():
+    test_dir = os.path.dirname(os.path.abspath(__file__))
+    foremanctl_dir = os.path.dirname(test_dir)
+    passwd_file = os.path.join(foremanctl_dir, '.var', 'lib', 'foremanctl', 'foreman-admin-init-passwd')
+    with open(passwd_file) as f:
+        return f.read().strip()
+
+
+@pytest.fixture(scope="module")
+def foremanapi(ssh_config, server_fqdn, foreman_admin_password):
     api = apypie.ForemanApi(
         uri=f'https://{ssh_config["hostname"]}',
         username='admin',
-        password='changeme',
+        password=foreman_admin_password,
         verify_ssl=False,
     )
     api._session.headers['Host'] = server_fqdn
@@ -174,31 +184,68 @@ def wait_for_metadata_generate(foremanapi):
     wait_for_tasks(foremanapi, 'label = Actions::Katello::Repository::MetadataGenerate')
 
 
-def is_iop_enabled():
-    test_dir = os.path.dirname(os.path.abspath(__file__))
-    foremanctl_dir = os.path.dirname(test_dir)
-    params_file = os.path.join(foremanctl_dir, '.var', 'lib', 'foremanctl', 'parameters.yaml')
+class UserParameters:
+    _enabled_features = None
 
-    if os.path.exists(params_file):
-        with open(params_file, 'r') as f:
-            params = yaml.safe_load(f)
-            features = params.get('features', [])
-            if isinstance(features, str):
-                features = features.split()
-            return 'iop' in features
+    @classmethod
+    def enabled_features(cls):
+        if cls._enabled_features is None:
+            cls._enabled_features = cls._load()
+        return cls._enabled_features
 
-    return False
+    @classmethod
+    def has_feature(cls, name):
+        return name in cls.enabled_features()
+
+    @classmethod
+    def _load(cls):
+        test_dir = os.path.dirname(os.path.abspath(__file__))
+        foremanctl_dir = os.path.dirname(test_dir)
+        params_file = os.path.join(foremanctl_dir, '.var', 'lib', 'foremanctl', 'parameters.yaml')
+        defaults_file = os.path.join(foremanctl_dir, 'src', 'vars', 'defaults.yml')
+
+        params = {}
+        if os.path.exists(params_file):
+            with open(params_file, 'r') as f:
+                params = yaml.safe_load(f) or {}
+
+        defaults = {}
+        if os.path.exists(defaults_file):
+            with open(defaults_file, 'r') as f:
+                defaults = yaml.safe_load(f) or {}
+
+        flavor = params.get('flavor', defaults.get('flavor', 'katello'))
+        flavor_file = os.path.join(foremanctl_dir, 'src', 'vars', 'flavors', f'{flavor}.yml')
+
+        flavor_features = []
+        if os.path.exists(flavor_file):
+            with open(flavor_file, 'r') as f:
+                flavor_data = yaml.safe_load(f) or {}
+                flavor_features = flavor_data.get('flavor_features', [])
+
+        features = params.get('features', [])
+        if isinstance(features, str):
+            features = features.split()
+
+        return set(flavor_features + features)
 
 
 def pytest_configure(config):
-    config.addinivalue_line("markers", "iop: tests requiring IOP to be enabled")
+    config.addinivalue_line("markers", "feature(name): skip test if named feature is not enabled")
 
 
 def pytest_collection_modifyitems(config, items):
-    if is_iop_enabled():
-        return
-
-    skip_iop = pytest.mark.skip(reason="IOP not enabled - skipping IOP tests ('iop' not in enabled_features)")
+    feature_dir = pathlib.Path(__file__).parent / 'feature'
     for item in items:
-        if "iop" in item.keywords:
-            item.add_marker(skip_iop)
+        try:
+            rel = pathlib.Path(item.fspath).relative_to(feature_dir)
+            item.add_marker(pytest.mark.feature(rel.parts[0]))
+        except ValueError:
+            pass
+
+
+def pytest_runtest_setup(item):
+    for marker in item.iter_markers("feature"):
+        feature = marker.args[0]
+        if not UserParameters.has_feature(feature):
+            pytest.skip(f"{feature} not enabled")

tests/feature/hammer/test_foreman_compute_resources.py

@@ -0,0 +1,17 @@
+import pytest
+
+FOREMAN_HOST = 'localhost'
+FOREMAN_PORT = 3000
+
+@pytest.mark.parametrize("compute_resource", [
+    pytest.param('AzureRm', marks=pytest.mark.feature('azure-rm')),
+    pytest.param('EC2'),
+    pytest.param('GCE', marks=pytest.mark.feature('google')),
+    pytest.param('Libvirt'),
+    pytest.param('Openstack'),
+    pytest.param('Vmware'),
+])
+def test_foreman_compute_resources(server, compute_resource):
+    hammer = server.run("hammer compute-resource create --help | grep provider")
+    assert hammer.succeeded
+    assert compute_resource in hammer.stdout

tests/iop/test_advisor.py → tests/feature/iop/test_advisor.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_advisor_backend_api_service(server):
     service = server.service("iop-service-advisor-backend-api")
     assert service.is_running

tests/iop/test_advisor_frontend.py → tests/feature/iop/test_advisor_frontend.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_advisor_frontend_assets_directory(server):
     assets_dir = server.file("/var/www/iop/assets/apps/advisor")
     assert assets_dir.exists

tests/iop/test_cvemap_downloader.py → tests/feature/iop/test_cvemap_downloader.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_cvemap_download_script(server):
     script = server.file("/usr/local/bin/iop-cvemap-download.sh")
     assert script.exists

tests/iop/test_engine.py → tests/feature/iop/test_engine.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_engine_service(server):
     service = server.service("iop-core-engine")
     assert service.is_running

tests/iop/test_gateway.py → tests/feature/iop/test_gateway.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_gateway_service(server):
     service = server.service("iop-core-gateway")
     assert service.is_running

tests/iop/test_ingress.py → tests/feature/iop/test_ingress.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_ingress_service(server):
     service = server.service("iop-core-ingress")
     assert service.is_running

tests/iop/test_integration.py → tests/feature/iop/test_integration.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_iop_core_kafka_service(server):
     service = server.service("iop-core-kafka")
     assert service.is_running

tests/iop/test_inventory.py → tests/feature/iop/test_inventory.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_inventory_migrate_service(server):
     service = server.service("iop-core-host-inventory-migrate")
     assert service.is_enabled

tests/iop/test_inventory_frontend.py → tests/feature/iop/test_inventory_frontend.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_inventory_frontend_assets_directory(server):
     assets_dir = server.file("/var/www/iop/assets/apps/inventory")
     assert assets_dir.exists

tests/iop/test_kafka.py → tests/feature/iop/test_kafka.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_kafka_service(server):
     service = server.service("iop-core-kafka")
     assert service.is_running

tests/iop/test_puptoo.py → tests/feature/iop/test_puptoo.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_puptoo_service(server):
     service = server.service("iop-core-puptoo")
     assert service.is_running

tests/iop/test_remediation.py → tests/feature/iop/test_remediation.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_remediation_api_service(server):
     service = server.service("iop-service-remediations-api")
     assert service.is_running

tests/iop/test_vmaas.py → tests/feature/iop/test_vmaas.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_vmaas_reposcan_service(server):
     service = server.service("iop-service-vmaas-reposcan")
     assert service.is_running

tests/iop/test_vulnerability.py → tests/feature/iop/test_vulnerability.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_vulnerability_manager_service(server):
     service = server.service("iop-service-vuln-manager")
     assert service.is_running

tests/iop/test_vulnerability_frontend.py → tests/feature/iop/test_vulnerability_frontend.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_vulnerability_frontend_assets_directory(server):
     assets_dir = server.file("/var/www/iop/assets/apps/vulnerability")
     assert assets_dir.exists

tests/iop/test_yuptoo.py → tests/feature/iop/test_yuptoo.py

@@ -1,8 +1,3 @@
-import pytest
-
-pytestmark = pytest.mark.iop
-
-
 def test_yuptoo_service(server):
     service = server.service("iop-core-yuptoo")
     assert service.is_running

tests/foreman_plugins_test.py

@@ -1,6 +1,9 @@
 import pytest
 
-@pytest.mark.parametrize("foreman_plugin", ['foreman_azure_rm', 'foreman_google'])
+@pytest.mark.parametrize("foreman_plugin", [
+    pytest.param('foreman_azure_rm', marks=pytest.mark.feature('azure-rm')),
+    pytest.param('foreman_google', marks=pytest.mark.feature('google')),
+])
 def test_foreman_compute_resources(foremanapi, foreman_plugin):
     plugins = [plugin['name'] for plugin in foremanapi.list('plugins')]
     assert foreman_plugin in plugins