diff --git a/docs/development/testing.md b/docs/development/testing.md index 844411696..d2be103c7 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -27,6 +27,80 @@ For connector changes, start with the [public test command](public-tests.md). It builds its prerequisites and runs without cloud credentials. The [test matrix](../connector-test-matrix.md) lists coverage and CI limits. +## Credential profiles + +The opt-in [profile registry and launcher](../../tools/ci_test_profiles.py) +separate test selection from isolated execution. GCP profiles start pytest +with an allowlisted environment and temporary credential/configuration +directories. General and Snowflake profiles currently support only collection +through this launcher: their remaining Azure and Databricks dependencies have +not been partitioned, so isolated execution fails before setup rather than +silently removing credentials and losing live coverage. + +| Profile | Credentials and ownership | +| --- | --- | +| `general` | Non-GCP selection; isolated launcher is collection-only during migration | +| `snowflake` | Non-GCP selection with `--runsnowflake`; isolated launcher is collection-only during migration | +| `gcp` | GCS/BigQuery tests with federated ADC; local storage for REST integration tests | +| `gcp-aws` | BigQuery training integration test with federated ADC and the S3 dataplane | + +Run from the checkout root with its test dependencies installed: + +```bash +.venv/bin/python -m tools.ci_test_profiles run gcp \ + test/io_utils/fsutil/test_gcs.py -q +.venv/bin/python -m tools.ci_test_profiles run gcp-aws +``` + +During migration, general and Snowflake execution uses pytest directly with +the selection plugin, preserving the existing credential supply: + +```bash +.venv/bin/python -m pytest -p tools.pytest_test_profiles \ + --test-profile general test/connector/ +.venv/bin/python -m pytest -p tools.pytest_test_profiles \ + --test-profile snowflake --runsnowflake test/connector/ +``` + +These direct commands select tests but do not scrub the environment or isolate +credential files. They are a transitional coverage-preserving path, not a +claim that general and Snowflake processes already have minimal credentials. + +GCP execution requires an explicit `GOOGLE_APPLICATION_CREDENTIALS` file of +type `external_account`; ambient user credentials and stored service-account +keys do not activate these profiles. Set `GCP_TEST_SERVICE_ACCOUNT` and +`GCP_PROJECT_ID` for identity-backed REST tests, plus +`GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT` and +`GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT` for BigQuery permission tests. +Missing required credentials fail setup. The +launcher copies ADC into its temporary directory and preserves its external +token-source references, which must remain accessible for the run. + +The GCP-only runtime removes AWS secret fallback and uses a local artifact +directory. The GCP+AWS profile retains S3 because the training test actually +uses it. Credentials are prepared before worker startup; function-specific +identity changes are restored during teardown. These profiles constrain +credential inputs, not network access. Use the public Docker suites when +network isolation is required. + +For a collection audit, `--manifest` saves the selected node IDs without +running fixtures: + +```bash +.venv/bin/python -m tools.ci_test_profiles run gcp \ + --manifest /tmp/gcp-nodes.json --collect-only +``` + +For the unpartitioned baseline, run pytest over the same source revision and +target scope with `-p tools.pytest_test_profiles --collect-only +--test-profile-manifest` and no profile. Its output contains a +`KUMO_TEST_PROFILE_MANIFEST=` JSON record. Compare its `nodeids` with the union +of the `general`, `gcp`, and `gcp-aws` manifests, checking both missing nodes +and overlaps. Audit the Snowflake mode separately with `--runsnowflake`; +existing Snowflake skip rules are independent of GCP ownership. Collection +still imports shared conftests and requires their dependencies. A collection +manifest or local plugin test does not verify live provider access. + ## Report evidence Record the command, environment, result, and any untested dependency. Keep diff --git a/kumo/testing/bigquery.py b/kumo/testing/bigquery.py index da91deaa5..e7811b8c3 100644 --- a/kumo/testing/bigquery.py +++ b/kumo/testing/bigquery.py @@ -20,6 +20,7 @@ ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH, ENV_GOOGLE_CREDENTIALS_JSON, ) +from tools.ci_test_profiles import TestProfile, profile_for DEFAULT_BIGQUERY_KEY = 'kumo-test-bigquery-service-account-key' FAIL_TABLE_PERMISSION_BIGQUERY_KEY = 'kumo-test-permission-fail-bigquery-service-account-key-2' @@ -49,6 +50,11 @@ class CredentialKind(Enum): KEY = 'service_account' +def configured_profile() -> TestProfile | None: + name = os.environ.get('KUMO_TEST_PROFILE') + return profile_for(name) if name else None + + def credential_kind(path: str) -> CredentialKind: if not fsutil.exists(path) or fsutil.isdir(path): raise RuntimeError('Configured BigQuery credential file does not exist') @@ -83,11 +89,20 @@ def setup_bigquery_credentials( if previous is None: environment.delenv(name) - path = (os.environ.get('GOOGLE_APPLICATION_CREDENTIALS') - or os.environ.get(ENV_GOOGLE_CREDENTIALS_JSON)) - if not path and fsutil.exists(ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH): + profile = configured_profile() + profiled_gcp = profile is not None and 'gcp' in profile.capabilities + path = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS') + if profiled_gcp and not path: + raise RuntimeError('GCP test profiles require explicit ' + 'GOOGLE_APPLICATION_CREDENTIALS') + if not profiled_gcp: + path = path or os.environ.get(ENV_GOOGLE_CREDENTIALS_JSON) + if (not profiled_gcp and not path + and fsutil.exists(ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH)): path = ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH kind = credential_kind(path) if path else None + if profiled_gcp and kind != CredentialKind.FEDERATED: + raise RuntimeError('GCP test profiles require external_account ADC') target_variable = _IMPERSONATION_ENV_BY_KEY.get(key) target = None if kind is not None and kind != CredentialKind.KEY and target_variable: @@ -131,9 +146,42 @@ def setup_bigquery_credentials( fsutil.rm(temporary_dir) +@pytest.fixture(scope='session', autouse=True) +def configure_profile_credentials(): + """Prepare ADC before session resources and worker processes start.""" + profile = configured_profile() + if profile is None or 'gcp' not in profile.capabilities: + yield + return + with setup_bigquery_credentials(): + yield + + +@pytest.fixture(autouse=True) +def configure_bigquery_credentials( + request: pytest.FixtureRequest, + configure_profile_credentials, +): + marker = request.node.get_closest_marker('bigquery_credentials') + if configured_profile() is None or marker is None: + yield + return + key, = marker.args + if key is not None and not isinstance(key, str): + raise TypeError('bigquery_credentials requires a string identity key') + with setup_bigquery_credentials(key): + yield + + def withBigQueryCredentials(func: Optional[Callable] = None, service_account_key: Optional[str] = None): def decorator(inner_func: Callable): + marked = pytest.mark.requires_credentials('gcp')( + pytest.mark.bigquery_credentials(service_account_key)(inner_func)) + if configured_profile() is not None: + return marked + # Temporary unprofiled path: existing workflows retain phase-one + # behavior until all invocations use the profile launcher. if asyncio.iscoroutinefunction(inner_func): async def async_wrapper(*args, **kwargs): @@ -148,6 +196,6 @@ def sync_wrapper(*args, **kwargs): return inner_func(*args, **kwargs) wrapper = sync_wrapper - return functools.wraps(inner_func)(wrapper) + return functools.wraps(marked)(wrapper) return decorator if func is None else decorator(func) diff --git a/kumo/testing/gcs.py b/kumo/testing/gcs.py index a511fd19d..c0c952c98 100644 --- a/kumo/testing/gcs.py +++ b/kumo/testing/gcs.py @@ -3,136 +3,135 @@ # SPDX-License-Identifier: Apache-2.0 import base64 +import functools import inspect import json import os -from collections.abc import Callable +from collections.abc import Callable, Generator +from contextlib import ExitStack, contextmanager from unittest import mock import pytest from google.cloud import storage -from io_utils import logging +from io_utils import fsutil, logging from io_utils.fsutil import impl_gcs from kumo.connector.bigquery_connector import ( ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH, ENV_GOOGLE_CREDENTIALS_JSON, ) +from kumo.testing.bigquery import ( + CredentialKind, + configured_profile, + credential_kind, +) logger = logging.getLogger(__name__) - -# Environment variable for GCS test bucket -ENV_GCS_TEST_BUCKET = "GCS_TEST_BUCKET" - -# Default Application Default Credentials path -ADC_DEFAULT_PATH = os.path.join(os.path.expanduser("~"), ".config", "gcloud", - "application_default_credentials.json") - - -def setup_gcs_credentials(): - """Set up GCS credentials for testing. - - This function fetches Google Cloud credentials from the - GOOGLE_CREDENTIALS_STR_JSON environment variable and sets up the necessary - environment variables for GCS testing. - - The credentials should be base64 encoded to avoid issues with special - characters in the JSON string. - """ - - if ENV_GCS_TEST_BUCKET not in os.environ: - os.environ[ENV_GCS_TEST_BUCKET] = 'kumo-test-bucket' - google_creds = os.getenv("GOOGLE_CREDENTIALS_STR_JSON") - if google_creds: - logger.info("Using GOOGLE_CREDENTIALS_STR_JSON environment variable") - google_creds = base64.b64decode(google_creds) - google_creds = google_creds.decode('utf-8') - - file_path = os.path.join(os.getcwd(), ".google_credential.json") - # Parse the JSON string to a dictionary first - try: - google_creds_dict = json.loads(google_creds) - with open(file_path, 'w') as f: - json.dump(google_creds_dict, f) - - with open(ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH, 'w') as f: - json.dump(google_creds_dict, f) - - # Extract and set GCP_PROJECT_ID from service account credentials - # This is needed for GCPSecretProvider initialization - if 'project_id' in google_creds_dict: - os.environ['GCP_PROJECT_ID'] = google_creds_dict['project_id'] - logger.info(f"Set GCP_PROJECT_ID from credentials: " - f"{google_creds_dict['project_id']}") - except json.JSONDecodeError as e: - logger.error(f"Failed to parse GOOGLE_CREDENTIALS_STR_JSON: {e}") - raise - os.environ[ENV_GOOGLE_CREDENTIALS_JSON] = file_path - os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = file_path - else: - logger.warning( - "GOOGLE_CREDENTIALS_STR_JSON environment variable not set. " - "Relying on existing GOOGLE_APPLICATION_CREDENTIALS.") - - -def _check_gcs_credentials() -> bool: - """Check if GCS credentials are available. - - Checks for credentials in the following order: - 1. GOOGLE_APPLICATION_CREDENTIALS env var - 2. ~/.google_credentials.json (ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH) - 3. ~/.config/gcloud/application_default_credentials.json (ADC default) - - If ADC exists but GOOGLE_APPLICATION_CREDENTIALS is not set, it will be - set automatically for the google-cloud libraries to use. - - Returns: - bool: True if credentials are available, False otherwise - """ - setup_gcs_credentials() - - # Check if credentials are already set - if os.getenv('GOOGLE_APPLICATION_CREDENTIALS') is not None: - return True - - if os.path.exists(ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH): - return True - - # Check for ADC default path and set env var if found - if os.path.exists(ADC_DEFAULT_PATH): - logger.info(f"Using ADC credentials from {ADC_DEFAULT_PATH}") - os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = ADC_DEFAULT_PATH - return True - - return False +ENV_GCS_TEST_BUCKET = 'GCS_TEST_BUCKET' +ADC_DEFAULT_PATH = os.path.join(os.path.expanduser('~'), '.config', 'gcloud', + 'application_default_credentials.json') + + +def gcs_inputs_present() -> bool: + return bool(os.environ.get('GOOGLE_APPLICATION_CREDENTIALS') + or os.environ.get('GOOGLE_CREDENTIALS_STR_JSON') + or fsutil.exists(ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH) + or fsutil.exists(ADC_DEFAULT_PATH)) + + +@pytest.hookimpl(tryfirst=True) +def pytest_runtest_setup(item: pytest.Item) -> None: + # Runtime setup precedes pytest's fixture resolution. Missing local inputs + # must skip this item before a session worker can start, without probing + # credentials while importing/collecting modules. + if (configured_profile() is None + and item.get_closest_marker('gcs_credentials') is not None + and not gcs_inputs_present()): + pytest.skip('No GCS credentials found') + + +@contextmanager +def setup_gcs_credentials() -> Generator[None, None, None]: + """Resolve selected test credentials during fixture setup, never import.""" + with ExitStack() as resources, pytest.MonkeyPatch.context() as environment: + for name in ('GOOGLE_APPLICATION_CREDENTIALS', ENV_GOOGLE_CREDENTIALS_JSON, + ENV_GCS_TEST_BUCKET, 'GCP_PROJECT_ID'): + previous = os.environ.get(name) + environment.setenv(name, previous or '') + if previous is None: + environment.delenv(name) + + profile = configured_profile() + profiled_gcp = profile is not None and 'gcp' in profile.capabilities + path = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS') + if profiled_gcp: + if not path: + raise RuntimeError('GCP test profiles require explicit ' + 'GOOGLE_APPLICATION_CREDENTIALS') + if credential_kind(path) != CredentialKind.FEDERATED: + raise RuntimeError('GCP test profiles require external_account ADC') + else: + # Transitional unprofiled inputs; remove when workflow routing and + # local setup migrate. Any generated key is private to this scope. + encoded = os.environ.get('GOOGLE_CREDENTIALS_STR_JSON') + if encoded: + payload = json.loads(base64.b64decode(encoded).decode('utf-8')) + directory = resources.enter_context(fsutil.TemporaryDirectory()) + path = os.path.join(directory, 'credentials.json') + fsutil.write_text(path, json.dumps(payload)) + if payload.get('project_id'): + environment.setenv('GCP_PROJECT_ID', payload['project_id']) + elif not path: + if fsutil.exists(ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH): + path = ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH + elif fsutil.exists(ADC_DEFAULT_PATH): + path = ADC_DEFAULT_PATH + if not path: + pytest.skip('No GCS credentials found') + + environment.setenv('GOOGLE_APPLICATION_CREDENTIALS', path) + environment.setenv(ENV_GOOGLE_CREDENTIALS_JSON, path) + environment.setenv(ENV_GCS_TEST_BUCKET, + os.environ.get(ENV_GCS_TEST_BUCKET) or 'kumo-test-bucket') + yield + + +@pytest.fixture(scope='session', autouse=True) +def configure_gcs_session(request: pytest.FixtureRequest, + configure_profile_credentials): + # Existing workflows can still supply encoded/home-directory credentials. + # Resolve them before workers inherit the environment and keep generated + # files alive until every dependent session resource has shut down. + selected = any(item.get_closest_marker('gcs_credentials') is not None + for item in request.session.items) + if configured_profile() is not None or not selected or not gcs_inputs_present(): + yield + return + with setup_gcs_credentials(): + yield + + +@pytest.fixture(autouse=True) +def configure_gcs_credentials(request: pytest.FixtureRequest, + configure_gcs_session): + if request.node.get_closest_marker('gcs_credentials') is None: + yield + return + with setup_gcs_credentials(): + yield def withGCSCredentials(func: Callable): - """Decorator that sets up GCS credentials and skips tests if unavailable. - - This decorator: - 1. Calls setup_gcs_credentials() to fetch and configure credentials - 2. Skips the test if credentials are not available - 3. Allows tests to run in CI with credentials but skip locally without them - - Args: - func: The test function to decorate - - Example: - @withGCSCredentials - def test_gcs_operation(): - # Test code here - pass - """ - # A collection-time mark prevents live GCS fixtures from being resolved - # when credentials are absent. A runtime wrapper skips too late: pytest - # sets up fixtures before invoking the wrapped test. - credentials_missing = not _check_gcs_credentials() - reason = "No GCS credentials found" if inspect.isgeneratorfunction(func): - from kumo.testing._skip import skip_if - return skip_if(credentials_missing, reason, func) - return pytest.mark.skipif(credentials_missing, reason=reason)(func) + # Pytest does not propagate marks on fixture functions to consumers. + # Dynamic root_gcs parameters carry their own capability marker. + @functools.wraps(func) + def generator(*args, **kwargs): + with setup_gcs_credentials(): + yield from func(*args, **kwargs) + return generator + return pytest.mark.requires_credentials('gcp')( + pytest.mark.gcs_credentials(func)) def withMockedGCSFilesystem(func: Callable): diff --git a/test/conftest.py b/test/conftest.py index 9b72acc80..63e328cf6 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -70,6 +70,16 @@ from kumo.graph.table import Table # noqa: E402 from kumo.task.task import Task # noqa: E402 from kumo.testing import withDatabricksCredentials # noqa: E402 +from kumo.testing.bigquery import ( # noqa: E402, F401 + configure_bigquery_credentials, + configure_profile_credentials, + configured_profile, +) +from kumo.testing.gcs import ( # noqa: E402, F401 + configure_gcs_credentials, + configure_gcs_session, + pytest_runtest_setup, +) from kumo.testing.generator import EdgeSpec, GraphSpec, TableSpec # noqa: E402 from kumo.testing.snowflake import setup_snowflake_credentials # noqa: E402 from kumo.utils.embedding_models import embedding_model_store # noqa: E402 @@ -80,15 +90,18 @@ # OIDC tokens are not available). OIDC-migrated workflows override this # with the dedicated preprod test role after configuring their scoped # workflow identity; legacy workflows retain the existing default. -os.environ.setdefault('AWS_TEST_ROLE_ARN', - 'arn:aws:iam::926922431314:role/unittest-iam-role') +_test_profile = configured_profile() +if _test_profile is None or 'aws' in _test_profile.capabilities: + os.environ.setdefault('AWS_TEST_ROLE_ARN', + 'arn:aws:iam::926922431314:role/unittest-iam-role') _TEMPORAL_LOCAL_STARTUP_TIMEOUT_SECONDS = 30 # Production code resolves GloVe artifacts through deploy_env_config.json. Keep # the public dataset fallback for tests of that configuration path; the # session fixture below supplies small local artifacts to encoder tests. -os.environ.setdefault('CUSTOM_ARTIFACT_ROOT', 's3://kumo-public-datasets') +if _test_profile is None or 'aws' in _test_profile.capabilities: + os.environ.setdefault('CUSTOM_ARTIFACT_ROOT', 's3://kumo-public-datasets') _ENABLE_FSUTIL_DEBUG_LOGS_ENV = 'KUMO_PYTEST_ENABLE_FSUTIL_DEBUG_LOGS' # Keep this in the environment so Kumo logging reinitialization in spawned @@ -191,6 +204,11 @@ def pytest_runtest_makereport(item, call): def pytest_configure(config): + for marker in ( + 'requires_credentials(*capabilities): external test requirements', + 'bigquery_credentials(key): BigQuery test identity', + 'gcs_credentials: GCS test credentials'): + config.addinivalue_line('markers', marker) # register nightly marker config.addinivalue_line("markers", ("nightly(name): Used to mark long-running" diff --git a/test/io_utils/fsutil/test_all.py b/test/io_utils/fsutil/test_all.py index f2b661d39..3b550707d 100644 --- a/test/io_utils/fsutil/test_all.py +++ b/test/io_utils/fsutil/test_all.py @@ -51,7 +51,7 @@ def root_ucv(temp_ucv) -> Generator[str, None, None]: @pytest.fixture( params=[ "root_s3", - "root_gcs", + pytest.param("root_gcs", marks=pytest.mark.requires_credentials('gcp')), "root_ucv", "root_adls", ], diff --git a/test/rest/integ/conftest.py b/test/rest/integ/conftest.py index 3c8589ca9..c3a68d5db 100644 --- a/test/rest/integ/conftest.py +++ b/test/rest/integ/conftest.py @@ -234,7 +234,10 @@ def storage_backend(request): @pytest.fixture(scope="session") def _diskgraph_deploy_config(tmp_path_factory, storage_backend, request, - temporal_workflow_env): + temporal_workflow_env, + configure_test_glove_artifacts, + configure_profile_credentials, + configure_gcs_session): """Load diskgraph JSON config, override storage root, and set DEPLOY_ENV_CONFIG_JSON so deploy_env() picks it up. diff --git a/test/rest/test_tables.py b/test/rest/test_tables.py index 459e22c2b..228042e4d 100644 --- a/test/rest/test_tables.py +++ b/test/rest/test_tables.py @@ -191,6 +191,7 @@ def sort_fn(x: ColumnSummary) -> str: @withBigQueryCredentials +@pytest.mark.requires_credentials('gcp') def test_validate_size(client: TestClient): endpoint = "/v1/tables/validate" diff --git a/test/scripts/test_ci_test_profiles.py b/test/scripts/test_ci_test_profiles.py new file mode 100644 index 000000000..8807eb515 --- /dev/null +++ b/test/scripts/test_ci_test_profiles.py @@ -0,0 +1,317 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import json +import os +import subprocess +import sys +from unittest import mock + +import pytest +from coverage import CoverageData + +from io_utils import fsutil +from tools import ci_test_profiles +from tools.ci_test_profiles import ( + GCP_AWS_MODULE, + GCP_MIXED_MODULE, + GCP_MODULES, + GCP_NODES, + MANIFEST_PREFIX, + collect_manifest, + explicit_targets, + integration_required, + main, + owns_node, + profile_for, + pytest_arguments, + run_profile, + runtime_environment, +) + + +def test_gcp_ownership(): + ordinary = profile_for('gcp') + cross_provider = profile_for('gcp-aws') + assert set(ordinary.targets).isdisjoint(cross_provider.targets) + assert set(ordinary.targets) | set(cross_provider.targets) == ( + GCP_MODULES | GCP_NODES | {GCP_MIXED_MODULE}) + assert cross_provider.targets == (GCP_AWS_MODULE,) + assert cross_provider.capabilities == {'gcp', 'aws'} + assert ordinary.capabilities == {'gcp'} + assert 'test/rest/integ/test_gcs_public_api_e2e.py' in ordinary.targets + + +def test_registry_import_has_no_application_side_effects(): + result = subprocess.run([ + sys.executable, '-c', + 'import sys; import tools.ci_test_profiles; ' + 'assert not any(n == "io_utils" or n.startswith("kumo.") ' + 'for n in sys.modules)', + ], check=False, capture_output=True, text=True) + assert result.returncode == 0, result.stderr + + +def test_environment_is_allowlisted(): + source = { + 'PATH': '/bin', 'GOOGLE_APPLICATION_CREDENTIALS': '/explicit/adc.json', + 'GCP_PROJECT_ID': 'project', 'GCP_STORAGE_TRANSFER_BUCKET': 'bucket', + 'AWS_ACCESS_KEY_ID': 'aws-secret', 'AWS_PROFILE': 'ambient', + 'AWS_CONTAINER_CREDENTIALS_FULL_URI': 'http://credential-service', + 'AZURE_CLIENT_SECRET': 'azure-secret', 'SNOWFLAKE_PASSWORD': 'password', + 'GITHUB_TOKEN': 'token', 'ACTIONS_ID_TOKEN_REQUEST_TOKEN': 'token', + 'GOOGLE_CREDENTIALS_STR_JSON': 'old-key', 'HOME': '/old/home', + 'DEPLOY_ENV_CONFIG_JSON': '{"secrets":{"backend":"aws_sm"}}', + 'PYTEST_ADDOPTS': '-p unsafe', 'NEW_PROVIDER_TOKEN': 'unlisted', + 'COVERAGE_FILE': '/ambient/coverage', + 'COVERAGE_PROCESS_START': '/ambient/config', + } + environment = runtime_environment(profile_for('gcp'), source, '/isolated') + for key in source.keys() - { + 'PATH', 'GOOGLE_APPLICATION_CREDENTIALS', 'GCP_PROJECT_ID', + 'GCP_STORAGE_TRANSFER_BUCKET', 'HOME'}: + assert key not in environment + assert environment['HOME'] == '/isolated/home' + assert environment['TMPDIR'] == '/isolated/tmp' + assert environment['AWS_EC2_METADATA_DISABLED'] == 'true' + assert environment['AWS_SHARED_CREDENTIALS_FILE'].startswith('/isolated/') + assert environment['KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED'] == 'true' + assert source['HOME'] == '/old/home' + + +def test_cross_provider_preserves_only_explicit_aws(): + source = {'AWS_ACCESS_KEY_ID': 'access', 'AWS_SECRET_ACCESS_KEY': 'secret', + 'AWS_SESSION_TOKEN': 'session', 'AWS_PROFILE': 'default', + 'AWS_TEST_ROLE_ARN': 'arn:aws:iam::123:role/explicit-test', + 'SNOWFLAKE_PASSWORD': 'snowflake'} + environment = runtime_environment(profile_for('gcp-aws'), source, '/tmp/run') + assert environment['AWS_ACCESS_KEY_ID'] == 'access' + assert environment['AWS_SECRET_ACCESS_KEY'] == 'secret' + assert environment['AWS_SESSION_TOKEN'] == 'session' + assert environment['AWS_TEST_ROLE_ARN'] == source['AWS_TEST_ROLE_ARN'] + assert 'AWS_PROFILE' not in environment + assert 'SNOWFLAKE_PASSWORD' not in environment + + +def test_general_does_not_receive_snowflake(): + source = {'AWS_ACCESS_KEY_ID': 'access', 'SNOWFLAKE_PASSWORD': 'secret'} + general = runtime_environment(profile_for('general'), source, '/tmp/general') + snowflake = runtime_environment(profile_for('snowflake'), source, '/tmp/snow') + assert 'SNOWFLAKE_PASSWORD' not in general + assert snowflake['SNOWFLAKE_PASSWORD'] == 'secret' + assert general['AWS_ACCESS_KEY_ID'] == snowflake['AWS_ACCESS_KEY_ID'] + + +def test_marked_parameters_have_one_owner(): + profiles = [profile_for(name) for name in ('general', 'gcp', 'gcp-aws')] + for module, nodeid, required, owner in [ + (GCP_MIXED_MODULE, 'test[root_gcs]', frozenset({'gcp'}), 'gcp'), + (GCP_MIXED_MODULE, 'test[root_local]', frozenset(), 'general'), + (GCP_AWS_MODULE, 'test', frozenset({'gcp', 'aws'}), 'gcp-aws'), + ('test/rest/test_tables.py', next(iter(GCP_NODES)), + frozenset(), 'gcp')]: + assert [profile.id for profile in profiles + if owns_node(profile, module, nodeid, required)] == [owner] + assert owns_node(profile_for('snowflake'), GCP_MIXED_MODULE) + assert not owns_node(profile_for('snowflake'), GCP_AWS_MODULE) + + +@pytest.mark.parametrize('paths,required', [ + ([], True), (['docs/setup.md', 'README.md'], False), + (['docs/example.py'], True), (['examples/README.md'], True), + (['docs/../tools/runner.py'], True), (['uv.lock'], True), + (['test/conftest.py'], True), (['.github/workflows/testing.yml'], True), + (['unrecognized'], True), ([''], True), +]) +def test_impact_is_conservative(paths, required): + assert integration_required(paths) is required + + +@pytest.mark.parametrize('arguments', [ + ['-p', 'unsafe'], ['--noconftest'], ['--test-profile', 'non-gcp'], + ['--override-ini', 'addopts=-p unsafe'], ['../other/test.py'], + ['--unknown'], ['--timeout'], +]) +def test_launcher_rejects_bypasses(arguments): + with pytest.raises(ValueError): + pytest_arguments(arguments) + + +def test_launcher_accepts_collection_controls(): + arguments = ['--collect-only', '-q', '-k', 'permission', '--tb=short', + '--capture=fd', '--verbosity=0', + 'test/connector/test_bigquery_connector.py'] + assert pytest_arguments(arguments) == arguments + with pytest.raises(ValueError, match='Unknown test profile'): + profile_for('unknown') + assert not explicit_targets(['-k', 'test/name']) + assert explicit_targets(arguments) + + +@pytest.mark.parametrize('identifier', ['gcp', 'gcp-aws']) +def test_launch_scopes_config_and_credentials(monkeypatch, tmp_path, identifier): + source = str(tmp_path / 'source.json') + payload = '{"type":"external_account","audience":"test"}' + fsutil.write_text(source, payload) + monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', source) + monkeypatch.setenv('DEPLOY_ENV_CONFIG_JSON', 'must-not-survive') + monkeypatch.setenv('AWS_ACCESS_KEY_ID', 'aws-access') + created = [] + + def execute(command, env): + assert command[:4] == [sys.executable, '-m', 'pytest', '-p'] + assert '-q' in command + assert command[command.index('--test-profile') + 1] == identifier + assert '--test-profile-manifest' in command + assert 'DEPLOY_ENV_CONFIG_JSON' not in env + copied = env['GOOGLE_APPLICATION_CREDENTIALS'] + assert copied != source + assert fsutil.read_text(copied) == payload + config = json.loads(fsutil.read_text(env['DEPLOY_ENV_CONFIG_FILE'])) + assert config['secrets'] == {'backend': 'env_var', 'fallbacks': []} + if identifier == 'gcp': + assert not config['storage']['artifact_root'].startswith('s3:') + assert 'AWS_ACCESS_KEY_ID' not in env + else: + assert config['storage']['artifact_root'].startswith('s3:') + assert env['AWS_ACCESS_KEY_ID'] == 'aws-access' + assert fsutil.read_text(env['AWS_CONFIG_FILE']) == '' + created.append(os.path.dirname(copied)) + return 17, json.dumps({'profile': identifier, 'nodeids': ['test']}) + + monkeypatch.setattr(ci_test_profiles, 'collect_manifest', execute) + assert run_profile(profile_for(identifier), ['-q'], + str(tmp_path / 'manifest.json')) == 17 + assert len(created) == 1 + assert not fsutil.exists(created[0]) + assert fsutil.read_text(source) == payload + assert os.environ['GOOGLE_APPLICATION_CREDENTIALS'] == source + assert json.loads(fsutil.read_text(str(tmp_path / 'manifest.json'))) == { + 'profile': identifier, 'nodeids': ['test']} + + +def test_manifest_stream_preserves_status(capsys): + payload = json.dumps({'profile': 'gcp', 'nodeids': ['test::case']}) + command = [sys.executable, '-c', + f'print("ordinary output"); print({MANIFEST_PREFIX + payload!r}); ' + 'raise SystemExit(3)'] + result, captured = collect_manifest(command, {'KUMO_TEST_PROFILE': 'gcp'}) + assert result == 3 + assert captured == payload + assert 'ordinary output' in capsys.readouterr().out + + +def test_wrong_profile_manifest_fails(): + payload = json.dumps({'profile': 'non-gcp', 'nodeids': []}) + with pytest.raises(ValueError, match='Invalid pytest profile manifest'): + collect_manifest([sys.executable, '-c', + f'print({MANIFEST_PREFIX + payload!r})'], + {'KUMO_TEST_PROFILE': 'gcp'}) + + +def test_missing_explicit_adc_never_launches(monkeypatch): + monkeypatch.delenv('GOOGLE_APPLICATION_CREDENTIALS', raising=False) + execute = mock.Mock() + monkeypatch.setattr(subprocess, 'run', execute) + with pytest.raises(ValueError, match='require explicit'): + run_profile(profile_for('gcp'), []) + execute.assert_not_called() + + +def test_collection_needs_no_credentials(monkeypatch): + monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', '/unreadable/input') + + def execute(command, *, env, check): + assert '--collect-only' in command + assert 'GOOGLE_APPLICATION_CREDENTIALS' not in env + assert 'GOOGLE_CREDENTIALS_JSON' not in env + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(subprocess, 'run', execute) + assert run_profile(profile_for('gcp'), ['--collect-only']) == 0 + + +def test_remote_adc_is_rejected_before_read(monkeypatch): + monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', 's3://private/key.json') + reader = mock.Mock(side_effect=AssertionError('must not read remote ADC')) + monkeypatch.setattr(fsutil, 'read_text', reader) + with pytest.raises(ValueError, match='local ADC file'): + run_profile(profile_for('gcp'), []) + reader.assert_not_called() + + +@pytest.mark.parametrize('identifier', ['general', 'snowflake'], + ids=['general', 'warehouse']) +def test_unclassified_runtime_refuses_before_setup(monkeypatch, identifier): + reader = mock.Mock(side_effect=AssertionError('must not read credentials')) + execute = mock.Mock(side_effect=AssertionError('must not execute pytest')) + temporary = mock.Mock(side_effect=AssertionError('must not create runtime')) + monkeypatch.setattr(fsutil, 'read_text', reader) + monkeypatch.setattr(fsutil, 'TemporaryDirectory', temporary) + monkeypatch.setattr(subprocess, 'run', execute) + with pytest.raises(ValueError, match='isolated collection only'): + run_profile(profile_for(identifier), ['-q']) + reader.assert_not_called() + execute.assert_not_called() + temporary.assert_not_called() + + +@pytest.mark.parametrize('identifier', ['general', 'snowflake'], + ids=['general', 'warehouse']) +def test_unclassified_collection_is_available(monkeypatch, identifier): + def execute(command, *, env, check): + assert '--collect-only' in command + assert ('--runsnowflake' in command) == (identifier == 'snowflake') + assert env['KUMO_TEST_PROFILE'] == identifier + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(subprocess, 'run', execute) + assert run_profile(profile_for(identifier), ['--collect-only']) == 0 + + +def test_unknown_cli_option_fails_before_launch(monkeypatch): + execute = mock.Mock() + monkeypatch.setattr(subprocess, 'run', execute) + with pytest.raises(SystemExit) as error: + main(['run', 'gcp', '--unknown']) + assert error.value.code == 2 + execute.assert_not_called() + + +def test_inner_child_produces_coverage(monkeypatch, tmp_path): + root = str(tmp_path) + config = fsutil.read_text('rest/devserver/configs/diskgraph-test.json') + for directory in ('tools', 'test/connector', 'rest/devserver/configs'): + fsutil.makedirs(os.path.join(root, directory), exist_ok=True) + fsutil.write_text(os.path.join(root, 'tools/__init__.py'), '') + for module in ('ci_test_profiles.py', 'pytest_test_profiles.py'): + fsutil.write_text(os.path.join(root, 'tools', module), + fsutil.read_text('tools/' + module)) + fsutil.write_text(os.path.join(root, 'rest/devserver/configs/diskgraph-test.json'), + config) + fsutil.write_text(os.path.join(root, 'example.py'), 'VALUE = 42\n') + fsutil.write_text(os.path.join(root, '.coveragerc'), + '[run]\nsource = example\nbranch = True\n' + 'concurrency = thread,multiprocessing\nrelative_files = True\n') + fsutil.write_text(os.path.join(root, 'test/connector/test_bigquery_connector.py'), + 'def test_measured():\n' + ' import example\n' + ' assert example.VALUE == 42\n') + adc = os.path.join(root, 'adc.json') + fsutil.write_text(adc, '{"type":"external_account"}') + monkeypatch.chdir(root) + monkeypatch.setenv('PYTHONPATH', root) + monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', adc) + monkeypatch.setenv('COVERAGE_FILE', '/must-not-write/ambient') + destination = os.path.join(root, '.coverage-profile') + assert main(['run', 'gcp', '--coverage-file', destination, + 'test/connector/test_bigquery_connector.py', '-q']) == 0 + files = fsutil.listdir(root, include=[destination + '*']) + assert files + measured = set() + for path in files: + data = CoverageData(basename=path) + data.read() + measured.update(data.measured_files()) + assert 'example.py' in measured diff --git a/test/scripts/test_pytest_test_profiles.py b/test/scripts/test_pytest_test_profiles.py new file mode 100644 index 000000000..3b624d116 --- /dev/null +++ b/test/scripts/test_pytest_test_profiles.py @@ -0,0 +1,219 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Exercise the profile plugin in independent, dependency-light pytest runs.""" + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from io_utils import fsutil + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +_GCP_MODULE = 'test/io_utils/fsutil/test_gcs.py' +_AWS_MODULE = 'test/rest/integ/test_bigquery_diskgraph_e2e.py' + + +def _write(root: Path, name: str, text: str) -> None: + target = root / name + fsutil.makedirs(str(target.parent), exist_ok=True) + fsutil.write_text(str(target), text) + + +def _run(root: Path, *args: str) -> subprocess.CompletedProcess[str]: + environment = { + key: value for key, value in os.environ.items() + if not key.startswith(('PYTEST_', 'KUMO_TEST_PROFILE')) + } + environment['PYTHONPATH'] = str(REPOSITORY_ROOT) + environment['PYTEST_DISABLE_PLUGIN_AUTOLOAD'] = '1' + return subprocess.run( + [sys.executable, '-m', 'pytest', '-c', str(root / 'pytest.ini'), + '--rootdir', str(root), '--confcutdir', str(root), + '-p', 'tools.pytest_test_profiles', '-q', *args], + cwd=root, env=environment, capture_output=True, text=True, check=False, + ) + + +@pytest.fixture +def suite(tmp_path: Path) -> Path: + _write(tmp_path, 'pytest.ini', '[pytest]\n') + _write(tmp_path, 'test_local.py', 'def test_local(): pass\n') + _write(tmp_path, _GCP_MODULE, 'def test_gcs(): pass\n') + _write(tmp_path, _AWS_MODULE, 'def test_training(): pass\n') + _write(tmp_path, 'test/rest/test_tables.py', ''' +def test_validate_size(): pass +def test_local_table(): pass +''') + _write(tmp_path, 'test/io_utils/fsutil/test_all.py', ''' +import os +import pytest + +@pytest.fixture(params=[ + "root_s3", + pytest.param("root_gcs", marks=pytest.mark.requires_credentials("gcp")), + "root_ucv", "root_adls", +], ids=["s3", "gcs", "ucv", "adls"]) +def root(request): + profile = os.getenv("KUMO_TEST_PROFILE") + if profile: + assert (request.param == "root_gcs") == (profile == "gcp") + return request.param + +def test_contract(root): pass +''') + return tmp_path + + +def test_partition_and_manifest(suite: Path) -> None: + baseline = _run(suite, '--collect-only') + assert baseline.returncode == 0, baseline.stdout + baseline.stderr + baseline_nodes = { + line for line in baseline.stdout.splitlines() if '::' in line + } + selected: set[str] = set() + for profile in ('general', 'gcp', 'gcp-aws'): + result = _run(suite, '--test-profile', profile, + '--test-profile-manifest') + assert result.returncode == 0, result.stdout + result.stderr + payload = json.loads(next( + line.removeprefix('KUMO_TEST_PROFILE_MANIFEST=') + for line in result.stdout.splitlines() + if line.startswith('KUMO_TEST_PROFILE_MANIFEST='))) + assert payload['profile'] == profile + nodes = set(payload['nodeids']) + assert len(nodes) == len(payload['nodeids']) + assert not selected.intersection(nodes) + selected.update(nodes) + assert selected == baseline_nodes + assert 'test/io_utils/fsutil/test_all.py::test_contract[gcs]' in selected + + +def test_ignored_before_import(suite: Path) -> None: + _write(suite, _GCP_MODULE, 'raise RuntimeError("provider imported")\n') + _write(suite, _AWS_MODULE, 'raise RuntimeError("AWS imported")\n') + result = _run(suite, '--test-profile', 'general') + assert result.returncode == 0, result.stdout + result.stderr + assert 'provider imported' not in result.stdout + + +@pytest.mark.parametrize('profile,target', [ + ('general', _GCP_MODULE), + ('general', _AWS_MODULE), + ('general', 'test/rest/test_tables.py::test_validate_size'), + ('gcp', _AWS_MODULE), + ('gcp-aws', _GCP_MODULE), +]) +def test_explicit_incompatible_target(suite: Path, profile: str, + target: str) -> None: + _write(suite, 'conftest.py', 'raise RuntimeError("conftest imported")\n') + result = _run(suite, '--test-profile', profile, target) + assert result.returncode == pytest.ExitCode.USAGE_ERROR + assert 'incompatible' in result.stderr + assert 'conftest imported' not in result.stderr + + +def test_unknown_profile(suite: Path) -> None: + _write(suite, 'conftest.py', 'raise RuntimeError("conftest imported")\n') + result = _run(suite, '--test-profile', 'typo') + assert result.returncode == pytest.ExitCode.USAGE_ERROR + assert 'conftest imported' not in result.stderr + + +def test_profile_precedes_conftest(suite: Path) -> None: + _write(suite, 'conftest.py', ''' +import os +assert os.environ["KUMO_TEST_PROFILE"] == "gcp" +''') + result = _run(suite, '--test-profile', 'gcp') + assert result.returncode == 0, result.stdout + result.stderr + + +def test_unprofiled_unchanged(suite: Path) -> None: + result = _run(suite) + assert result.returncode == 0, result.stdout + result.stderr + assert '9 passed' in result.stdout + + +def test_unprofiled_manifest(suite: Path) -> None: + result = _run(suite, '--collect-only', '--test-profile-manifest') + assert result.returncode == 0, result.stdout + result.stderr + assert '"profile": "unprofiled"' in result.stdout + + +def test_missing_capability_fails(suite: Path) -> None: + _write(suite, _GCP_MODULE, ''' +import pytest +@pytest.mark.requires_credentials("gcp", "aws") +def test_gcs(): pass +''') + result = _run(suite, '--test-profile', 'gcp') + assert result.returncode == pytest.ExitCode.USAGE_ERROR + assert 'undeclared profile capabilities: aws' in result.stderr + + +@pytest.mark.parametrize('profile', ['gcp', 'gcp-aws']) +@pytest.mark.parametrize('target', ['test_local.py', + 'test/unrelated/test_import.py']) +def test_positive_explicit_import_guard(suite: Path, profile: str, + target: str) -> None: + _write(suite, target, 'raise RuntimeError("unowned module imported")\n') + result = _run(suite, '--test-profile', profile, target) + assert result.returncode == pytest.ExitCode.USAGE_ERROR + assert 'incompatible' in result.stderr + assert 'unowned module imported' not in result.stdout + result.stderr + + +@pytest.mark.parametrize('profile', ['gcp', 'gcp-aws']) +def test_positive_directory_import_guard(suite: Path, profile: str) -> None: + _write(suite, 'test/unrelated/test_import.py', + 'raise RuntimeError("unowned module imported")\n') + _write(suite, 'test/unrelated/conftest.py', + 'raise RuntimeError("unowned conftest imported")\n') + _write(suite, 'test/io_utils/fsutil/test_unowned.py', + 'raise RuntimeError("neighbor module imported")\n') + result = _run(suite, '--test-profile', profile, 'test') + assert result.returncode == 0, result.stdout + result.stderr + + +@pytest.mark.parametrize('profile,target', [ + ('gcp', 'test/rest/test_tables.py::test_local_table'), + ('gcp-aws', 'test/rest/test_tables.py'), + ('general', 'test/io_utils/fsutil/test_all.py::test_contract[gcs]'), + ('gcp', 'test/io_utils/fsutil/test_all.py::test_contract[s3]'), + ('gcp', 'test/io_utils/fsutil/test_all.py::test_contract[adls]'), + ('gcp', 'test/io_utils/fsutil/test_all.py::test_contract[ucv]'), +]) +def test_mixed_explicit_import_guard(suite: Path, profile: str, + target: str) -> None: + _write(suite, target.split('::')[0], + 'raise RuntimeError("incompatible mixed module imported")\n') + result = _run(suite, '--test-profile', profile, target) + assert result.returncode == pytest.ExitCode.USAGE_ERROR + assert 'incompatible' in result.stderr + assert 'mixed module imported' not in result.stdout + result.stderr + + +def test_unregistered_gcp_fails(suite: Path) -> None: + _write(suite, 'test_new_provider.py', ''' +import pytest +@pytest.mark.requires_credentials("gcp") +def test_new_provider(): pass +''') + result = _run(suite, '--test-profile', 'general') + assert result.returncode == pytest.ExitCode.USAGE_ERROR + assert 'not registered' in result.stderr + + +def test_external_explicit_import_guard(suite: Path) -> None: + _write(suite.parent, 'external_test.py', + 'raise RuntimeError("external module imported")\n') + result = _run(suite, '--test-profile', 'gcp', + str(suite.parent / 'external_test.py')) + assert result.returncode == pytest.ExitCode.USAGE_ERROR + assert 'incompatible' in result.stderr + assert 'external module imported' not in result.stdout + result.stderr diff --git a/test/testing/test_bigquery_credentials.py b/test/testing/test_bigquery_credentials.py index 156e3422a..9483fa067 100644 --- a/test/testing/test_bigquery_credentials.py +++ b/test/testing/test_bigquery_credentials.py @@ -24,6 +24,7 @@ 'GOOGLE_CLOUD_PROJECT', testing_bigquery.ENV_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT, testing_bigquery.ENV_BIGQUERY_NO_JOB_SERVICE_ACCOUNT, + 'KUMO_TEST_PROFILE', ) _RESTRICTED_KEYS = ( (testing_bigquery.FAIL_TABLE_PERMISSION_BIGQUERY_KEY, @@ -243,6 +244,50 @@ def test_fallback_identity_sequence(monkeypatch): assert secrets.call_args_list == [mock.call(key) for key in keys] +@pytest.mark.parametrize('profile', ['gcp', 'gcp-aws']) +@pytest.mark.parametrize('kind', ['service_account', 'authorized_user', + 'impersonated_service_account']) +def test_profile_rejects_nonfederated(monkeypatch, tmp_path, profile, kind): + monkeypatch.setenv('KUMO_TEST_PROFILE', profile) + path = write_credentials(tmp_path, kind) + monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', path) + before = {name: os.environ.get(name) for name in _ENV_NAMES} + with pytest.raises(RuntimeError, match='external_account ADC'): + with testing_bigquery.setup_bigquery_credentials(): + pytest.fail('nonfederated ADC must fail before test setup') + testing_bigquery.get_secrets.assert_not_called() + assert {name: os.environ.get(name) for name in _ENV_NAMES} == before + + +@pytest.mark.parametrize('profile', ['gcp', 'gcp-aws']) +def test_profile_requires_explicit_adc(monkeypatch, tmp_path, profile): + monkeypatch.setenv('KUMO_TEST_PROFILE', profile) + path = write_credentials(tmp_path, 'external_account') + monkeypatch.setenv(testing_bigquery.ENV_GOOGLE_CREDENTIALS_JSON, path) + monkeypatch.setattr(testing_bigquery, 'ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH', + path) + with pytest.raises(RuntimeError, match='GOOGLE_APPLICATION_CREDENTIALS'): + with testing_bigquery.setup_bigquery_credentials(): + pytest.fail('ambient files must not activate profiled tests') + testing_bigquery.get_secrets.assert_not_called() + + +def test_profile_decorator_only_marks(monkeypatch): + monkeypatch.setenv('KUMO_TEST_PROFILE', 'gcp') + read = mock.Mock(side_effect=AssertionError('credential I/O during import')) + monkeypatch.setattr(fsutil, 'read_text', read) + + def test_body(): + return 'test body' + + decorated = testing_bigquery.withBigQueryCredentials(test_body) + assert decorated is test_body + assert {mark.name for mark in decorated.pytestmark} == { + 'requires_credentials', 'bigquery_credentials'} + read.assert_not_called() + testing_bigquery.get_secrets.assert_not_called() + + def test_bigquery_rest_identity_uses_workload_exchange(monkeypatch): service_account = ( 'gha-kumo-gcp-tests@example-project.iam.gserviceaccount.com') diff --git a/test/testing/test_gcs_credentials.py b/test/testing/test_gcs_credentials.py new file mode 100644 index 000000000..bb29dde8a --- /dev/null +++ b/test/testing/test_gcs_credentials.py @@ -0,0 +1,241 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import base64 +import json +import os +from unittest import mock + +import pytest + +from io_utils import fsutil +from kumo.testing import bigquery as testing_bigquery +from kumo.testing import gcs as testing_gcs + +pytest_plugins = ['pytester'] + +_ENV_NAMES = ( + 'KUMO_TEST_PROFILE', 'GOOGLE_APPLICATION_CREDENTIALS', + 'GOOGLE_CREDENTIALS_JSON', 'GOOGLE_CREDENTIALS_STR_JSON', 'GCS_TEST_BUCKET', + 'GCP_PROJECT_ID', 'GOOGLE_IMPERSONATE_SERVICE_ACCOUNT', + 'GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT', 'GCP_STORAGE_TRANSFER_BUCKET', +) + + +@pytest.fixture(autouse=True) +def isolated_credentials(monkeypatch, tmp_path): + for name in _ENV_NAMES: + monkeypatch.setenv(name, '') + monkeypatch.delenv(name) + for module, attribute in ( + (testing_gcs, 'ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH'), + (testing_gcs, 'ADC_DEFAULT_PATH')): + monkeypatch.setattr(module, attribute, str(tmp_path / attribute)) + + +def test_decorators_do_not_probe(monkeypatch): + setup = mock.Mock(side_effect=AssertionError('setup during collection')) + monkeypatch.setattr(testing_gcs, 'setup_gcs_credentials', setup) + + def test_body(): + return 'test body' + + def fixture_body(): + yield 'fixture body' + + assert testing_gcs.withGCSCredentials(test_body) is test_body + testing_gcs.withGCSCredentials(fixture_body) + setup.assert_not_called() + + +@pytest.mark.parametrize('kind', ['service_account', 'authorized_user']) +def test_profile_rejects_nonfederated(monkeypatch, tmp_path, kind): + monkeypatch.setenv('KUMO_TEST_PROFILE', 'gcp') + path = str(tmp_path / 'adc.json') + fsutil.write_text(path, json.dumps({'type': kind})) + monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', path) + before = {name: os.environ.get(name) for name in _ENV_NAMES} + with pytest.raises(RuntimeError, match='external_account ADC'): + with testing_gcs.setup_gcs_credentials(): + pytest.fail('profile must reject nonfederated ADC') + assert {name: os.environ.get(name) for name in _ENV_NAMES} == before + + +def test_profile_requires_explicit_adc(monkeypatch): + monkeypatch.setenv('KUMO_TEST_PROFILE', 'gcp') + monkeypatch.setenv('GOOGLE_CREDENTIALS_STR_JSON', base64.b64encode( + b'{"type":"service_account"}').decode()) + with pytest.raises(RuntimeError, match='GOOGLE_APPLICATION_CREDENTIALS'): + with testing_gcs.setup_gcs_credentials(): + pytest.fail('profile must ignore ambient encoded keys') + + +@pytest.mark.parametrize('failure', [False, True]) +def test_unprofiled_key_lifetime(monkeypatch, failure): + encoded = base64.b64encode(json.dumps({ + 'type': 'service_account', 'project_id': 'test-project'}).encode()).decode() + monkeypatch.setenv('GOOGLE_CREDENTIALS_STR_JSON', encoded) + before = {name: os.environ.get(name) for name in _ENV_NAMES} + created = [] + + def invoke(): + with testing_gcs.setup_gcs_credentials(): + path = os.environ['GOOGLE_APPLICATION_CREDENTIALS'] + created.append(path) + assert fsutil.exists(path) + assert os.environ['GCP_PROJECT_ID'] == 'test-project' + if failure: + raise ValueError('test failure') + + if failure: + with pytest.raises(ValueError, match='test failure'): + invoke() + else: + invoke() + assert len(created) == 1 + assert not fsutil.exists(created[0]) + assert {name: os.environ.get(name) for name in _ENV_NAMES} == before + + +@pytest.mark.parametrize('failure', [False, True]) +def test_profile_fixture_order(pytester, monkeypatch, tmp_path, failure): + path = str(tmp_path / 'adc.json') + fsutil.write_text(path, '{"type":"external_account"}') + monkeypatch.setenv('KUMO_TEST_PROFILE', 'gcp') + monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', path) + monkeypatch.setenv('GOOGLE_CREDENTIALS_JSON', 'before-scope') + monkeypatch.setenv('GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT', 'no-job@example.test') + before = {name: os.environ.get(name) for name in _ENV_NAMES} + pytester.makeini(''' +[pytest] +markers = + requires_credentials: external capabilities + bigquery_credentials: BigQuery identity + gcs_credentials: GCS identity +''') + pytester.makepyfile(''' +import os +import pytest +from kumo.testing.bigquery import withBigQueryCredentials, FAIL_PROJECT_PERMISSION_BIGQUERY_KEY +from kumo.testing.gcs import withGCSCredentials + +@pytest.fixture(scope='session') +def workers(configure_profile_credentials): + assert os.environ['GOOGLE_CREDENTIALS_JSON'] == os.environ['GOOGLE_APPLICATION_CREDENTIALS'] + assert 'GOOGLE_IMPERSONATE_SERVICE_ACCOUNT' not in os.environ + yield + assert 'GOOGLE_IMPERSONATE_SERVICE_ACCOUNT' not in os.environ + +@pytest.fixture +def cloud_resource(workers): + assert os.environ['GOOGLE_IMPERSONATE_SERVICE_ACCOUNT'] == 'no-job@example.test' + yield + assert os.environ['GOOGLE_IMPERSONATE_SERVICE_ACCOUNT'] == 'no-job@example.test' + +@withBigQueryCredentials(service_account_key=FAIL_PROJECT_PERMISSION_BIGQUERY_KEY) +def test_restricted(cloud_resource): + os.environ['GOOGLE_CREDENTIALS_JSON'] = 'test mutation' + ''' + ("raise ValueError('expected failure')" if failure else "pass") + ''' + +@withGCSCredentials +def test_gcs(workers): + assert os.environ['GOOGLE_CREDENTIALS_JSON'] == os.environ['GOOGLE_APPLICATION_CREDENTIALS'] + assert 'GOOGLE_IMPERSONATE_SERVICE_ACCOUNT' not in os.environ +''') + result = pytester.runpytest_inprocess( + '-q', '-o', 'addopts=', plugins=[testing_bigquery, testing_gcs]) + result.assert_outcomes(passed=1 if failure else 2, failed=1 if failure else 0) + assert {name: os.environ.get(name) for name in _ENV_NAMES} == before + assert fsutil.read_text(path) == '{"type":"external_account"}' + + +def test_profile_missing_adc_before_resources(pytester, monkeypatch): + monkeypatch.setenv('KUMO_TEST_PROFILE', 'gcp') + pytester.makepyfile(''' +import pytest +@pytest.fixture(scope='session') +def workers(configure_profile_credentials): + raise AssertionError('resource fixture must not run') +def test_cloud(workers): + pass +''') + result = pytester.runpytest_inprocess( + '-q', '-o', 'addopts=', plugins=[testing_bigquery]) + result.assert_outcomes(errors=1) + result.stdout.fnmatch_lines(['*require explicit*GOOGLE_APPLICATION_CREDENTIALS*']) + + +@pytest.mark.parametrize('source', ['encoded', 'home']) +def test_unprofiled_session_credentials(pytester, monkeypatch, tmp_path, source): + payload = '{"type":"service_account","project_id":"test-project"}' + if source == 'encoded': + monkeypatch.setenv('GOOGLE_CREDENTIALS_STR_JSON', + base64.b64encode(payload.encode()).decode()) + else: + fsutil.write_text(testing_gcs.ADC_DEFAULT_PATH, payload) + before = {name: os.environ.get(name) for name in _ENV_NAMES} + pytester.makeini(''' +[pytest] +markers = + requires_credentials: external capabilities + gcs_credentials: GCS identity +''') + record = str(tmp_path / 'worker-input-path') + monkeypatch.setenv('GCS_TEST_WORKER_INPUT', record) + pytester.makepyfile(''' +import os +import pytest +from io_utils import fsutil +from kumo.testing.gcs import withGCSCredentials + +@pytest.fixture(scope='session') +def workers(configure_gcs_session): + path = os.environ['GOOGLE_APPLICATION_CREDENTIALS'] + assert fsutil.exists(path) + fsutil.write_text(os.environ['GCS_TEST_WORKER_INPUT'], path) + yield path + # The input inherited when workers started still exists at shutdown. + assert fsutil.exists(path) + +@withGCSCredentials +def test_first(workers): + assert fsutil.exists(workers) + +@withGCSCredentials +def test_second(workers): + assert fsutil.exists(workers) +''') + result = pytester.runpytest_inprocess( + '-q', '-o', 'addopts=', plugins=[testing_bigquery, testing_gcs]) + result.assert_outcomes(passed=2) + path = fsutil.read_text(record) + assert fsutil.exists(path) == (source == 'home') + assert {name: os.environ.get(name) for name in _ENV_NAMES} == before + + +def test_unprofiled_missing_inputs_skip_workers(pytester): + pytester.makeini(''' +[pytest] +markers = + requires_credentials: external capabilities + gcs_credentials: GCS identity +''') + pytester.makepyfile(''' +import pytest +from kumo.testing.gcs import withGCSCredentials + +@pytest.fixture(scope='session') +def workers(): + raise AssertionError('worker must not start without credentials') + +@withGCSCredentials +def test_cloud(workers): + pass + +def test_local(): + pass +''') + result = pytester.runpytest_inprocess( + '-q', '-o', 'addopts=', plugins=[testing_bigquery, testing_gcs]) + result.assert_outcomes(passed=1, skipped=1) diff --git a/tools/ci_test_profiles.py b/tools/ci_test_profiles.py new file mode 100644 index 000000000..7dddf17b7 --- /dev/null +++ b/tools/ci_test_profiles.py @@ -0,0 +1,351 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Declare test ownership and launch pytest with scoped runtime credentials. + +The registry imports only the standard library so collection can consult it +before importing application modules. Run the launcher from the checkout root +after installing dependencies; package installation is outside this boundary. +""" + +import argparse +import json +import os +import subprocess +import sys +from collections.abc import Mapping, Sequence +from dataclasses import dataclass + +GCP_AWS_MODULE = 'test/rest/integ/test_bigquery_diskgraph_e2e.py' +GCP_MODULES = frozenset({ + 'test/connector/test_bigquery_connector.py', + 'test/io_utils/fsutil/test_gcs.py', + 'test/rest/integ/test_gcs_e2e.py', + 'test/rest/integ/test_gcs_public_api_e2e.py', + 'test/rest/integ/test_bigquery_ingestion_diskgraph_e2e.py', + GCP_AWS_MODULE, +}) +GCP_NODES = frozenset({'test/rest/test_tables.py::test_validate_size'}) +GCP_MIXED_MODULE = 'test/io_utils/fsutil/test_all.py' +MANIFEST_PREFIX = 'KUMO_TEST_PROFILE_MANIFEST=' + + +@dataclass(frozen=True) +class TestProfile: + id: str + label: str + targets: tuple[str, ...] + selection: str + backend: str + capabilities: frozenset[str] + environment: str + + +PROFILES = { + # These two profiles describe selection during migration. Isolated runtime + # execution is disabled until their other provider dependencies are split. + 'general': TestProfile( + 'general', 'non-GCP tests', ('test',), 'non-gcp', 'default', + frozenset({'aws'}), 'azure-adls-tests'), + 'snowflake': TestProfile( + 'snowflake', 'Snowflake tests', ('test',), 'non-gcp', 'snowflake', + frozenset({'aws', 'snowflake'}), 'gha-kumo-workflow-tests'), + 'gcp': TestProfile( + 'gcp', 'GCP tests', + (*sorted(GCP_MODULES - {GCP_AWS_MODULE}), GCP_MIXED_MODULE, + *sorted(GCP_NODES)), + 'gcp', 'local', frozenset({'gcp'}), 'gha-kumo-workflow-tests'), + 'gcp-aws': TestProfile( + 'gcp-aws', f'gcs-e2e-tests ({GCP_AWS_MODULE})', (GCP_AWS_MODULE,), + 'gcp-aws', 's3', frozenset({'gcp', 'aws'}), + 'gha-kumo-workflow-tests'), +} + +# Explicit runtime inputs, not inherited prefixes: newly introduced secrets do +# not silently become available to every test process. +RUNTIME_ENV = frozenset({ + 'PATH', 'PYTHONPATH', 'VIRTUAL_ENV', 'LD_LIBRARY_PATH', 'LANG', 'LC_ALL', + 'TZ', 'TERM', 'NO_COLOR', 'SSL_CERT_FILE', 'SSL_CERT_DIR', + 'REQUESTS_CA_BUNDLE', 'CUDA_VISIBLE_DEVICES', 'CUDA', 'FULL_TEST', + 'DEPLOY_STAGE', 'DEPLOY_ENV_CONFIG_FILE', 'KUMO_TESTING', + 'TEMPORAL_SERVER', 'SPARK_LOCAL_IP', + 'SPARK_VERSION', 'DATA_SPARK_MEM', 'MAX_DATA_BACKEND_WORKER_PROCESSES', + 'KUMO_ENABLE_GLOVE', 'KUMO_GLOVE_PATH', 'ARTIFACTS_DIR', 'GLOG_logtostdout', + 'OMP_NUM_THREADS', 'MKL_NUM_THREADS', 'OPENBLAS_NUM_THREADS', + 'PYARROW_IGNORE_TIMEZONE', 'PYTHONUNBUFFERED', +}) +PROVIDER_ENV = { + 'gcp': frozenset({ + 'GOOGLE_APPLICATION_CREDENTIALS', 'GCP_PROJECT_ID', + 'GOOGLE_CLOUD_PROJECT', 'GCP_TEST_SERVICE_ACCOUNT', + 'GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT', + 'GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT', 'GCS_TEST_BUCKET', + 'GCP_STORAGE_TRANSFER_BUCKET', + }), + 'aws': frozenset({ + 'AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_SESSION_TOKEN', + 'AWS_REGION', 'AWS_DEFAULT_REGION', 'AWS_TEST_ROLE_ARN', + }), + 'snowflake': frozenset({ + 'SNOWFLAKE_USER', 'SNOWFLAKE_PASSWORD', 'SNOWFLAKE_ACCOUNT', + 'SNOWFLAKE_PRIVATE_KEY', 'SNOWFLAKE_PRIVATE_KEY_PASSPHRASE', + 'SNOWFLAKE_WAREHOUSE', 'SNOWFLAKE_DATABASE', 'SNOWFLAKE_SCHEMA', + 'SNOWFLAKE_ROLE', + }), + 'azure': frozenset({ + 'AZURE_CLIENT_ID', 'AZURE_TENANT_ID', 'AZURE_CLIENT_SECRET', + 'AZURE_STORAGE_ACCOUNT', 'AZURE_CONTAINER_NAME', + 'AZURE_STORAGE_KEY', 'AZURE_STORAGE_SAS_TOKEN', 'KUMO_RUN_ADLS_TESTS', + }), + 'databricks': frozenset({ + 'DATABRICKS_HOST', 'DATABRICKS_TOKEN', 'DATABRICKS_CLIENT_ID', + 'DATABRICKS_CLIENT_SECRET', 'DATABRICKS_WAREHOUSE_ID', + }), +} +PYTEST_VALUE_OPTIONS = frozenset({ + '-k', '-m', '--maxfail', '--tb', '--durations', '--timeout', + '--timeout-method', '--splits', '--group', '--splitting-algorithm', + '--junitxml', '--capture', '--verbosity', +}) + + +def profile_for(identifier: str) -> TestProfile: + try: + return PROFILES[identifier] + except KeyError: + raise ValueError(f'Unknown test profile: {identifier}') from None + + +def owns_node(profile: TestProfile, module: str, nodeid: str = '', + capabilities: frozenset[str] = frozenset()) -> bool: + gcp = (module in GCP_MODULES or nodeid.split('[', 1)[0] in GCP_NODES + or 'gcp' in capabilities) + if profile.selection == 'non-gcp': + return not gcp + if profile.selection == 'gcp-aws': + return module == GCP_AWS_MODULE + return gcp and module != GCP_AWS_MODULE + + +def permits_target(profile: TestProfile, target: str) -> bool: + """Reject incompatible files and explicit nodes before importing them.""" + while target.startswith('./'): + target = target[2:] + if target in {'', '.'}: + return True + module = target.split('::', 1)[0].rstrip('/') + if module.startswith('/') or '..' in module.split('/'): + return False + node = target.split('[', 1)[0] + if module == GCP_MIXED_MODULE and '[' in target: + parameters = target.split('[', 1)[1].rstrip(']').split('-') + providers = set(parameters) & {'gcs', 's3', 'ucv', 'adls'} + if providers: + if profile.selection == 'non-gcp': + return 'gcs' not in providers + return profile.selection == 'gcp' and providers == {'gcs'} + if profile.selection == 'non-gcp': + return module not in GCP_MODULES and node not in GCP_NODES + for owned in profile.targets: + owned_module = owned.split('::', 1)[0] + if owned_module.startswith(module + '/'): + return True + if module != owned_module: + continue + if '::' not in target or '::' not in owned: + return True + if node == owned or owned.startswith(node + '::'): + return True + return False + + +def integration_required(paths: Sequence[str]) -> bool: + """Only known prose-only changes may avoid live integration execution.""" + if not paths: + return True + for path in paths: + if not path or path.startswith('/') or '..' in path.split('/'): + return True + if path in {'README.md', 'CONTRIBUTING.md'}: + continue + if path.startswith('docs/') and path.endswith('.md'): + continue + return True + return False + + +def runtime_environment(profile: TestProfile, + source: Mapping[str, str], root: str) -> dict[str, str]: + allowed = set(RUNTIME_ENV) + for capability in profile.capabilities: + allowed.update(PROVIDER_ENV[capability]) + environment = {key: value for key, value in source.items() if key in allowed} + environment.update({ + 'HOME': os.path.join(root, 'home'), + 'TMPDIR': os.path.join(root, 'tmp'), + 'XDG_CONFIG_HOME': os.path.join(root, 'home', '.config'), + 'XDG_CACHE_HOME': os.path.join(root, 'home', '.cache'), + 'AWS_CONFIG_FILE': os.path.join(root, 'empty-aws-config'), + 'AWS_SHARED_CREDENTIALS_FILE': os.path.join(root, 'empty-aws-credentials'), + 'BOTO_CONFIG': os.path.join(root, 'empty-boto-config'), + 'AWS_EC2_METADATA_DISABLED': 'true', + 'KUMO_TEST_PROFILE': profile.id, + 'KUMO_RUN_GCP_TESTS': '1' if 'gcp' in profile.capabilities else '0', + 'KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED': ( + 'true' if 'gcp' in profile.capabilities else 'false'), + }) + return environment + + +def pytest_arguments(arguments: Sequence[str]) -> list[str]: + """Allow test selectors and reporting controls, not plugin/config bypasses.""" + flags = {'-q', '-v', '-vv', '-s', '-x', '--collect-only', + '--disable-warnings', '--strict-markers'} + result = [] + index = 0 + while index < len(arguments): + argument = arguments[index] + if argument in flags: + result.append(argument) + elif argument.split('=', 1)[0] in PYTEST_VALUE_OPTIONS: + result.append(argument) + if '=' not in argument: + index += 1 + if index == len(arguments): + raise ValueError(f'Missing value for pytest option {argument}') + result.append(arguments[index]) + elif argument.startswith('test/') and '..' not in argument.split('/'): + result.append(argument) + else: + raise ValueError(f'Unsupported pytest argument: {argument}') + index += 1 + return result + + +def explicit_targets(arguments: Sequence[str]) -> bool: + index = 0 + while index < len(arguments): + argument = arguments[index] + if argument.startswith('test/'): + return True + if argument in PYTEST_VALUE_OPTIONS: + index += 1 + index += 1 + return False + + +def run_profile(profile: TestProfile, arguments: Sequence[str], + manifest: str | None = None, + coverage_file: str | None = None) -> int: + extra = pytest_arguments(arguments) + if profile.selection == 'non-gcp' and '--collect-only' not in extra: + raise ValueError( + f'Profile {profile.id} supports isolated collection only until ' + 'Azure and Databricks credential dependencies are classified; ' + 'run pytest directly with the profile plugin and existing ' + 'credential configuration for execution') + # Registry consumers run before application bootstrap; filesystem adapters + # are needed only when executing a prepared test environment. + from io_utils import fsutil + + with fsutil.TemporaryDirectory() as root: + environment = runtime_environment(profile, os.environ, root) + fsutil.makedirs(environment['HOME'], exist_ok=True) + fsutil.makedirs(environment['TMPDIR'], exist_ok=True) + for key in ('AWS_CONFIG_FILE', 'AWS_SHARED_CREDENTIALS_FILE', + 'BOTO_CONFIG'): + fsutil.write_text(environment[key], '') + if 'gcp' in profile.capabilities and '--collect-only' not in extra: + source = environment.get('GOOGLE_APPLICATION_CREDENTIALS') + if not source: + raise ValueError('GCP profiles require explicit ' + 'GOOGLE_APPLICATION_CREDENTIALS') + if '://' in source: + raise ValueError('GCP profiles require a local ADC file') + path = os.path.join(root, 'adc.json') + fsutil.write_text(path, fsutil.read_text(os.path.abspath(source))) + environment['GOOGLE_APPLICATION_CREDENTIALS'] = path + environment['GOOGLE_CREDENTIALS_JSON'] = path + else: + environment.pop('GOOGLE_APPLICATION_CREDENTIALS', None) + if 'gcp' in profile.capabilities: + config = json.loads(fsutil.read_text( + 'rest/devserver/configs/diskgraph-test.json')) + config['secrets'] = {'backend': 'env_var', 'fallbacks': []} + if 'aws' not in profile.capabilities: + config['storage']['artifact_root'] = os.path.join(root, 'artifacts') + fsutil.makedirs(config['storage']['artifact_root'], exist_ok=True) + config_path = os.path.join(root, 'deploy.json') + fsutil.write_text(config_path, json.dumps(config)) + environment['DEPLOY_ENV_CONFIG_FILE'] = config_path + # Explicit targets replace the profile's default roots. The plugin + # rejects targets belonging to a different profile before collection. + has_targets = explicit_targets(extra) + command = [sys.executable] + if coverage_file: + command.extend(['-m', 'coverage', 'run', + '--data-file=' + os.path.abspath(coverage_file)]) + command.extend(['-m', 'pytest', '-p', + 'tools.pytest_test_profiles', '--test-profile', profile.id]) + if manifest: + command.append('--test-profile-manifest') + if profile.backend == 'snowflake': + command.append('--runsnowflake') + if not has_targets: + command.extend(profile.targets) + command.extend(extra) + if not manifest: + return subprocess.run(command, env=environment, check=False).returncode + result, payload = collect_manifest(command, environment) + if payload is not None: + fsutil.write_text(os.path.abspath(manifest), payload + '\n') + elif result == 0: + raise ValueError('Pytest completed without a profile manifest') + return result + + +def collect_manifest(command: list[str], + environment: dict[str, str]) -> tuple[int, str | None]: + payload = None + with subprocess.Popen(command, env=environment, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True) as process: + assert process.stdout is not None + for line in process.stdout: + print(line, end='', flush=True) + if line.startswith(MANIFEST_PREFIX): + payload = line[len(MANIFEST_PREFIX):].strip() + document = json.loads(payload) + if (not isinstance(document, dict) + or document.get('profile') != environment['KUMO_TEST_PROFILE'] + or not isinstance(document.get('nodeids'), list) + or not all(isinstance(node, str) + for node in document['nodeids'])): + raise ValueError('Invalid pytest profile manifest') + return process.wait(), payload + + +def main(arguments: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest='command', required=True) + run = subparsers.add_parser('run') + run.add_argument('profile', choices=tuple(PROFILES)) + run.add_argument('--manifest') + run.add_argument('--coverage-file') + subparsers.add_parser('impact') + options, extra = parser.parse_known_args(arguments) + if options.command == 'impact': + if extra: + parser.error('impact does not accept arguments') + print(json.dumps({'run_integrations': integration_required( + sys.stdin.read().splitlines())})) + return 0 + try: + return run_profile(profile_for(options.profile), extra, options.manifest, + options.coverage_file) + except ValueError as error: + parser.error(str(error)) + return 2 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/pytest_test_profiles.py b/tools/pytest_test_profiles.py new file mode 100644 index 000000000..7cd6d2a0a --- /dev/null +++ b/tools/pytest_test_profiles.py @@ -0,0 +1,144 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Select credential profiles before provider imports and fixture setup.""" + +import json +import os +from pathlib import Path + +import pytest + +from tools.ci_test_profiles import ( + GCP_MIXED_MODULE, + GCP_MODULES, + GCP_NODES, + MANIFEST_PREFIX, + TestProfile, + owns_node, + permits_target, + profile_for, +) + + +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption('--test-profile', default=None, + help='Select an isolated CI credential profile') + parser.addoption('--test-profile-manifest', action='store_true', + help='Emit selected node IDs after collection') + + +def _profile(name: str) -> TestProfile: + try: + return profile_for(name) + except ValueError as error: + raise pytest.UsageError(str(error)) from error + + +def _relative(path: Path, root: Path) -> str: + try: + return path.resolve().relative_to(root.resolve()).as_posix() + except ValueError: + return path.resolve().as_posix() + + +def pytest_load_initial_conftests(early_config: pytest.Config, + parser: pytest.Parser, + args: list[str]) -> None: + options = parser.parse_known_args(args) + if options.test_profile is None: + return + profile = _profile(options.test_profile) + os.environ['KUMO_TEST_PROFILE'] = profile.id + for target in options.file_or_dir: + filename, *nodes = target.split('::') + path = Path(filename) + if not path.is_absolute(): + path = early_config.invocation_params.dir / path + relative = _relative(path, early_config.rootpath) + nodeid = '::'.join([relative, *nodes]) + if not permits_target(profile, nodeid): + raise pytest.UsageError( + f'{target} is incompatible with test profile {profile.id}') + + +def pytest_configure(config: pytest.Config) -> None: + for marker in ( + 'requires_credentials(*capabilities): explicit live providers', + 'bigquery_credentials(service_account_key): BigQuery prerequisite', + 'gcs_credentials: GCS prerequisite', + ): + config.addinivalue_line('markers', marker) + name = config.getoption('test_profile') + if name is not None: + _profile(name) + + +def pytest_ignore_collect(collection_path: Path, + config: pytest.Config) -> bool | None: + name = config.getoption('test_profile') + if name is None: + return None + relative = _relative(collection_path, config.rootpath) + if not permits_target(_profile(name), relative): + return True + return None + + +@pytest.hookimpl(tryfirst=True) +def pytest_collection_modifyitems(config: pytest.Config, + items: list[pytest.Item]) -> None: + name = config.getoption('test_profile') + if name is None: + return + profile = _profile(name) + selected = [] + deselected = [] + for item in items: + relative = _relative(item.path, config.rootpath) + capabilities = { + capability + for marker in item.iter_markers('requires_credentials') + for capability in marker.args + } + base_nodeid = item.nodeid.split('[', 1)[0] + if (item.get_closest_marker('gcs_credentials') is not None + or item.get_closest_marker('bigquery_credentials') is not None): + capabilities.add('gcp') + if ('gcp' in capabilities and relative not in GCP_MODULES + and relative != GCP_MIXED_MODULE + and base_nodeid not in GCP_NODES): + raise pytest.UsageError( + f'{item.nodeid} requires GCP but is not registered in ' + 'the test profile collection targets') + if owns_node(profile, relative, base_nodeid, frozenset(capabilities)): + missing = capabilities - profile.capabilities + if missing: + raise pytest.UsageError( + f'{item.nodeid} requires undeclared profile capabilities: ' + f'{", ".join(sorted(missing))}') + selected.append(item) + else: + deselected.append(item) + items[:] = selected + if deselected: + config.hook.pytest_deselected(items=deselected) + + +def pytest_collection_finish(session: pytest.Session) -> None: + if not session.config.getoption('test_profile_manifest'): + return + profile = session.config.getoption('test_profile') + if profile is None: + if not session.config.getoption('collectonly'): + raise pytest.UsageError( + 'Unprofiled manifests require --collect-only') + profile = 'unprofiled' + payload = json.dumps({ + 'profile': profile, + 'nodeids': [item.nodeid for item in session.items], + }) + reporter = session.config.pluginmanager.get_plugin('terminalreporter') + if reporter is None: + raise pytest.UsageError('Profile manifests require the terminal reporter') + reporter.write_line(MANIFEST_PREFIX + payload)