diff --git a/kumo/testing/bigquery.py b/kumo/testing/bigquery.py index 82854aaf9..0301cb6bc 100644 --- a/kumo/testing/bigquery.py +++ b/kumo/testing/bigquery.py @@ -2,174 +2,144 @@ # All rights reserved. # SPDX-License-Identifier: Apache-2.0 -import asyncio -import functools import json import os -from collections.abc import Callable -from typing import Optional +from collections.abc import Callable, Generator +from contextlib import contextmanager +from enum import Enum -from io_utils import fsutil, logging -from io_utils.secrets import get_secrets +import pytest + +from io_utils import fsutil from kumo.connector.bigquery_connector import ( ENV_GCS_KUMO_BUCKET, - ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH, ENV_GOOGLE_CREDENTIALS_JSON, ) from service_framework.env_var import EnvVar -logger = logging.getLogger(__name__) - -DEFAULT_BIGQUERY_KEY = "kumo-test-bigquery-service-account-key" - -# Missing required table read permission -FAIL_TABLE_PERMISSION_BIGQUERY_KEY = "kumo-test-permission-fail-bigquery-service-account-key-2" -# Missing required jobs.create permission -FAIL_PROJECT_PERMISSION_BIGQUERY_KEY = "kumo-test-permission-fail-bigquery-service-account-key" - -# Keyless CI maps the two legacy negative-test identities to deliberately -# restricted service accounts. The names remain keyed by the old secret -# identifiers so callers do not need to know how credentials are sourced. -ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT = "GOOGLE_IMPERSONATE_SERVICE_ACCOUNT" -ENV_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT = ( - "GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT") -ENV_BIGQUERY_NO_JOB_SERVICE_ACCOUNT = "GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT" - -_IMPERSONATION_ENV_BY_KEY = { - FAIL_TABLE_PERMISSION_BIGQUERY_KEY: ENV_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT, - FAIL_PROJECT_PERMISSION_BIGQUERY_KEY: ENV_BIGQUERY_NO_JOB_SERVICE_ACCOUNT, -} - - -def _configure_application_default_credentials( - service_account_key: str) -> bool: - """Configure an existing ADC file for BigQuery connector tests. - - GitHub's OIDC authentication action writes an ``external_account`` ADC - file and exports ``GOOGLE_APPLICATION_CREDENTIALS``. Negative permission - tests select a restricted service account through IAM impersonation rather - than loading a second private key. - """ - require_wif = EnvVar('KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED', bool, - default_value=False).parse_value() - credentials_path = os.getenv('GOOGLE_APPLICATION_CREDENTIALS') - if not credentials_path or not fsutil.exists(credentials_path): - if require_wif: - raise RuntimeError( - 'Workload exchange BigQuery tests require federated ADC in ' - 'GOOGLE_APPLICATION_CREDENTIALS') - return False - - if require_wif: - credentials = json.loads(fsutil.read_text(credentials_path)) - if credentials.get('type') != 'external_account': - raise RuntimeError( - 'Workload exchange BigQuery tests require external_account ADC') - - impersonation_env = _IMPERSONATION_ENV_BY_KEY.get(service_account_key) - if impersonation_env is None: - os.environ.pop(ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT, None) - else: - target = os.getenv(impersonation_env) - if not target: - raise RuntimeError( - f"{impersonation_env} must be configured for the restricted " - "BigQuery permission test identity") - os.environ[ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT] = target - - os.environ[ENV_GOOGLE_CREDENTIALS_JSON] = credentials_path - project_id = (os.getenv('GCP_PROJECT_ID') - or os.getenv('GOOGLE_CLOUD_PROJECT')) - if project_id: - os.environ['GCP_PROJECT_ID'] = project_id - logger.info("Using Application Default Credentials for BigQuery tests") - return True - - -def setup_bigquery_credentials(service_account_key: Optional[str] = None): - # Set env var for GCS kumo bucket for testing. - os.environ[ENV_GCS_KUMO_BUCKET.name] = 'kumo-test-bucket' - service_account_key = service_account_key or DEFAULT_BIGQUERY_KEY - - # Prefer short-lived ADC supplied by Workload Identity Federation. This - # keeps GitHub Actions from reading any stored GCP service-account key. - if _configure_application_default_credentials(service_account_key): - return +ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT = 'GOOGLE_IMPERSONATE_SERVICE_ACCOUNT' +ENV_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT = 'GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT' +ENV_BIGQUERY_NO_JOB_SERVICE_ACCOUNT = 'GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT' +ENV_BIGQUERY_NO_TABLE_CREDENTIALS = 'GCP_BIGQUERY_NO_TABLE_CREDENTIALS' +ENV_BIGQUERY_NO_JOB_CREDENTIALS = 'GCP_BIGQUERY_NO_JOB_CREDENTIALS' +GCP_WORKERS_STARTED = pytest.StashKey[bool]() - try: - # Get the GCloud secrets from AWS secrets manager and store it in the - # default file. This remains a local/deployment fallback; CI supplies - # ADC above and never reaches this path. - secrets = get_secrets(service_account_key) - file_path = os.path.join(os.getcwd(), ".google_credential.json") - with open(file_path, 'w') as f: - json.dump(secrets, f) - - with open(ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH, 'w') as f: - # This is a hack. allows big query connector - # to fall back to default credentials - json.dump(secrets, f) - os.environ[ENV_GOOGLE_CREDENTIALS_JSON] = file_path - # Also set the standard ADC env var so any google-cloud-* client - # built without explicit credentials (e.g. fsutil's GCS client - # used by integration tests to verify EXPORT DATA output) picks - # up the same service account. - os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = file_path - except Exception as e: - # Use built-in credentials if this env doesn't have access to read the - # secrets from AWS - logger.exception(e) - - -def withBigQueryCredentials(func: Optional[Callable] = None, - service_account_key: Optional[str] = None): - import pytest +_CREDENTIAL_ENV_VARS = ( + 'GOOGLE_APPLICATION_CREDENTIALS', + ENV_GOOGLE_CREDENTIALS_JSON, + ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT, + ENV_GCS_KUMO_BUCKET.name, + 'GCP_PROJECT_ID', +) - def decorator(inner_func: Callable): - def _setup(): - setup_bigquery_credentials(service_account_key) - if (not os.getenv(ENV_GOOGLE_CREDENTIALS_JSON) and - not os.path.exists(ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH)): - pytest.skip("No BigQuery credentials found") - - if asyncio.iscoroutinefunction(inner_func): - - async def async_wrapper(*args, **kwargs): - previous_target = os.environ.get( - ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT) - try: - _setup() - return await inner_func(*args, **kwargs) - finally: - if previous_target is None: - os.environ.pop(ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT, - None) - else: - os.environ[ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT] = ( - previous_target) - - wrapper = async_wrapper + +class BigQueryIdentity(Enum): + DEFAULT = 'default' + NO_TABLE = 'no_table' + NO_JOB = 'no_job' + + +class CredentialKind(Enum): + FEDERATED = 'external_account' + USER = 'authorized_user' + IMPERSONATED = 'impersonated_service_account' + KEY = 'service_account' + + +def credential_kind(path: str) -> CredentialKind: + if '://' in path: + raise RuntimeError('Google test credentials require a local ADC file') + if not fsutil.exists(path) or fsutil.isdir(path): + raise RuntimeError('Configured BigQuery credential file does not exist') + try: + document = json.loads(fsutil.read_text(path)) + if not isinstance(document, dict): + raise ValueError('credential document must be a JSON object') + return CredentialKind(document.get('type')) + except (ValueError, TypeError) as error: + raise RuntimeError('BigQuery credential file has an invalid or ' + 'unsupported credential type') from error + + +def resolve_credentials(identity: BigQueryIdentity) -> tuple[str, str | None]: + """Resolve explicit inputs without changing the environment or files.""" + if not isinstance(identity, BigQueryIdentity): + raise TypeError('BigQuery credentials require a BigQueryIdentity') + path = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS') + if not path: + raise RuntimeError('BigQuery tests require explicit ' + 'GOOGLE_APPLICATION_CREDENTIALS') + kind = credential_kind(path) + require_wif = EnvVar( + 'KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED', bool, default_value=False).parse_value() + if require_wif and kind != CredentialKind.FEDERATED: + raise RuntimeError('Workload exchange BigQuery tests require ' + 'external_account ADC') + if identity == BigQueryIdentity.DEFAULT: + return path, None + + if kind == CredentialKind.KEY: + variable = { + BigQueryIdentity.NO_TABLE: ENV_BIGQUERY_NO_TABLE_CREDENTIALS, + BigQueryIdentity.NO_JOB: ENV_BIGQUERY_NO_JOB_CREDENTIALS, + }[identity] + restricted_path = os.environ.get(variable) + if not restricted_path: + raise RuntimeError(f'{variable} must identify the restricted ' + 'BigQuery service-account credential file') + if credential_kind(restricted_path) != CredentialKind.KEY: + raise RuntimeError(f'{variable} must contain service_account ' + 'credentials') + return restricted_path, None + + variable = { + BigQueryIdentity.NO_TABLE: ENV_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT, + BigQueryIdentity.NO_JOB: ENV_BIGQUERY_NO_JOB_SERVICE_ACCOUNT, + }[identity] + target = os.environ.get(variable) + if not target: + raise RuntimeError(f'{variable} must be configured for ' + 'the restricted BigQuery test identity') + return path, target + + +@contextmanager +def setup_bigquery_credentials( + identity: BigQueryIdentity = BigQueryIdentity.DEFAULT, +) -> Generator[None, None, None]: + """Scope explicit ADC to a test or session; input files remain read-only.""" + path, target = resolve_credentials(identity) + with pytest.MonkeyPatch.context() as environment: + for name in _CREDENTIAL_ENV_VARS: + previous = os.environ.get(name) + environment.setenv(name, previous or '') + if previous is None: + environment.delenv(name) + + environment.setenv('GOOGLE_APPLICATION_CREDENTIALS', path) + environment.setenv(ENV_GOOGLE_CREDENTIALS_JSON, path) + if target: + environment.setenv(ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT, target) else: + environment.delenv(ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT, + raising=False) + environment.setenv( + ENV_GCS_KUMO_BUCKET.name, + os.environ.get(ENV_GCS_KUMO_BUCKET.name) or 'kumo-test-bucket') + project = (os.environ.get('GCP_PROJECT_ID') + or os.environ.get('GOOGLE_CLOUD_PROJECT')) + if project: + environment.setenv('GCP_PROJECT_ID', project) + yield + + +def withBigQueryCredentials( + func: Callable | None = None, + *, + identity: BigQueryIdentity = BigQueryIdentity.DEFAULT, +): + """Declare credentials without performing auth or wrapping the test body.""" + def decorator(inner_func: Callable): + return pytest.mark.bigquery_credentials(identity)(inner_func) - def sync_wrapper(*args, **kwargs): - previous_target = os.environ.get( - ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT) - try: - _setup() - return inner_func(*args, **kwargs) - finally: - if previous_target is None: - os.environ.pop(ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT, - None) - else: - os.environ[ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT] = ( - previous_target) - - wrapper = sync_wrapper - - return functools.wraps(inner_func)(wrapper) - - if func is None: - return decorator - else: - return decorator(func) + return decorator if func is None else decorator(func) diff --git a/kumo/testing/gcs.py b/kumo/testing/gcs.py index 3d0846bee..c18e330d7 100644 --- a/kumo/testing/gcs.py +++ b/kumo/testing/gcs.py @@ -2,130 +2,37 @@ # All rights reserved. # SPDX-License-Identifier: Apache-2.0 -import base64 -import json import os -from collections.abc import Callable +from collections.abc import Callable, Generator +from contextlib import contextmanager from unittest import mock import pytest from google.cloud import storage -from io_utils import logging from io_utils.fsutil import impl_gcs -from kumo.connector.bigquery_connector import ( - ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH, - ENV_GOOGLE_CREDENTIALS_JSON, -) +from kumo.testing.bigquery import setup_bigquery_credentials -logger = logging.getLogger(__name__) +ENV_GCS_TEST_BUCKET = 'GCS_TEST_BUCKET' -# Environment variable for GCS test bucket -ENV_GCS_TEST_BUCKET = "GCS_TEST_BUCKET" -# Default Application Default Credentials path -ADC_DEFAULT_PATH = os.path.join(os.path.expanduser("~"), ".config", "gcloud", - "application_default_credentials.json") - - -def setup_gcs_credentials(): - """Set up GCS credentials for testing. - - This function fetches Google Cloud credentials from the - GOOGLE_CREDENTIALS_STR_JSON environment variable and sets up the necessary - environment variables for GCS testing. - - The credentials should be base64 encoded to avoid issues with special - characters in the JSON string. - """ - - if ENV_GCS_TEST_BUCKET not in os.environ: - os.environ[ENV_GCS_TEST_BUCKET] = 'kumo-test-bucket' - # CI supplies short-lived ADC. Never replace it with a stored key, and - # reuse the selected credentials when another test requests this setup. - credentials_path = os.getenv('GOOGLE_APPLICATION_CREDENTIALS') - if credentials_path: - os.environ[ENV_GOOGLE_CREDENTIALS_JSON] = credentials_path - return - google_creds = os.getenv("GOOGLE_CREDENTIALS_STR_JSON") - if google_creds: - logger.info("Using GOOGLE_CREDENTIALS_STR_JSON environment variable") - google_creds = base64.b64decode(google_creds) - google_creds = google_creds.decode('utf-8') - - file_path = os.path.join(os.getcwd(), ".google_credential.json") - # Parse the JSON string to a dictionary first - try: - google_creds_dict = json.loads(google_creds) - with open(file_path, 'w') as f: - json.dump(google_creds_dict, f) - - with open(ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH, 'w') as f: - json.dump(google_creds_dict, f) - - # Extract and set GCP_PROJECT_ID from service account credentials - # This is needed for GCPSecretProvider initialization - if 'project_id' in google_creds_dict: - os.environ['GCP_PROJECT_ID'] = google_creds_dict['project_id'] - logger.info(f"Set GCP_PROJECT_ID from credentials: " - f"{google_creds_dict['project_id']}") - except json.JSONDecodeError as e: - logger.error(f"Failed to parse GOOGLE_CREDENTIALS_STR_JSON: {e}") - raise - os.environ[ENV_GOOGLE_CREDENTIALS_JSON] = file_path - os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = file_path - else: - logger.warning( - "GOOGLE_CREDENTIALS_STR_JSON environment variable not set. " - "Relying on existing GOOGLE_APPLICATION_CREDENTIALS.") - - -def _check_gcs_credentials() -> bool: - """Check if GCS credentials are available. - - Checks for credentials in the following order: - 1. GOOGLE_APPLICATION_CREDENTIALS env var - 2. ~/.google_credentials.json (ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH) - 3. ~/.config/gcloud/application_default_credentials.json (ADC default) - - If ADC exists but GOOGLE_APPLICATION_CREDENTIALS is not set, it will be - set automatically for the google-cloud libraries to use. - - Returns: - bool: True if credentials are available, False otherwise - """ - setup_gcs_credentials() - - # Check if credentials are already set - if os.getenv('GOOGLE_APPLICATION_CREDENTIALS') is not None: - return True - - if os.path.exists(ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH): - return True - - # Check for ADC default path and set env var if found - if os.path.exists(ADC_DEFAULT_PATH): - logger.info(f"Using ADC credentials from {ADC_DEFAULT_PATH}") - os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = ADC_DEFAULT_PATH - return True - - return False +@contextmanager +def setup_gcs_credentials() -> Generator[None, None, None]: + """Scope explicit ADC without discovering or writing credential files.""" + with setup_bigquery_credentials(), pytest.MonkeyPatch.context() as environment: + environment.setenv(ENV_GCS_TEST_BUCKET, + os.environ.get(ENV_GCS_TEST_BUCKET) or 'kumo-test-bucket') + # Cached clients otherwise retain an identity from an earlier scope. + environment.setattr(impl_gcs, 'threadsafe_gcs', + impl_gcs.ThreadLocalGCSClient()) + yield def withGCSCredentials(func: Callable): - """Declare credentials without performing IO during pytest collection. - - The test setup hook resolves this prerequisite after skip selection and - before resource fixtures, including session workers, are initialized. - """ + """Declare credentials without performing IO during pytest collection.""" return pytest.mark.gcs_credentials(func) -def require_gcs_credentials() -> None: - if not _check_gcs_credentials(): - pytest.skip('No GCS credentials found') - - def withMockedGCSFilesystem(func: Callable): """Decorator to mock GCS filesystem for testing. diff --git a/test/conftest.py b/test/conftest.py index f1c10f8f7..bdae8389c 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -20,6 +20,7 @@ import time # noqa: E402 import uuid # noqa: E402 from collections.abc import Callable # noqa: E402 +from contextlib import ExitStack # noqa: E402 from test.fixtures.backend_cache import ( # noqa: E402, F401 auto_setup_kumo_backend, backend_cache, @@ -70,7 +71,12 @@ from kumo.graph.table import Table # noqa: E402 from kumo.task.task import Task # noqa: E402 from kumo.testing import withDatabricksCredentials # noqa: E402 -from kumo.testing.gcs import require_gcs_credentials # noqa: E402 +from kumo.testing.bigquery import ( # noqa: E402 + GCP_WORKERS_STARTED, + BigQueryIdentity, + setup_bigquery_credentials, +) +from kumo.testing.gcs import setup_gcs_credentials # noqa: E402 from kumo.testing.generator import EdgeSpec, GraphSpec, TableSpec # noqa: E402 from kumo.testing.snowflake import setup_snowflake_credentials # noqa: E402 from kumo.utils.embedding_models import embedding_model_store # noqa: E402 @@ -191,21 +197,55 @@ def pytest_runtest_makereport(item, call): _flake_diagnostics.run_on_failure(item.nodeid, tc) +_GCP_SESSION = pytest.StashKey[ExitStack]() +_GCP_TEST = pytest.StashKey[ExitStack]() + + def pytest_runtest_setup(item: pytest.Item) -> None: - # Pytest's tryfirst skip hook runs before this normal setup hook; its - # fixture setup runs afterward. Skipped Snowflake targets never load GCP - # credentials, and selected GCS tests configure them before session workers. - if item.get_closest_marker('gcs_credentials') is not None: - require_gcs_credentials() + # Normal setup runs after pytest's skip hook and before fixture setup. + # Keep the base identity until all session workers have been torn down. + marker = item.get_closest_marker('bigquery_credentials') + needs_gcs = (item.get_closest_marker('gcs_credentials') is not None + or 'gcs_credentials' in item.fixturenames) + if marker is None and not needs_gcs: + return + if _GCP_SESSION not in item.config.stash: + if item.config.stash.get(GCP_WORKERS_STARTED, False): + raise RuntimeError( + 'GCP tests cannot reuse workers started before credential setup; ' + 'run GCP integration modules in a separate pytest invocation') + session = ExitStack() + item.config.add_cleanup(session.close) + session.enter_context(setup_gcs_credentials()) + item.config.stash[_GCP_SESSION] = session + identity = BigQueryIdentity.DEFAULT + if marker is not None: + identity, = marker.args + scoped = ExitStack() + item.stash[_GCP_TEST] = scoped + scoped.enter_context(setup_gcs_credentials()) + scoped.enter_context(setup_bigquery_credentials(identity)) + + +@pytest.hookimpl(wrapper=True, tryfirst=True) +def pytest_runtest_teardown(item: pytest.Item): + try: + yield + finally: + if _GCP_TEST in item.stash: + item.stash[_GCP_TEST].close() + del item.stash[_GCP_TEST] @pytest.fixture def gcs_credentials() -> None: - require_gcs_credentials() + # The setup hook resolves fixture prerequisites before resource startup. + pass def pytest_configure(config): config.addinivalue_line('markers', 'gcs_credentials: live GCS prerequisite') + config.addinivalue_line('markers', 'bigquery_credentials(identity): BigQuery identity') # register nightly marker config.addinivalue_line("markers", ("nightly(name): Used to mark long-running" diff --git a/test/connector/test_bigquery_connector.py b/test/connector/test_bigquery_connector.py index 23a9eb21a..c70bf2645 100644 --- a/test/connector/test_bigquery_connector.py +++ b/test/connector/test_bigquery_connector.py @@ -27,11 +27,7 @@ SourceTableNotFoundError, ) from kumo.stats.compute_stats_python import compute_col_stats_python -from kumo.testing.bigquery import ( - FAIL_PROJECT_PERMISSION_BIGQUERY_KEY, - FAIL_TABLE_PERMISSION_BIGQUERY_KEY, - withBigQueryCredentials, -) +from kumo.testing.bigquery import BigQueryIdentity, withBigQueryCredentials def _write_df_to_bigquery(connector, table_name, df, replace=False): @@ -367,7 +363,7 @@ def test_set_credentials_and_open(): connector.open() -@withBigQueryCredentials(service_account_key=FAIL_TABLE_PERMISSION_BIGQUERY_KEY) +@withBigQueryCredentials(identity=BigQueryIdentity.NO_TABLE) def test_bigquery_table_permission_fail_read_table(): connector = BigQueryConnector( name='bigquery_fail', @@ -386,7 +382,7 @@ def test_bigquery_table_permission_fail_read_table(): @withBigQueryCredentials( - service_account_key=FAIL_PROJECT_PERMISSION_BIGQUERY_KEY) + identity=BigQueryIdentity.NO_JOB) def test_bigquery_project_permission_fail_read_table(): connector = BigQueryConnector( name='bigquery_fail', diff --git a/test/io_utils/fsutil/test_all.py b/test/io_utils/fsutil/test_all.py index e3a06744c..b65688c58 100644 --- a/test/io_utils/fsutil/test_all.py +++ b/test/io_utils/fsutil/test_all.py @@ -49,7 +49,7 @@ def root_ucv(temp_ucv) -> Generator[str, None, None]: @pytest.fixture( params=[ "root_s3", - "root_gcs", + pytest.param("root_gcs", marks=pytest.mark.gcs_credentials), "root_ucv", "root_adls", ], diff --git a/test/rest/integ/conftest.py b/test/rest/integ/conftest.py index 3c8589ca9..956a0a0d4 100644 --- a/test/rest/integ/conftest.py +++ b/test/rest/integ/conftest.py @@ -15,6 +15,7 @@ from io_utils import fsutil, logging from io_utils.json import load_json +from kumo.testing.bigquery import GCP_WORKERS_STARTED from monitoring.settings import MonitoringSettings from rest.backends.data_backend.context import DataBackendContext from rest.backends.ml_backend.ml_job_backend import MLJobBackend @@ -467,6 +468,7 @@ def _spawn(name: str, args: List[str], env_overrides: Dict[str, str]): env = {**os.environ, **env_overrides} log.info('Spawning %s: %s', name, ' '.join(args)) proc = subprocess.Popen(args, env=env) + request.config.stash[GCP_WORKERS_STARTED] = True procs.append(proc) _POOL_SUBPROCESSES_LIVE.append(proc) return proc diff --git a/test/rest/integ/test_bigquery_diskgraph_e2e.py b/test/rest/integ/test_bigquery_diskgraph_e2e.py index 749b8d11a..4e0b72418 100644 --- a/test/rest/integ/test_bigquery_diskgraph_e2e.py +++ b/test/rest/integ/test_bigquery_diskgraph_e2e.py @@ -15,7 +15,8 @@ - Temporal dev server running: make up-local-temporal - diskgraph Python package installed - DEPLOY_ENV_CONFIG_FILE pointing to a diskgraph deploy config - - BigQuery credentials available (via AWS Secrets Manager or env vars) + - GOOGLE_APPLICATION_CREDENTIALS pointing to an explicit local ADC file + (external_account credentials when workload exchange is enabled) - GCP_STORAGE_TRANSFER_BUCKET set to a writable GCS bucket Usage: diff --git a/test/rest/integ/test_bigquery_ingestion_diskgraph_e2e.py b/test/rest/integ/test_bigquery_ingestion_diskgraph_e2e.py index 4f92ba144..ea89cb839 100644 --- a/test/rest/integ/test_bigquery_ingestion_diskgraph_e2e.py +++ b/test/rest/integ/test_bigquery_ingestion_diskgraph_e2e.py @@ -52,10 +52,9 @@ - ``DEPLOY_ENV_CONFIG_FILE`` pointing at a diskgraph deploy config (CI sets it for the ``bigquery_*_diskgraph`` matrix entries; this module also defaults it at import time) - - BigQuery credentials available via the - ``withBigQueryCredentials`` decorator (from AWS Secrets - Manager in CI, ``GOOGLE_CREDENTIALS_STR_JSON`` fallback - otherwise) + - ``GOOGLE_APPLICATION_CREDENTIALS`` pointing to an explicit local ADC file; + ``external_account`` credentials are required when workload exchange is + enabled. - ``GCP_STORAGE_TRANSFER_BUCKET`` env var set to a writable GCS bucket (CI sets it to ``kumo-test-bucket``) diff --git a/test/testing/test_bigquery_credentials.py b/test/testing/test_bigquery_credentials.py index 668c2b0bb..52f5b9751 100644 --- a/test/testing/test_bigquery_credentials.py +++ b/test/testing/test_bigquery_credentials.py @@ -39,58 +39,10 @@ def test_invalid_bigquery_identity_remains_key_backed(): data_source_api.GoogleServiceAccountKeyAuthentication) -def test_setup_prefers_application_default_credentials(monkeypatch, tmp_path): - credentials_path = tmp_path / "gha-creds.json" - credentials_path.write_text('{"type": "external_account"}') - monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', str(credentials_path)) - monkeypatch.setenv('GOOGLE_CLOUD_PROJECT', 'example-project') - get_secrets = mock.Mock(side_effect=AssertionError( - "Workload Identity must not read a stored service-account key")) - monkeypatch.setattr(testing_bigquery, 'get_secrets', get_secrets) - - testing_bigquery.setup_bigquery_credentials() - - assert os.environ[bigquery_connector.ENV_GOOGLE_CREDENTIALS_JSON] == str( - credentials_path) - assert os.environ['GCP_PROJECT_ID'] == 'example-project' - assert testing_bigquery.ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT not in ( - os.environ) - get_secrets.assert_not_called() - - -def test_restricted_identity_uses_iam_impersonation(monkeypatch, tmp_path): - credentials_path = tmp_path / "gha-creds.json" - credentials_path.write_text('{"type": "external_account"}') - target = 'gha-kumo-bq-no-table@example-project.iam.gserviceaccount.com' - monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', str(credentials_path)) - monkeypatch.setenv(testing_bigquery.ENV_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT, - target) - - testing_bigquery.setup_bigquery_credentials( - testing_bigquery.FAIL_TABLE_PERMISSION_BIGQUERY_KEY) - - assert os.environ[ - testing_bigquery.ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT] == target - - -def test_restricted_identity_requires_target_variable(monkeypatch, tmp_path): - credentials_path = tmp_path / "gha-creds.json" - credentials_path.write_text('{"type": "external_account"}') - monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', str(credentials_path)) - monkeypatch.delenv( - testing_bigquery.ENV_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT, - raising=False, - ) - - with pytest.raises(RuntimeError, match='GCP_BIGQUERY_NO_TABLE'): - testing_bigquery.setup_bigquery_credentials( - testing_bigquery.FAIL_TABLE_PERMISSION_BIGQUERY_KEY) - - def test_connector_loads_external_account_and_impersonates( monkeypatch, tmp_path): credentials_path = tmp_path / "gha-creds.json" - credentials_path.write_text('{"type": "external_account"}') + fsutil.write_text(str(credentials_path), '{"type": "external_account"}') target = 'gha-kumo-bq-no-job@example-project.iam.gserviceaccount.com' source_credential = mock.Mock(name='source_credential') target_credential = mock.Mock(name='target_credential') @@ -145,22 +97,123 @@ def test_connector_loads_external_account_and_impersonates( } -@pytest.mark.parametrize('credential_type', [ - None, 'service_account', 'authorized_user', 'impersonated_service_account', +@pytest.fixture +def explicit_adc(monkeypatch, tmp_path): + path = str(tmp_path / 'adc.json') + fsutil.write_text(path, '{"type": "external_account"}') + monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', path) + monkeypatch.delenv('KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED', raising=False) + return path + + +@pytest.mark.parametrize('kind', list(testing_bigquery.CredentialKind)) +def test_explicit_types(monkeypatch, explicit_adc, kind): + fsutil.write_text(explicit_adc, json.dumps({'type': kind.value})) + monkeypatch.setenv(testing_bigquery.ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT, + 'previous') + with testing_bigquery.setup_bigquery_credentials(): + assert os.environ[bigquery_connector.ENV_GOOGLE_CREDENTIALS_JSON] == explicit_adc + assert testing_bigquery.ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT not in os.environ + assert os.environ[testing_bigquery.ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT] == 'previous' + + +@pytest.mark.parametrize('identity, target_var, key_var', [ + (testing_bigquery.BigQueryIdentity.NO_TABLE, + testing_bigquery.ENV_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT, + testing_bigquery.ENV_BIGQUERY_NO_TABLE_CREDENTIALS), + (testing_bigquery.BigQueryIdentity.NO_JOB, + testing_bigquery.ENV_BIGQUERY_NO_JOB_SERVICE_ACCOUNT, + testing_bigquery.ENV_BIGQUERY_NO_JOB_CREDENTIALS), ]) -def test_workload_mode_requires_federation(monkeypatch, tmp_path, - credential_type): +@pytest.mark.parametrize('kind', list(testing_bigquery.CredentialKind)) +def test_restricted_identity(monkeypatch, tmp_path, explicit_adc, identity, + target_var, key_var, kind): + fsutil.write_text(explicit_adc, json.dumps({'type': kind.value})) + restricted = str(tmp_path / 'restricted.json') + fsutil.write_text(restricted, '{"type": "service_account"}') + monkeypatch.setenv(key_var, restricted) + monkeypatch.setenv(target_var, 'restricted@example.com') + with testing_bigquery.setup_bigquery_credentials(identity): + if kind == testing_bigquery.CredentialKind.KEY: + assert os.environ['GOOGLE_APPLICATION_CREDENTIALS'] == restricted + assert testing_bigquery.ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT not in os.environ + else: + assert os.environ['GOOGLE_APPLICATION_CREDENTIALS'] == explicit_adc + assert os.environ[testing_bigquery.ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT] == 'restricted@example.com' + assert os.environ['GOOGLE_APPLICATION_CREDENTIALS'] == explicit_adc + assert fsutil.read_text(restricted) == '{"type": "service_account"}' + + +@pytest.mark.parametrize('document', ['not-json', '[]', '{}', '{"type":"unknown"}']) +def test_invalid_adc(explicit_adc, document): + fsutil.write_text(explicit_adc, document) + with pytest.raises(RuntimeError, match='invalid|unsupported'): + with testing_bigquery.setup_bigquery_credentials(): + pytest.fail('Invalid credentials accepted') + + +@pytest.mark.parametrize('kind', ['service_account', 'authorized_user', + 'impersonated_service_account']) +def test_workload_requires_federation(monkeypatch, explicit_adc, kind): monkeypatch.setenv('KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED', 'true') + fsutil.write_text(explicit_adc, json.dumps({'type': kind})) + with pytest.raises(RuntimeError, match='external_account'): + with testing_bigquery.setup_bigquery_credentials(): + pytest.fail('Non-federated workload credentials accepted') + + +def test_missing_adc_fails(monkeypatch): monkeypatch.delenv('GOOGLE_APPLICATION_CREDENTIALS', raising=False) - monkeypatch.setenv(bigquery_connector.ENV_GCS_KUMO_BUCKET.name, 'test-bucket') - if credential_type is not None: - adc = str(tmp_path / 'adc.json') - fsutil.write_text(adc, json.dumps({'type': credential_type})) - monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', adc) - get_secrets = mock.Mock(side_effect=AssertionError('No stored-key fallback')) - monkeypatch.setattr(testing_bigquery, 'get_secrets', get_secrets) - - with pytest.raises(RuntimeError, match='require'): - testing_bigquery.setup_bigquery_credentials() - - get_secrets.assert_not_called() + monkeypatch.setenv('GOOGLE_CREDENTIALS_STR_JSON', 'invalid-base64!') + with pytest.raises(RuntimeError, match='explicit'): + with testing_bigquery.setup_bigquery_credentials(): + pytest.fail('Implicit credentials accepted') + + +@pytest.mark.parametrize('failure', [RuntimeError, pytest.skip.Exception]) +def test_environment_restored(monkeypatch, explicit_adc, failure): + names = testing_bigquery._CREDENTIAL_ENV_VARS + monkeypatch.setenv('GCP_PROJECT_ID', 'previous-project') + monkeypatch.delenv(bigquery_connector.ENV_GOOGLE_CREDENTIALS_JSON, raising=False) + before = {name: os.environ.get(name) for name in names} + with pytest.raises(failure): + with testing_bigquery.setup_bigquery_credentials(): + for name in names: + os.environ[name] = 'changed-by-test' + raise failure('test failure') + assert {name: os.environ.get(name) for name in names} == before + assert fsutil.read_text(explicit_adc) == '{"type": "external_account"}' + + +@pytest.mark.parametrize('identity', [testing_bigquery.BigQueryIdentity.NO_TABLE, + testing_bigquery.BigQueryIdentity.NO_JOB]) +def test_restricted_requires_input(monkeypatch, explicit_adc, identity): + for variable in (testing_bigquery.ENV_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT, + testing_bigquery.ENV_BIGQUERY_NO_JOB_SERVICE_ACCOUNT): + monkeypatch.delenv(variable, raising=False) + before = dict(os.environ) + with pytest.raises(RuntimeError, match='restricted'): + with testing_bigquery.setup_bigquery_credentials(identity): + pytest.fail('Missing target accepted') + assert dict(os.environ) == before + + +@pytest.mark.parametrize('path', ['s3://bucket/key.json', 'gs://bucket/key.json']) +def test_remote_adc_rejected(monkeypatch, path): + monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', path) + with mock.patch.object(fsutil, 'exists', side_effect=AssertionError('Remote IO')): + with pytest.raises(RuntimeError, match='local ADC'): + testing_bigquery.resolve_credentials(testing_bigquery.BigQueryIdentity.DEFAULT) + + +@pytest.mark.parametrize('document', [None, '{"type":"authorized_user"}']) +def test_restricted_key_required(monkeypatch, tmp_path, explicit_adc, document): + fsutil.write_text(explicit_adc, '{"type":"service_account"}') + variable = testing_bigquery.ENV_BIGQUERY_NO_TABLE_CREDENTIALS + monkeypatch.delenv(variable, raising=False) + if document is not None: + restricted = str(tmp_path / 'restricted.json') + fsutil.write_text(restricted, document) + monkeypatch.setenv(variable, restricted) + with pytest.raises(RuntimeError, match=variable): + testing_bigquery.resolve_credentials(testing_bigquery.BigQueryIdentity.NO_TABLE) diff --git a/test/testing/test_gcs_credentials.py b/test/testing/test_gcs_credentials.py index 7abf64420..db9809cbd 100644 --- a/test/testing/test_gcs_credentials.py +++ b/test/testing/test_gcs_credentials.py @@ -3,10 +3,12 @@ # SPDX-License-Identifier: Apache-2.0 import os +from unittest import mock import pytest from io_utils import fsutil +from io_utils.fsutil import impl_gcs from kumo.connector.bigquery_connector import ENV_GOOGLE_CREDENTIALS_JSON from kumo.testing import gcs @@ -17,7 +19,7 @@ def test_decoration_never_configures(monkeypatch): def unexpected(): pytest.fail('Collection must not access credentials') - monkeypatch.setattr(gcs, '_check_gcs_credentials', unexpected) + monkeypatch.setattr(gcs, 'setup_bigquery_credentials', unexpected) @gcs.withGCSCredentials def example(): @@ -26,68 +28,156 @@ def example(): assert example.pytestmark[0].name == 'gcs_credentials' -def test_explicit_adc_wins(monkeypatch, tmp_path): +def test_explicit_adc_and_cache(monkeypatch, tmp_path): adc = str(tmp_path / 'adc.json') fsutil.write_text(adc, '{"type": "external_account"}') monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', adc) monkeypatch.setenv('GOOGLE_CREDENTIALS_STR_JSON', 'invalid-base64!') monkeypatch.setenv(ENV_GOOGLE_CREDENTIALS_JSON, 'previous') monkeypatch.setenv(gcs.ENV_GCS_TEST_BUCKET, 'test-bucket') - - gcs.require_gcs_credentials() - - assert os.environ[ENV_GOOGLE_CREDENTIALS_JSON] == adc + previous = impl_gcs.threadsafe_gcs + client = mock.Mock() + monkeypatch.setattr(impl_gcs.storage, 'Client', mock.Mock(return_value=client)) + + with gcs.setup_gcs_credentials(): + assert os.environ[ENV_GOOGLE_CREDENTIALS_JSON] == adc + assert impl_gcs.threadsafe_gcs is not previous + assert impl_gcs.threadsafe_gcs.client() is client + assert impl_gcs.threadsafe_gcs.client() is client + impl_gcs.storage.Client.assert_called_once() + os.environ[gcs.ENV_GCS_TEST_BUCKET] = 'changed' + + assert impl_gcs.threadsafe_gcs is previous + assert os.environ[gcs.ENV_GCS_TEST_BUCKET] == 'test-bucket' + assert os.environ[ENV_GOOGLE_CREDENTIALS_JSON] == 'previous' assert fsutil.read_text(adc) == '{"type": "external_account"}' -@pytest.mark.parametrize('skipped', [False, True]) -@pytest.mark.parametrize('available', [False, True]) -def test_prerequisite_precedes_resources(pytester, monkeypatch, skipped, - available): - calls = [] - - def credentials_available(): - calls.append('credentials') - if available: - monkeypatch.setenv('GCS_TEST_READY', '1') - return available - - monkeypatch.delenv('GCS_TEST_READY', raising=False) - monkeypatch.setattr(gcs, '_check_gcs_credentials', credentials_available) - # Exercise the real hook in an isolated pytest session, including pytest's - # built-in skip handling and session-resource ordering. - pytester.makeconftest(''' +@pytest.mark.parametrize('failure', [ + 'none', 'fixture', 'test', 'teardown', 'missing', 'skip', +]) +def test_lifecycle(pytester, monkeypatch, failure): + adc = str(pytester.path / 'adc.json') + fsutil.write_text(adc, '{"type": "external_account"}') + monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', adc) + monkeypatch.setenv('GOOGLE_CREDENTIALS_STR_JSON', 'invalid-base64!') + monkeypatch.setenv(ENV_GOOGLE_CREDENTIALS_JSON, 'original') + monkeypatch.setenv('GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT', 'restricted') + monkeypatch.setenv('GOOGLE_IMPERSONATE_SERVICE_ACCOUNT', 'original-target') + if failure == 'missing': + monkeypatch.delenv('GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT') + before = dict(os.environ) + pytester.makeconftest(f''' import os import pytest -from test.conftest import pytest_runtest_setup +from test.conftest import pytest_runtest_setup, pytest_runtest_teardown +from kumo.testing.bigquery import GCP_WORKERS_STARTED def pytest_configure(config): - config.addinivalue_line('markers', 'gcs_credentials: GCS prerequisite') + config.addinivalue_line('markers', 'bigquery_credentials(identity): identity') @pytest.fixture(scope='session') -def resource(): - assert os.environ.get('GCS_TEST_READY') == '1' +def worker(request): + assert os.environ['GOOGLE_CREDENTIALS_JSON'] == {adc!r} + request.config.stash[GCP_WORKERS_STARTED] = True + yield + assert os.environ['GOOGLE_CREDENTIALS_JSON'] == {adc!r} + +@pytest.fixture +def resource(worker): + assert os.environ['GOOGLE_IMPERSONATE_SERVICE_ACCOUNT'] == 'restricted' + if {failure!r} == 'fixture': + raise RuntimeError('fixture failure') yield + assert os.environ['GOOGLE_IMPERSONATE_SERVICE_ACCOUNT'] == 'restricted' + if {failure!r} == 'teardown': + raise RuntimeError('teardown failure') +''') + pytester.makepyfile(f''' +import os +import pytest +from kumo.testing.bigquery import BigQueryIdentity, withBigQueryCredentials + +@pytest.mark.skipif({failure!r} == 'skip', reason='Snowflake mode') +@withBigQueryCredentials(identity=BigQueryIdentity.NO_TABLE) +def test_live(resource): + if {failure!r} == 'test': + raise RuntimeError('test failure') + + +def test_next(): + expected = 'original-target' if {failure!r} == 'skip' else None + assert os.environ.get('GOOGLE_IMPERSONATE_SERVICE_ACCOUNT') == expected +''') + result = pytester.runpytest_inprocess( + '-q', '-o', 'addopts=', '--confcutdir', str(pytester.path)) + if failure in ('fixture', 'missing'): + result.assert_outcomes(passed=1, errors=1) + elif failure == 'teardown': + result.assert_outcomes(passed=2, errors=1) + elif failure == 'test': + result.assert_outcomes(passed=1, failed=1) + elif failure == 'skip': + result.assert_outcomes(passed=1, skipped=1) + else: + result.assert_outcomes(passed=2) + for variable in ('GOOGLE_APPLICATION_CREDENTIALS', ENV_GOOGLE_CREDENTIALS_JSON, + 'GOOGLE_IMPERSONATE_SERVICE_ACCOUNT', 'GCP_PROJECT_ID', + gcs.ENV_GCS_TEST_BUCKET): + assert os.environ.get(variable) == before.get(variable) + + +@pytest.mark.parametrize('skipped', [False, True]) +def test_existing_workers_rejected(pytester, monkeypatch, skipped): + adc = str(pytester.path / 'adc.json') + fsutil.write_text(adc, '{"type": "external_account"}') + monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', adc) + pytester.makeconftest(''' +import pytest +from test.conftest import pytest_runtest_setup, pytest_runtest_teardown +from kumo.testing.bigquery import GCP_WORKERS_STARTED + +def pytest_configure(config): + config.addinivalue_line('markers', 'gcs_credentials: identity') + config.stash[GCP_WORKERS_STARTED] = True ''') pytester.makepyfile(f''' import pytest from kumo.testing.gcs import withGCSCredentials -@pytest.mark.skipif({skipped!r}, reason='Snowflake mode') +@pytest.mark.skipif({skipped!r}, reason='Snowflake') @withGCSCredentials -def test_live(resource): +def test_late_gcp(): pass ''') - result = pytester.runpytest_inprocess( '-q', '-o', 'addopts=', '--confcutdir', str(pytester.path)) - if skipped: result.assert_outcomes(skipped=1) - assert calls == [] - elif available: - result.assert_outcomes(passed=1) - assert calls == ['credentials'] else: - result.assert_outcomes(skipped=1) - assert calls == ['credentials'] + result.assert_outcomes(errors=1) + result.stdout.fnmatch_lines(['*separate pytest invocation*']) + + +def test_mixed_filesystem_marker(pytester): + pytester.makeconftest(''' +def pytest_configure(config): + config.addinivalue_line('markers', 'gcs_credentials: identity') + +def pytest_collection_modifyitems(items): + marked = [item for item in items + if item.get_closest_marker('gcs_credentials') is not None] + assert len(marked) == 1 + assert marked[0].name == 'test_mixed[gcs]' + assert len(items) == 4 +''') + pytester.makepyfile(''' +from test.io_utils.fsutil.test_all import root + +def test_mixed(root): + pass +''') + result = pytester.runpytest_inprocess( + '--collect-only', '-q', '-o', 'addopts=', + '--confcutdir', str(pytester.path)) + assert result.ret == 0