From 0aa38a366d501b52923169d3bfede58e102dab2d Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Thu, 5 Nov 2026 12:46:07 -0600 Subject: [PATCH 1/3] Make GCP test credentials explicit and scoped --- docs/development/testing.md | 29 +++ kumo/testing/bigquery.py | 286 ++++++++++------------ kumo/testing/gcs.py | 123 ++-------- test/conftest.py | 54 +++- test/connector/test_bigquery_connector.py | 10 +- test/io_utils/fsutil/test_all.py | 2 +- test/rest/integ/conftest.py | 2 + test/testing/test_bigquery_credentials.py | 183 +++++++++----- test/testing/test_gcs_credentials.py | 164 ++++++++++--- 9 files changed, 470 insertions(+), 383 deletions(-) diff --git a/docs/development/testing.md b/docs/development/testing.md index 7833b87e5..64097a66b 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -86,3 +86,32 @@ The [Engine PR guide](engine-pr-ci.md#local-feedback) gives one pytest command for the small pure output and workflow checks. The existing OSS Readiness job runs the same tests. They need no GPU or cloud access. The GPU job runs the eight actual training and save/load cases. + +## Live Google Cloud tests + +Selected BigQuery and GCS tests require an explicit +`GOOGLE_APPLICATION_CREDENTIALS` JSON file. Local runs accept `service_account`, +`authorized_user`, `impersonated_service_account`, and `external_account` ADC. +With `KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED=true`, only `external_account` is +accepted. Missing or invalid inputs fail setup. Tests do not fetch Google keys +from AWS Secrets Manager, decode `GOOGLE_CREDENTIALS_STR_JSON`, discover home +credentials, or write credential files. Existing input files remain unchanged. + +Restricted BigQuery permission tests use `GCP_BIGQUERY_NO_TABLE_CREDENTIALS` +and `GCP_BIGQUERY_NO_JOB_CREDENTIALS` for separate service-account key files. +Other ADC types use `GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT` and +`GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT` impersonation targets. The source identity +must be allowed to impersonate those deliberately restricted accounts. + +The decorators declare credentials without collection-time IO. Selected tests +prepare the base identity before fixtures start, scope restricted identities +through fixture teardown, and restore the original environment and cached GCS +client after the session's workers stop. Skipped Snowflake tests do not resolve +Google credentials. No credentials are required for collection alone. + +Run GCP integration modules in a separate pytest invocation from non-GCP +DiskGraph integration modules. If pool workers have already started without the +GCP test context, later GCP tests fail with an explicit instruction rather than +silently reusing workers with stale credentials. CI runs each integration module +in its own process. Shared AWS, Azure, and Databricks environment isolation is +outside this credential-helper change. diff --git a/kumo/testing/bigquery.py b/kumo/testing/bigquery.py index 82854aaf9..0301cb6bc 100644 --- a/kumo/testing/bigquery.py +++ b/kumo/testing/bigquery.py @@ -2,174 +2,144 @@ # All rights reserved. # SPDX-License-Identifier: Apache-2.0 -import asyncio -import functools import json import os -from collections.abc import Callable -from typing import Optional +from collections.abc import Callable, Generator +from contextlib import contextmanager +from enum import Enum -from io_utils import fsutil, logging -from io_utils.secrets import get_secrets +import pytest + +from io_utils import fsutil from kumo.connector.bigquery_connector import ( ENV_GCS_KUMO_BUCKET, - ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH, ENV_GOOGLE_CREDENTIALS_JSON, ) from service_framework.env_var import EnvVar -logger = logging.getLogger(__name__) - -DEFAULT_BIGQUERY_KEY = "kumo-test-bigquery-service-account-key" - -# Missing required table read permission -FAIL_TABLE_PERMISSION_BIGQUERY_KEY = "kumo-test-permission-fail-bigquery-service-account-key-2" -# Missing required jobs.create permission -FAIL_PROJECT_PERMISSION_BIGQUERY_KEY = "kumo-test-permission-fail-bigquery-service-account-key" - -# Keyless CI maps the two legacy negative-test identities to deliberately -# restricted service accounts. The names remain keyed by the old secret -# identifiers so callers do not need to know how credentials are sourced. -ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT = "GOOGLE_IMPERSONATE_SERVICE_ACCOUNT" -ENV_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT = ( - "GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT") -ENV_BIGQUERY_NO_JOB_SERVICE_ACCOUNT = "GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT" - -_IMPERSONATION_ENV_BY_KEY = { - FAIL_TABLE_PERMISSION_BIGQUERY_KEY: ENV_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT, - FAIL_PROJECT_PERMISSION_BIGQUERY_KEY: ENV_BIGQUERY_NO_JOB_SERVICE_ACCOUNT, -} - - -def _configure_application_default_credentials( - service_account_key: str) -> bool: - """Configure an existing ADC file for BigQuery connector tests. - - GitHub's OIDC authentication action writes an ``external_account`` ADC - file and exports ``GOOGLE_APPLICATION_CREDENTIALS``. Negative permission - tests select a restricted service account through IAM impersonation rather - than loading a second private key. - """ - require_wif = EnvVar('KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED', bool, - default_value=False).parse_value() - credentials_path = os.getenv('GOOGLE_APPLICATION_CREDENTIALS') - if not credentials_path or not fsutil.exists(credentials_path): - if require_wif: - raise RuntimeError( - 'Workload exchange BigQuery tests require federated ADC in ' - 'GOOGLE_APPLICATION_CREDENTIALS') - return False - - if require_wif: - credentials = json.loads(fsutil.read_text(credentials_path)) - if credentials.get('type') != 'external_account': - raise RuntimeError( - 'Workload exchange BigQuery tests require external_account ADC') - - impersonation_env = _IMPERSONATION_ENV_BY_KEY.get(service_account_key) - if impersonation_env is None: - os.environ.pop(ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT, None) - else: - target = os.getenv(impersonation_env) - if not target: - raise RuntimeError( - f"{impersonation_env} must be configured for the restricted " - "BigQuery permission test identity") - os.environ[ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT] = target - - os.environ[ENV_GOOGLE_CREDENTIALS_JSON] = credentials_path - project_id = (os.getenv('GCP_PROJECT_ID') - or os.getenv('GOOGLE_CLOUD_PROJECT')) - if project_id: - os.environ['GCP_PROJECT_ID'] = project_id - logger.info("Using Application Default Credentials for BigQuery tests") - return True - - -def setup_bigquery_credentials(service_account_key: Optional[str] = None): - # Set env var for GCS kumo bucket for testing. - os.environ[ENV_GCS_KUMO_BUCKET.name] = 'kumo-test-bucket' - service_account_key = service_account_key or DEFAULT_BIGQUERY_KEY - - # Prefer short-lived ADC supplied by Workload Identity Federation. This - # keeps GitHub Actions from reading any stored GCP service-account key. - if _configure_application_default_credentials(service_account_key): - return +ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT = 'GOOGLE_IMPERSONATE_SERVICE_ACCOUNT' +ENV_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT = 'GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT' +ENV_BIGQUERY_NO_JOB_SERVICE_ACCOUNT = 'GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT' +ENV_BIGQUERY_NO_TABLE_CREDENTIALS = 'GCP_BIGQUERY_NO_TABLE_CREDENTIALS' +ENV_BIGQUERY_NO_JOB_CREDENTIALS = 'GCP_BIGQUERY_NO_JOB_CREDENTIALS' +GCP_WORKERS_STARTED = pytest.StashKey[bool]() - try: - # Get the GCloud secrets from AWS secrets manager and store it in the - # default file. This remains a local/deployment fallback; CI supplies - # ADC above and never reaches this path. - secrets = get_secrets(service_account_key) - file_path = os.path.join(os.getcwd(), ".google_credential.json") - with open(file_path, 'w') as f: - json.dump(secrets, f) - - with open(ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH, 'w') as f: - # This is a hack. allows big query connector - # to fall back to default credentials - json.dump(secrets, f) - os.environ[ENV_GOOGLE_CREDENTIALS_JSON] = file_path - # Also set the standard ADC env var so any google-cloud-* client - # built without explicit credentials (e.g. fsutil's GCS client - # used by integration tests to verify EXPORT DATA output) picks - # up the same service account. - os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = file_path - except Exception as e: - # Use built-in credentials if this env doesn't have access to read the - # secrets from AWS - logger.exception(e) - - -def withBigQueryCredentials(func: Optional[Callable] = None, - service_account_key: Optional[str] = None): - import pytest +_CREDENTIAL_ENV_VARS = ( + 'GOOGLE_APPLICATION_CREDENTIALS', + ENV_GOOGLE_CREDENTIALS_JSON, + ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT, + ENV_GCS_KUMO_BUCKET.name, + 'GCP_PROJECT_ID', +) - def decorator(inner_func: Callable): - def _setup(): - setup_bigquery_credentials(service_account_key) - if (not os.getenv(ENV_GOOGLE_CREDENTIALS_JSON) and - not os.path.exists(ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH)): - pytest.skip("No BigQuery credentials found") - - if asyncio.iscoroutinefunction(inner_func): - - async def async_wrapper(*args, **kwargs): - previous_target = os.environ.get( - ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT) - try: - _setup() - return await inner_func(*args, **kwargs) - finally: - if previous_target is None: - os.environ.pop(ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT, - None) - else: - os.environ[ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT] = ( - previous_target) - - wrapper = async_wrapper + +class BigQueryIdentity(Enum): + DEFAULT = 'default' + NO_TABLE = 'no_table' + NO_JOB = 'no_job' + + +class CredentialKind(Enum): + FEDERATED = 'external_account' + USER = 'authorized_user' + IMPERSONATED = 'impersonated_service_account' + KEY = 'service_account' + + +def credential_kind(path: str) -> CredentialKind: + if '://' in path: + raise RuntimeError('Google test credentials require a local ADC file') + if not fsutil.exists(path) or fsutil.isdir(path): + raise RuntimeError('Configured BigQuery credential file does not exist') + try: + document = json.loads(fsutil.read_text(path)) + if not isinstance(document, dict): + raise ValueError('credential document must be a JSON object') + return CredentialKind(document.get('type')) + except (ValueError, TypeError) as error: + raise RuntimeError('BigQuery credential file has an invalid or ' + 'unsupported credential type') from error + + +def resolve_credentials(identity: BigQueryIdentity) -> tuple[str, str | None]: + """Resolve explicit inputs without changing the environment or files.""" + if not isinstance(identity, BigQueryIdentity): + raise TypeError('BigQuery credentials require a BigQueryIdentity') + path = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS') + if not path: + raise RuntimeError('BigQuery tests require explicit ' + 'GOOGLE_APPLICATION_CREDENTIALS') + kind = credential_kind(path) + require_wif = EnvVar( + 'KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED', bool, default_value=False).parse_value() + if require_wif and kind != CredentialKind.FEDERATED: + raise RuntimeError('Workload exchange BigQuery tests require ' + 'external_account ADC') + if identity == BigQueryIdentity.DEFAULT: + return path, None + + if kind == CredentialKind.KEY: + variable = { + BigQueryIdentity.NO_TABLE: ENV_BIGQUERY_NO_TABLE_CREDENTIALS, + BigQueryIdentity.NO_JOB: ENV_BIGQUERY_NO_JOB_CREDENTIALS, + }[identity] + restricted_path = os.environ.get(variable) + if not restricted_path: + raise RuntimeError(f'{variable} must identify the restricted ' + 'BigQuery service-account credential file') + if credential_kind(restricted_path) != CredentialKind.KEY: + raise RuntimeError(f'{variable} must contain service_account ' + 'credentials') + return restricted_path, None + + variable = { + BigQueryIdentity.NO_TABLE: ENV_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT, + BigQueryIdentity.NO_JOB: ENV_BIGQUERY_NO_JOB_SERVICE_ACCOUNT, + }[identity] + target = os.environ.get(variable) + if not target: + raise RuntimeError(f'{variable} must be configured for ' + 'the restricted BigQuery test identity') + return path, target + + +@contextmanager +def setup_bigquery_credentials( + identity: BigQueryIdentity = BigQueryIdentity.DEFAULT, +) -> Generator[None, None, None]: + """Scope explicit ADC to a test or session; input files remain read-only.""" + path, target = resolve_credentials(identity) + with pytest.MonkeyPatch.context() as environment: + for name in _CREDENTIAL_ENV_VARS: + previous = os.environ.get(name) + environment.setenv(name, previous or '') + if previous is None: + environment.delenv(name) + + environment.setenv('GOOGLE_APPLICATION_CREDENTIALS', path) + environment.setenv(ENV_GOOGLE_CREDENTIALS_JSON, path) + if target: + environment.setenv(ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT, target) else: + environment.delenv(ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT, + raising=False) + environment.setenv( + ENV_GCS_KUMO_BUCKET.name, + os.environ.get(ENV_GCS_KUMO_BUCKET.name) or 'kumo-test-bucket') + project = (os.environ.get('GCP_PROJECT_ID') + or os.environ.get('GOOGLE_CLOUD_PROJECT')) + if project: + environment.setenv('GCP_PROJECT_ID', project) + yield + + +def withBigQueryCredentials( + func: Callable | None = None, + *, + identity: BigQueryIdentity = BigQueryIdentity.DEFAULT, +): + """Declare credentials without performing auth or wrapping the test body.""" + def decorator(inner_func: Callable): + return pytest.mark.bigquery_credentials(identity)(inner_func) - def sync_wrapper(*args, **kwargs): - previous_target = os.environ.get( - ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT) - try: - _setup() - return inner_func(*args, **kwargs) - finally: - if previous_target is None: - os.environ.pop(ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT, - None) - else: - os.environ[ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT] = ( - previous_target) - - wrapper = sync_wrapper - - return functools.wraps(inner_func)(wrapper) - - if func is None: - return decorator - else: - return decorator(func) + return decorator if func is None else decorator(func) diff --git a/kumo/testing/gcs.py b/kumo/testing/gcs.py index 3d0846bee..c18e330d7 100644 --- a/kumo/testing/gcs.py +++ b/kumo/testing/gcs.py @@ -2,130 +2,37 @@ # All rights reserved. # SPDX-License-Identifier: Apache-2.0 -import base64 -import json import os -from collections.abc import Callable +from collections.abc import Callable, Generator +from contextlib import contextmanager from unittest import mock import pytest from google.cloud import storage -from io_utils import logging from io_utils.fsutil import impl_gcs -from kumo.connector.bigquery_connector import ( - ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH, - ENV_GOOGLE_CREDENTIALS_JSON, -) +from kumo.testing.bigquery import setup_bigquery_credentials -logger = logging.getLogger(__name__) +ENV_GCS_TEST_BUCKET = 'GCS_TEST_BUCKET' -# Environment variable for GCS test bucket -ENV_GCS_TEST_BUCKET = "GCS_TEST_BUCKET" -# Default Application Default Credentials path -ADC_DEFAULT_PATH = os.path.join(os.path.expanduser("~"), ".config", "gcloud", - "application_default_credentials.json") - - -def setup_gcs_credentials(): - """Set up GCS credentials for testing. - - This function fetches Google Cloud credentials from the - GOOGLE_CREDENTIALS_STR_JSON environment variable and sets up the necessary - environment variables for GCS testing. - - The credentials should be base64 encoded to avoid issues with special - characters in the JSON string. - """ - - if ENV_GCS_TEST_BUCKET not in os.environ: - os.environ[ENV_GCS_TEST_BUCKET] = 'kumo-test-bucket' - # CI supplies short-lived ADC. Never replace it with a stored key, and - # reuse the selected credentials when another test requests this setup. - credentials_path = os.getenv('GOOGLE_APPLICATION_CREDENTIALS') - if credentials_path: - os.environ[ENV_GOOGLE_CREDENTIALS_JSON] = credentials_path - return - google_creds = os.getenv("GOOGLE_CREDENTIALS_STR_JSON") - if google_creds: - logger.info("Using GOOGLE_CREDENTIALS_STR_JSON environment variable") - google_creds = base64.b64decode(google_creds) - google_creds = google_creds.decode('utf-8') - - file_path = os.path.join(os.getcwd(), ".google_credential.json") - # Parse the JSON string to a dictionary first - try: - google_creds_dict = json.loads(google_creds) - with open(file_path, 'w') as f: - json.dump(google_creds_dict, f) - - with open(ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH, 'w') as f: - json.dump(google_creds_dict, f) - - # Extract and set GCP_PROJECT_ID from service account credentials - # This is needed for GCPSecretProvider initialization - if 'project_id' in google_creds_dict: - os.environ['GCP_PROJECT_ID'] = google_creds_dict['project_id'] - logger.info(f"Set GCP_PROJECT_ID from credentials: " - f"{google_creds_dict['project_id']}") - except json.JSONDecodeError as e: - logger.error(f"Failed to parse GOOGLE_CREDENTIALS_STR_JSON: {e}") - raise - os.environ[ENV_GOOGLE_CREDENTIALS_JSON] = file_path - os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = file_path - else: - logger.warning( - "GOOGLE_CREDENTIALS_STR_JSON environment variable not set. " - "Relying on existing GOOGLE_APPLICATION_CREDENTIALS.") - - -def _check_gcs_credentials() -> bool: - """Check if GCS credentials are available. - - Checks for credentials in the following order: - 1. GOOGLE_APPLICATION_CREDENTIALS env var - 2. ~/.google_credentials.json (ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH) - 3. ~/.config/gcloud/application_default_credentials.json (ADC default) - - If ADC exists but GOOGLE_APPLICATION_CREDENTIALS is not set, it will be - set automatically for the google-cloud libraries to use. - - Returns: - bool: True if credentials are available, False otherwise - """ - setup_gcs_credentials() - - # Check if credentials are already set - if os.getenv('GOOGLE_APPLICATION_CREDENTIALS') is not None: - return True - - if os.path.exists(ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH): - return True - - # Check for ADC default path and set env var if found - if os.path.exists(ADC_DEFAULT_PATH): - logger.info(f"Using ADC credentials from {ADC_DEFAULT_PATH}") - os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = ADC_DEFAULT_PATH - return True - - return False +@contextmanager +def setup_gcs_credentials() -> Generator[None, None, None]: + """Scope explicit ADC without discovering or writing credential files.""" + with setup_bigquery_credentials(), pytest.MonkeyPatch.context() as environment: + environment.setenv(ENV_GCS_TEST_BUCKET, + os.environ.get(ENV_GCS_TEST_BUCKET) or 'kumo-test-bucket') + # Cached clients otherwise retain an identity from an earlier scope. + environment.setattr(impl_gcs, 'threadsafe_gcs', + impl_gcs.ThreadLocalGCSClient()) + yield def withGCSCredentials(func: Callable): - """Declare credentials without performing IO during pytest collection. - - The test setup hook resolves this prerequisite after skip selection and - before resource fixtures, including session workers, are initialized. - """ + """Declare credentials without performing IO during pytest collection.""" return pytest.mark.gcs_credentials(func) -def require_gcs_credentials() -> None: - if not _check_gcs_credentials(): - pytest.skip('No GCS credentials found') - - def withMockedGCSFilesystem(func: Callable): """Decorator to mock GCS filesystem for testing. diff --git a/test/conftest.py b/test/conftest.py index f1c10f8f7..bdae8389c 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -20,6 +20,7 @@ import time # noqa: E402 import uuid # noqa: E402 from collections.abc import Callable # noqa: E402 +from contextlib import ExitStack # noqa: E402 from test.fixtures.backend_cache import ( # noqa: E402, F401 auto_setup_kumo_backend, backend_cache, @@ -70,7 +71,12 @@ from kumo.graph.table import Table # noqa: E402 from kumo.task.task import Task # noqa: E402 from kumo.testing import withDatabricksCredentials # noqa: E402 -from kumo.testing.gcs import require_gcs_credentials # noqa: E402 +from kumo.testing.bigquery import ( # noqa: E402 + GCP_WORKERS_STARTED, + BigQueryIdentity, + setup_bigquery_credentials, +) +from kumo.testing.gcs import setup_gcs_credentials # noqa: E402 from kumo.testing.generator import EdgeSpec, GraphSpec, TableSpec # noqa: E402 from kumo.testing.snowflake import setup_snowflake_credentials # noqa: E402 from kumo.utils.embedding_models import embedding_model_store # noqa: E402 @@ -191,21 +197,55 @@ def pytest_runtest_makereport(item, call): _flake_diagnostics.run_on_failure(item.nodeid, tc) +_GCP_SESSION = pytest.StashKey[ExitStack]() +_GCP_TEST = pytest.StashKey[ExitStack]() + + def pytest_runtest_setup(item: pytest.Item) -> None: - # Pytest's tryfirst skip hook runs before this normal setup hook; its - # fixture setup runs afterward. Skipped Snowflake targets never load GCP - # credentials, and selected GCS tests configure them before session workers. - if item.get_closest_marker('gcs_credentials') is not None: - require_gcs_credentials() + # Normal setup runs after pytest's skip hook and before fixture setup. + # Keep the base identity until all session workers have been torn down. + marker = item.get_closest_marker('bigquery_credentials') + needs_gcs = (item.get_closest_marker('gcs_credentials') is not None + or 'gcs_credentials' in item.fixturenames) + if marker is None and not needs_gcs: + return + if _GCP_SESSION not in item.config.stash: + if item.config.stash.get(GCP_WORKERS_STARTED, False): + raise RuntimeError( + 'GCP tests cannot reuse workers started before credential setup; ' + 'run GCP integration modules in a separate pytest invocation') + session = ExitStack() + item.config.add_cleanup(session.close) + session.enter_context(setup_gcs_credentials()) + item.config.stash[_GCP_SESSION] = session + identity = BigQueryIdentity.DEFAULT + if marker is not None: + identity, = marker.args + scoped = ExitStack() + item.stash[_GCP_TEST] = scoped + scoped.enter_context(setup_gcs_credentials()) + scoped.enter_context(setup_bigquery_credentials(identity)) + + +@pytest.hookimpl(wrapper=True, tryfirst=True) +def pytest_runtest_teardown(item: pytest.Item): + try: + yield + finally: + if _GCP_TEST in item.stash: + item.stash[_GCP_TEST].close() + del item.stash[_GCP_TEST] @pytest.fixture def gcs_credentials() -> None: - require_gcs_credentials() + # The setup hook resolves fixture prerequisites before resource startup. + pass def pytest_configure(config): config.addinivalue_line('markers', 'gcs_credentials: live GCS prerequisite') + config.addinivalue_line('markers', 'bigquery_credentials(identity): BigQuery identity') # register nightly marker config.addinivalue_line("markers", ("nightly(name): Used to mark long-running" diff --git a/test/connector/test_bigquery_connector.py b/test/connector/test_bigquery_connector.py index 23a9eb21a..c70bf2645 100644 --- a/test/connector/test_bigquery_connector.py +++ b/test/connector/test_bigquery_connector.py @@ -27,11 +27,7 @@ SourceTableNotFoundError, ) from kumo.stats.compute_stats_python import compute_col_stats_python -from kumo.testing.bigquery import ( - FAIL_PROJECT_PERMISSION_BIGQUERY_KEY, - FAIL_TABLE_PERMISSION_BIGQUERY_KEY, - withBigQueryCredentials, -) +from kumo.testing.bigquery import BigQueryIdentity, withBigQueryCredentials def _write_df_to_bigquery(connector, table_name, df, replace=False): @@ -367,7 +363,7 @@ def test_set_credentials_and_open(): connector.open() -@withBigQueryCredentials(service_account_key=FAIL_TABLE_PERMISSION_BIGQUERY_KEY) +@withBigQueryCredentials(identity=BigQueryIdentity.NO_TABLE) def test_bigquery_table_permission_fail_read_table(): connector = BigQueryConnector( name='bigquery_fail', @@ -386,7 +382,7 @@ def test_bigquery_table_permission_fail_read_table(): @withBigQueryCredentials( - service_account_key=FAIL_PROJECT_PERMISSION_BIGQUERY_KEY) + identity=BigQueryIdentity.NO_JOB) def test_bigquery_project_permission_fail_read_table(): connector = BigQueryConnector( name='bigquery_fail', diff --git a/test/io_utils/fsutil/test_all.py b/test/io_utils/fsutil/test_all.py index e3a06744c..b65688c58 100644 --- a/test/io_utils/fsutil/test_all.py +++ b/test/io_utils/fsutil/test_all.py @@ -49,7 +49,7 @@ def root_ucv(temp_ucv) -> Generator[str, None, None]: @pytest.fixture( params=[ "root_s3", - "root_gcs", + pytest.param("root_gcs", marks=pytest.mark.gcs_credentials), "root_ucv", "root_adls", ], diff --git a/test/rest/integ/conftest.py b/test/rest/integ/conftest.py index 3c8589ca9..956a0a0d4 100644 --- a/test/rest/integ/conftest.py +++ b/test/rest/integ/conftest.py @@ -15,6 +15,7 @@ from io_utils import fsutil, logging from io_utils.json import load_json +from kumo.testing.bigquery import GCP_WORKERS_STARTED from monitoring.settings import MonitoringSettings from rest.backends.data_backend.context import DataBackendContext from rest.backends.ml_backend.ml_job_backend import MLJobBackend @@ -467,6 +468,7 @@ def _spawn(name: str, args: List[str], env_overrides: Dict[str, str]): env = {**os.environ, **env_overrides} log.info('Spawning %s: %s', name, ' '.join(args)) proc = subprocess.Popen(args, env=env) + request.config.stash[GCP_WORKERS_STARTED] = True procs.append(proc) _POOL_SUBPROCESSES_LIVE.append(proc) return proc diff --git a/test/testing/test_bigquery_credentials.py b/test/testing/test_bigquery_credentials.py index 668c2b0bb..52f5b9751 100644 --- a/test/testing/test_bigquery_credentials.py +++ b/test/testing/test_bigquery_credentials.py @@ -39,58 +39,10 @@ def test_invalid_bigquery_identity_remains_key_backed(): data_source_api.GoogleServiceAccountKeyAuthentication) -def test_setup_prefers_application_default_credentials(monkeypatch, tmp_path): - credentials_path = tmp_path / "gha-creds.json" - credentials_path.write_text('{"type": "external_account"}') - monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', str(credentials_path)) - monkeypatch.setenv('GOOGLE_CLOUD_PROJECT', 'example-project') - get_secrets = mock.Mock(side_effect=AssertionError( - "Workload Identity must not read a stored service-account key")) - monkeypatch.setattr(testing_bigquery, 'get_secrets', get_secrets) - - testing_bigquery.setup_bigquery_credentials() - - assert os.environ[bigquery_connector.ENV_GOOGLE_CREDENTIALS_JSON] == str( - credentials_path) - assert os.environ['GCP_PROJECT_ID'] == 'example-project' - assert testing_bigquery.ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT not in ( - os.environ) - get_secrets.assert_not_called() - - -def test_restricted_identity_uses_iam_impersonation(monkeypatch, tmp_path): - credentials_path = tmp_path / "gha-creds.json" - credentials_path.write_text('{"type": "external_account"}') - target = 'gha-kumo-bq-no-table@example-project.iam.gserviceaccount.com' - monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', str(credentials_path)) - monkeypatch.setenv(testing_bigquery.ENV_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT, - target) - - testing_bigquery.setup_bigquery_credentials( - testing_bigquery.FAIL_TABLE_PERMISSION_BIGQUERY_KEY) - - assert os.environ[ - testing_bigquery.ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT] == target - - -def test_restricted_identity_requires_target_variable(monkeypatch, tmp_path): - credentials_path = tmp_path / "gha-creds.json" - credentials_path.write_text('{"type": "external_account"}') - monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', str(credentials_path)) - monkeypatch.delenv( - testing_bigquery.ENV_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT, - raising=False, - ) - - with pytest.raises(RuntimeError, match='GCP_BIGQUERY_NO_TABLE'): - testing_bigquery.setup_bigquery_credentials( - testing_bigquery.FAIL_TABLE_PERMISSION_BIGQUERY_KEY) - - def test_connector_loads_external_account_and_impersonates( monkeypatch, tmp_path): credentials_path = tmp_path / "gha-creds.json" - credentials_path.write_text('{"type": "external_account"}') + fsutil.write_text(str(credentials_path), '{"type": "external_account"}') target = 'gha-kumo-bq-no-job@example-project.iam.gserviceaccount.com' source_credential = mock.Mock(name='source_credential') target_credential = mock.Mock(name='target_credential') @@ -145,22 +97,123 @@ def test_connector_loads_external_account_and_impersonates( } -@pytest.mark.parametrize('credential_type', [ - None, 'service_account', 'authorized_user', 'impersonated_service_account', +@pytest.fixture +def explicit_adc(monkeypatch, tmp_path): + path = str(tmp_path / 'adc.json') + fsutil.write_text(path, '{"type": "external_account"}') + monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', path) + monkeypatch.delenv('KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED', raising=False) + return path + + +@pytest.mark.parametrize('kind', list(testing_bigquery.CredentialKind)) +def test_explicit_types(monkeypatch, explicit_adc, kind): + fsutil.write_text(explicit_adc, json.dumps({'type': kind.value})) + monkeypatch.setenv(testing_bigquery.ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT, + 'previous') + with testing_bigquery.setup_bigquery_credentials(): + assert os.environ[bigquery_connector.ENV_GOOGLE_CREDENTIALS_JSON] == explicit_adc + assert testing_bigquery.ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT not in os.environ + assert os.environ[testing_bigquery.ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT] == 'previous' + + +@pytest.mark.parametrize('identity, target_var, key_var', [ + (testing_bigquery.BigQueryIdentity.NO_TABLE, + testing_bigquery.ENV_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT, + testing_bigquery.ENV_BIGQUERY_NO_TABLE_CREDENTIALS), + (testing_bigquery.BigQueryIdentity.NO_JOB, + testing_bigquery.ENV_BIGQUERY_NO_JOB_SERVICE_ACCOUNT, + testing_bigquery.ENV_BIGQUERY_NO_JOB_CREDENTIALS), ]) -def test_workload_mode_requires_federation(monkeypatch, tmp_path, - credential_type): +@pytest.mark.parametrize('kind', list(testing_bigquery.CredentialKind)) +def test_restricted_identity(monkeypatch, tmp_path, explicit_adc, identity, + target_var, key_var, kind): + fsutil.write_text(explicit_adc, json.dumps({'type': kind.value})) + restricted = str(tmp_path / 'restricted.json') + fsutil.write_text(restricted, '{"type": "service_account"}') + monkeypatch.setenv(key_var, restricted) + monkeypatch.setenv(target_var, 'restricted@example.com') + with testing_bigquery.setup_bigquery_credentials(identity): + if kind == testing_bigquery.CredentialKind.KEY: + assert os.environ['GOOGLE_APPLICATION_CREDENTIALS'] == restricted + assert testing_bigquery.ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT not in os.environ + else: + assert os.environ['GOOGLE_APPLICATION_CREDENTIALS'] == explicit_adc + assert os.environ[testing_bigquery.ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT] == 'restricted@example.com' + assert os.environ['GOOGLE_APPLICATION_CREDENTIALS'] == explicit_adc + assert fsutil.read_text(restricted) == '{"type": "service_account"}' + + +@pytest.mark.parametrize('document', ['not-json', '[]', '{}', '{"type":"unknown"}']) +def test_invalid_adc(explicit_adc, document): + fsutil.write_text(explicit_adc, document) + with pytest.raises(RuntimeError, match='invalid|unsupported'): + with testing_bigquery.setup_bigquery_credentials(): + pytest.fail('Invalid credentials accepted') + + +@pytest.mark.parametrize('kind', ['service_account', 'authorized_user', + 'impersonated_service_account']) +def test_workload_requires_federation(monkeypatch, explicit_adc, kind): monkeypatch.setenv('KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED', 'true') + fsutil.write_text(explicit_adc, json.dumps({'type': kind})) + with pytest.raises(RuntimeError, match='external_account'): + with testing_bigquery.setup_bigquery_credentials(): + pytest.fail('Non-federated workload credentials accepted') + + +def test_missing_adc_fails(monkeypatch): monkeypatch.delenv('GOOGLE_APPLICATION_CREDENTIALS', raising=False) - monkeypatch.setenv(bigquery_connector.ENV_GCS_KUMO_BUCKET.name, 'test-bucket') - if credential_type is not None: - adc = str(tmp_path / 'adc.json') - fsutil.write_text(adc, json.dumps({'type': credential_type})) - monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', adc) - get_secrets = mock.Mock(side_effect=AssertionError('No stored-key fallback')) - monkeypatch.setattr(testing_bigquery, 'get_secrets', get_secrets) - - with pytest.raises(RuntimeError, match='require'): - testing_bigquery.setup_bigquery_credentials() - - get_secrets.assert_not_called() + monkeypatch.setenv('GOOGLE_CREDENTIALS_STR_JSON', 'invalid-base64!') + with pytest.raises(RuntimeError, match='explicit'): + with testing_bigquery.setup_bigquery_credentials(): + pytest.fail('Implicit credentials accepted') + + +@pytest.mark.parametrize('failure', [RuntimeError, pytest.skip.Exception]) +def test_environment_restored(monkeypatch, explicit_adc, failure): + names = testing_bigquery._CREDENTIAL_ENV_VARS + monkeypatch.setenv('GCP_PROJECT_ID', 'previous-project') + monkeypatch.delenv(bigquery_connector.ENV_GOOGLE_CREDENTIALS_JSON, raising=False) + before = {name: os.environ.get(name) for name in names} + with pytest.raises(failure): + with testing_bigquery.setup_bigquery_credentials(): + for name in names: + os.environ[name] = 'changed-by-test' + raise failure('test failure') + assert {name: os.environ.get(name) for name in names} == before + assert fsutil.read_text(explicit_adc) == '{"type": "external_account"}' + + +@pytest.mark.parametrize('identity', [testing_bigquery.BigQueryIdentity.NO_TABLE, + testing_bigquery.BigQueryIdentity.NO_JOB]) +def test_restricted_requires_input(monkeypatch, explicit_adc, identity): + for variable in (testing_bigquery.ENV_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT, + testing_bigquery.ENV_BIGQUERY_NO_JOB_SERVICE_ACCOUNT): + monkeypatch.delenv(variable, raising=False) + before = dict(os.environ) + with pytest.raises(RuntimeError, match='restricted'): + with testing_bigquery.setup_bigquery_credentials(identity): + pytest.fail('Missing target accepted') + assert dict(os.environ) == before + + +@pytest.mark.parametrize('path', ['s3://bucket/key.json', 'gs://bucket/key.json']) +def test_remote_adc_rejected(monkeypatch, path): + monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', path) + with mock.patch.object(fsutil, 'exists', side_effect=AssertionError('Remote IO')): + with pytest.raises(RuntimeError, match='local ADC'): + testing_bigquery.resolve_credentials(testing_bigquery.BigQueryIdentity.DEFAULT) + + +@pytest.mark.parametrize('document', [None, '{"type":"authorized_user"}']) +def test_restricted_key_required(monkeypatch, tmp_path, explicit_adc, document): + fsutil.write_text(explicit_adc, '{"type":"service_account"}') + variable = testing_bigquery.ENV_BIGQUERY_NO_TABLE_CREDENTIALS + monkeypatch.delenv(variable, raising=False) + if document is not None: + restricted = str(tmp_path / 'restricted.json') + fsutil.write_text(restricted, document) + monkeypatch.setenv(variable, restricted) + with pytest.raises(RuntimeError, match=variable): + testing_bigquery.resolve_credentials(testing_bigquery.BigQueryIdentity.NO_TABLE) diff --git a/test/testing/test_gcs_credentials.py b/test/testing/test_gcs_credentials.py index 7abf64420..db9809cbd 100644 --- a/test/testing/test_gcs_credentials.py +++ b/test/testing/test_gcs_credentials.py @@ -3,10 +3,12 @@ # SPDX-License-Identifier: Apache-2.0 import os +from unittest import mock import pytest from io_utils import fsutil +from io_utils.fsutil import impl_gcs from kumo.connector.bigquery_connector import ENV_GOOGLE_CREDENTIALS_JSON from kumo.testing import gcs @@ -17,7 +19,7 @@ def test_decoration_never_configures(monkeypatch): def unexpected(): pytest.fail('Collection must not access credentials') - monkeypatch.setattr(gcs, '_check_gcs_credentials', unexpected) + monkeypatch.setattr(gcs, 'setup_bigquery_credentials', unexpected) @gcs.withGCSCredentials def example(): @@ -26,68 +28,156 @@ def example(): assert example.pytestmark[0].name == 'gcs_credentials' -def test_explicit_adc_wins(monkeypatch, tmp_path): +def test_explicit_adc_and_cache(monkeypatch, tmp_path): adc = str(tmp_path / 'adc.json') fsutil.write_text(adc, '{"type": "external_account"}') monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', adc) monkeypatch.setenv('GOOGLE_CREDENTIALS_STR_JSON', 'invalid-base64!') monkeypatch.setenv(ENV_GOOGLE_CREDENTIALS_JSON, 'previous') monkeypatch.setenv(gcs.ENV_GCS_TEST_BUCKET, 'test-bucket') - - gcs.require_gcs_credentials() - - assert os.environ[ENV_GOOGLE_CREDENTIALS_JSON] == adc + previous = impl_gcs.threadsafe_gcs + client = mock.Mock() + monkeypatch.setattr(impl_gcs.storage, 'Client', mock.Mock(return_value=client)) + + with gcs.setup_gcs_credentials(): + assert os.environ[ENV_GOOGLE_CREDENTIALS_JSON] == adc + assert impl_gcs.threadsafe_gcs is not previous + assert impl_gcs.threadsafe_gcs.client() is client + assert impl_gcs.threadsafe_gcs.client() is client + impl_gcs.storage.Client.assert_called_once() + os.environ[gcs.ENV_GCS_TEST_BUCKET] = 'changed' + + assert impl_gcs.threadsafe_gcs is previous + assert os.environ[gcs.ENV_GCS_TEST_BUCKET] == 'test-bucket' + assert os.environ[ENV_GOOGLE_CREDENTIALS_JSON] == 'previous' assert fsutil.read_text(adc) == '{"type": "external_account"}' -@pytest.mark.parametrize('skipped', [False, True]) -@pytest.mark.parametrize('available', [False, True]) -def test_prerequisite_precedes_resources(pytester, monkeypatch, skipped, - available): - calls = [] - - def credentials_available(): - calls.append('credentials') - if available: - monkeypatch.setenv('GCS_TEST_READY', '1') - return available - - monkeypatch.delenv('GCS_TEST_READY', raising=False) - monkeypatch.setattr(gcs, '_check_gcs_credentials', credentials_available) - # Exercise the real hook in an isolated pytest session, including pytest's - # built-in skip handling and session-resource ordering. - pytester.makeconftest(''' +@pytest.mark.parametrize('failure', [ + 'none', 'fixture', 'test', 'teardown', 'missing', 'skip', +]) +def test_lifecycle(pytester, monkeypatch, failure): + adc = str(pytester.path / 'adc.json') + fsutil.write_text(adc, '{"type": "external_account"}') + monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', adc) + monkeypatch.setenv('GOOGLE_CREDENTIALS_STR_JSON', 'invalid-base64!') + monkeypatch.setenv(ENV_GOOGLE_CREDENTIALS_JSON, 'original') + monkeypatch.setenv('GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT', 'restricted') + monkeypatch.setenv('GOOGLE_IMPERSONATE_SERVICE_ACCOUNT', 'original-target') + if failure == 'missing': + monkeypatch.delenv('GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT') + before = dict(os.environ) + pytester.makeconftest(f''' import os import pytest -from test.conftest import pytest_runtest_setup +from test.conftest import pytest_runtest_setup, pytest_runtest_teardown +from kumo.testing.bigquery import GCP_WORKERS_STARTED def pytest_configure(config): - config.addinivalue_line('markers', 'gcs_credentials: GCS prerequisite') + config.addinivalue_line('markers', 'bigquery_credentials(identity): identity') @pytest.fixture(scope='session') -def resource(): - assert os.environ.get('GCS_TEST_READY') == '1' +def worker(request): + assert os.environ['GOOGLE_CREDENTIALS_JSON'] == {adc!r} + request.config.stash[GCP_WORKERS_STARTED] = True + yield + assert os.environ['GOOGLE_CREDENTIALS_JSON'] == {adc!r} + +@pytest.fixture +def resource(worker): + assert os.environ['GOOGLE_IMPERSONATE_SERVICE_ACCOUNT'] == 'restricted' + if {failure!r} == 'fixture': + raise RuntimeError('fixture failure') yield + assert os.environ['GOOGLE_IMPERSONATE_SERVICE_ACCOUNT'] == 'restricted' + if {failure!r} == 'teardown': + raise RuntimeError('teardown failure') +''') + pytester.makepyfile(f''' +import os +import pytest +from kumo.testing.bigquery import BigQueryIdentity, withBigQueryCredentials + +@pytest.mark.skipif({failure!r} == 'skip', reason='Snowflake mode') +@withBigQueryCredentials(identity=BigQueryIdentity.NO_TABLE) +def test_live(resource): + if {failure!r} == 'test': + raise RuntimeError('test failure') + + +def test_next(): + expected = 'original-target' if {failure!r} == 'skip' else None + assert os.environ.get('GOOGLE_IMPERSONATE_SERVICE_ACCOUNT') == expected +''') + result = pytester.runpytest_inprocess( + '-q', '-o', 'addopts=', '--confcutdir', str(pytester.path)) + if failure in ('fixture', 'missing'): + result.assert_outcomes(passed=1, errors=1) + elif failure == 'teardown': + result.assert_outcomes(passed=2, errors=1) + elif failure == 'test': + result.assert_outcomes(passed=1, failed=1) + elif failure == 'skip': + result.assert_outcomes(passed=1, skipped=1) + else: + result.assert_outcomes(passed=2) + for variable in ('GOOGLE_APPLICATION_CREDENTIALS', ENV_GOOGLE_CREDENTIALS_JSON, + 'GOOGLE_IMPERSONATE_SERVICE_ACCOUNT', 'GCP_PROJECT_ID', + gcs.ENV_GCS_TEST_BUCKET): + assert os.environ.get(variable) == before.get(variable) + + +@pytest.mark.parametrize('skipped', [False, True]) +def test_existing_workers_rejected(pytester, monkeypatch, skipped): + adc = str(pytester.path / 'adc.json') + fsutil.write_text(adc, '{"type": "external_account"}') + monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', adc) + pytester.makeconftest(''' +import pytest +from test.conftest import pytest_runtest_setup, pytest_runtest_teardown +from kumo.testing.bigquery import GCP_WORKERS_STARTED + +def pytest_configure(config): + config.addinivalue_line('markers', 'gcs_credentials: identity') + config.stash[GCP_WORKERS_STARTED] = True ''') pytester.makepyfile(f''' import pytest from kumo.testing.gcs import withGCSCredentials -@pytest.mark.skipif({skipped!r}, reason='Snowflake mode') +@pytest.mark.skipif({skipped!r}, reason='Snowflake') @withGCSCredentials -def test_live(resource): +def test_late_gcp(): pass ''') - result = pytester.runpytest_inprocess( '-q', '-o', 'addopts=', '--confcutdir', str(pytester.path)) - if skipped: result.assert_outcomes(skipped=1) - assert calls == [] - elif available: - result.assert_outcomes(passed=1) - assert calls == ['credentials'] else: - result.assert_outcomes(skipped=1) - assert calls == ['credentials'] + result.assert_outcomes(errors=1) + result.stdout.fnmatch_lines(['*separate pytest invocation*']) + + +def test_mixed_filesystem_marker(pytester): + pytester.makeconftest(''' +def pytest_configure(config): + config.addinivalue_line('markers', 'gcs_credentials: identity') + +def pytest_collection_modifyitems(items): + marked = [item for item in items + if item.get_closest_marker('gcs_credentials') is not None] + assert len(marked) == 1 + assert marked[0].name == 'test_mixed[gcs]' + assert len(items) == 4 +''') + pytester.makepyfile(''' +from test.io_utils.fsutil.test_all import root + +def test_mixed(root): + pass +''') + result = pytester.runpytest_inprocess( + '--collect-only', '-q', '-o', 'addopts=', + '--confcutdir', str(pytester.path)) + assert result.ret == 0 From 3fc15f9e5d5f3452bb70cb6b44bf3d8b7dccfa01 Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Thu, 5 Nov 2026 12:48:52 -0600 Subject: [PATCH 2/3] Document explicit ADC for BigQuery integration tests --- test/rest/integ/test_bigquery_diskgraph_e2e.py | 3 ++- test/rest/integ/test_bigquery_ingestion_diskgraph_e2e.py | 7 +++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/rest/integ/test_bigquery_diskgraph_e2e.py b/test/rest/integ/test_bigquery_diskgraph_e2e.py index 749b8d11a..4e0b72418 100644 --- a/test/rest/integ/test_bigquery_diskgraph_e2e.py +++ b/test/rest/integ/test_bigquery_diskgraph_e2e.py @@ -15,7 +15,8 @@ - Temporal dev server running: make up-local-temporal - diskgraph Python package installed - DEPLOY_ENV_CONFIG_FILE pointing to a diskgraph deploy config - - BigQuery credentials available (via AWS Secrets Manager or env vars) + - GOOGLE_APPLICATION_CREDENTIALS pointing to an explicit local ADC file + (external_account credentials when workload exchange is enabled) - GCP_STORAGE_TRANSFER_BUCKET set to a writable GCS bucket Usage: diff --git a/test/rest/integ/test_bigquery_ingestion_diskgraph_e2e.py b/test/rest/integ/test_bigquery_ingestion_diskgraph_e2e.py index 4f92ba144..a7cd46a14 100644 --- a/test/rest/integ/test_bigquery_ingestion_diskgraph_e2e.py +++ b/test/rest/integ/test_bigquery_ingestion_diskgraph_e2e.py @@ -52,10 +52,9 @@ - ``DEPLOY_ENV_CONFIG_FILE`` pointing at a diskgraph deploy config (CI sets it for the ``bigquery_*_diskgraph`` matrix entries; this module also defaults it at import time) - - BigQuery credentials available via the - ``withBigQueryCredentials`` decorator (from AWS Secrets - Manager in CI, ``GOOGLE_CREDENTIALS_STR_JSON`` fallback - otherwise) + - ``GOOGLE_APPLICATION_CREDENTIALS`` pointing to an explicit local ADC file; + ``external_account`` credentials are required when workload exchange is + enabled. See ``docs/development/testing.md`` for supported local ADC types. - ``GCP_STORAGE_TRANSFER_BUCKET`` env var set to a writable GCS bucket (CI sets it to ``kumo-test-bucket``) From c9a246b77e0a8250ad19be4504d3031c570e56b7 Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Fri, 6 Nov 2026 11:47:36 -0600 Subject: [PATCH 3/3] Remove CI credential internals from the user testing guide --- docs/development/testing.md | 29 ------------------- .../test_bigquery_ingestion_diskgraph_e2e.py | 2 +- 2 files changed, 1 insertion(+), 30 deletions(-) diff --git a/docs/development/testing.md b/docs/development/testing.md index 64097a66b..7833b87e5 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -86,32 +86,3 @@ The [Engine PR guide](engine-pr-ci.md#local-feedback) gives one pytest command for the small pure output and workflow checks. The existing OSS Readiness job runs the same tests. They need no GPU or cloud access. The GPU job runs the eight actual training and save/load cases. - -## Live Google Cloud tests - -Selected BigQuery and GCS tests require an explicit -`GOOGLE_APPLICATION_CREDENTIALS` JSON file. Local runs accept `service_account`, -`authorized_user`, `impersonated_service_account`, and `external_account` ADC. -With `KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED=true`, only `external_account` is -accepted. Missing or invalid inputs fail setup. Tests do not fetch Google keys -from AWS Secrets Manager, decode `GOOGLE_CREDENTIALS_STR_JSON`, discover home -credentials, or write credential files. Existing input files remain unchanged. - -Restricted BigQuery permission tests use `GCP_BIGQUERY_NO_TABLE_CREDENTIALS` -and `GCP_BIGQUERY_NO_JOB_CREDENTIALS` for separate service-account key files. -Other ADC types use `GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT` and -`GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT` impersonation targets. The source identity -must be allowed to impersonate those deliberately restricted accounts. - -The decorators declare credentials without collection-time IO. Selected tests -prepare the base identity before fixtures start, scope restricted identities -through fixture teardown, and restore the original environment and cached GCS -client after the session's workers stop. Skipped Snowflake tests do not resolve -Google credentials. No credentials are required for collection alone. - -Run GCP integration modules in a separate pytest invocation from non-GCP -DiskGraph integration modules. If pool workers have already started without the -GCP test context, later GCP tests fail with an explicit instruction rather than -silently reusing workers with stale credentials. CI runs each integration module -in its own process. Shared AWS, Azure, and Databricks environment isolation is -outside this credential-helper change. diff --git a/test/rest/integ/test_bigquery_ingestion_diskgraph_e2e.py b/test/rest/integ/test_bigquery_ingestion_diskgraph_e2e.py index a7cd46a14..ea89cb839 100644 --- a/test/rest/integ/test_bigquery_ingestion_diskgraph_e2e.py +++ b/test/rest/integ/test_bigquery_ingestion_diskgraph_e2e.py @@ -54,7 +54,7 @@ entries; this module also defaults it at import time) - ``GOOGLE_APPLICATION_CREDENTIALS`` pointing to an explicit local ADC file; ``external_account`` credentials are required when workload exchange is - enabled. See ``docs/development/testing.md`` for supported local ADC types. + enabled. - ``GCP_STORAGE_TRANSFER_BUCKET`` env var set to a writable GCS bucket (CI sets it to ``kumo-test-bucket``)