From 0f112879fac77d5ca92f55035fbdfd09389cbd49 Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Thu, 22 Oct 2026 14:08:51 -0500 Subject: [PATCH] Restore keyless GCP authentication in NVIDIA CI --- .github/workflows/testing.yml | 4 +- .github/workflows/testing_gcs.yml | 2 +- .github/workflows/testing_snowflake.yml | 2 +- kumo/testing/bigquery.py | 28 +++- test/testing/test_bigquery_credentials.py | 164 ++++++++++++++++++++-- 5 files changed, 183 insertions(+), 17 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index cd20b2b7e..601de0e23 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -81,7 +81,7 @@ jobs: runs-on: ubuntu-24.04 continue-on-error: true environment: - name: azure-adls-tests + name: ${{ github.repository == 'NVIDIA/kumo-engine' && (contains(matrix.test_dir, 'test/io_utils/') || contains(matrix.test_dir, 'test/connector/') || contains(matrix.test_dir, 'test/rest/test_*.py')) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && 'gha-kumo-workflow-tests' || 'azure-adls-tests' }} # This job uses environment-scoped credentials for tests; it does not # deploy an application. Avoid one deployment record per matrix shard. deployment: false @@ -90,7 +90,7 @@ jobs: id-token: write env: KUMO_RUN_ADLS_TESTS: ${{ github.repository == 'kumo-ai/kumo' && contains(matrix.test_dir, 'test/io_utils/') && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && '1' || '0' }} - KUMO_RUN_GCP_TESTS: ${{ github.repository == 'kumo-ai/kumo' && (contains(matrix.test_dir, 'test/io_utils/') || contains(matrix.test_dir, 'test/connector/') || contains(matrix.test_dir, 'test/rest/test_*.py')) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && '1' || '0' }} + KUMO_RUN_GCP_TESTS: ${{ github.repository == 'NVIDIA/kumo-engine' && (contains(matrix.test_dir, 'test/io_utils/') || contains(matrix.test_dir, 'test/connector/') || contains(matrix.test_dir, 'test/rest/test_*.py')) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && '1' || '0' }} AZURE_STORAGE_ACCOUNT: ${{ vars.AZURE_STORAGE_ACCOUNT }} AZURE_CONTAINER_NAME: ${{ vars.AZURE_CONTAINER_NAME }} GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }} diff --git a/.github/workflows/testing_gcs.yml b/.github/workflows/testing_gcs.yml index a49396bae..65c680100 100644 --- a/.github/workflows/testing_gcs.yml +++ b/.github/workflows/testing_gcs.yml @@ -50,7 +50,7 @@ jobs: contents: read id-token: write env: - KUMO_RUN_GCP_TESTS: ${{ github.repository == 'kumo-ai/kumo' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && '1' || '0' }} + KUMO_RUN_GCP_TESTS: ${{ github.repository == 'NVIDIA/kumo-engine' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && '1' || '0' }} GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }} GCP_TEST_SERVICE_ACCOUNT: ${{ vars.GCP_TEST_SERVICE_ACCOUNT }} diff --git a/.github/workflows/testing_snowflake.yml b/.github/workflows/testing_snowflake.yml index e665ef75e..fcef9453f 100644 --- a/.github/workflows/testing_snowflake.yml +++ b/.github/workflows/testing_snowflake.yml @@ -50,7 +50,7 @@ jobs: contents: read id-token: write env: - KUMO_RUN_GCP_TESTS: ${{ github.repository == 'kumo-ai/kumo' && (contains(matrix.test_dir, 'test/io_utils/fsutil') || contains(matrix.test_dir, 'test/connector') || contains(matrix.test_dir, 'test/rest')) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && '1' || '0' }} + KUMO_RUN_GCP_TESTS: ${{ github.repository == 'NVIDIA/kumo-engine' && (contains(matrix.test_dir, 'test/io_utils/fsutil') || contains(matrix.test_dir, 'test/connector') || contains(matrix.test_dir, 'test/rest')) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && '1' || '0' }} GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }} GCP_TEST_SERVICE_ACCOUNT: ${{ vars.GCP_TEST_SERVICE_ACCOUNT }} GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT: ${{ vars.GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT }} diff --git a/kumo/testing/bigquery.py b/kumo/testing/bigquery.py index db7ad10be..84a8ff4ae 100644 --- a/kumo/testing/bigquery.py +++ b/kumo/testing/bigquery.py @@ -9,13 +9,14 @@ from collections.abc import Callable from typing import Optional -from io_utils import logging +from io_utils import fsutil, logging from io_utils.secrets import get_secrets from kumo.connector.bigquery_connector import ( ENV_GCS_KUMO_BUCKET, ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH, ENV_GOOGLE_CREDENTIALS_JSON, ) +from service_framework.env_var import EnvVar logger = logging.getLogger(__name__) @@ -49,10 +50,27 @@ def _configure_application_default_credentials( tests select a restricted service account through IAM impersonation rather than loading a second private key. """ + # GCP test jobs enable workload exchange explicitly, independently of the + # runner. Reuse that configuration to require federated test credentials. + require_wif = EnvVar('KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED', bool, + default_value=False).parse_value() credentials_path = os.getenv('GOOGLE_APPLICATION_CREDENTIALS') - if not credentials_path or not os.path.exists(credentials_path): + if not credentials_path or not fsutil.exists(credentials_path): + if require_wif: + raise RuntimeError( + 'Workload exchange BigQuery tests require Workload Identity ' + 'Federation ADC configured through ' + 'GOOGLE_APPLICATION_CREDENTIALS') return False + if require_wif: + credentials = json.loads(fsutil.read_text(credentials_path)) + if credentials.get('type') != 'external_account': + raise RuntimeError( + 'Workload exchange BigQuery tests require external_account ADC; ' + 'stored service-account keys and user credentials are not ' + 'supported with workload exchange enabled') + impersonation_env = _IMPERSONATION_ENV_BY_KEY.get(service_account_key) if impersonation_env is None: os.environ.pop(ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT, None) @@ -79,14 +97,14 @@ def setup_bigquery_credentials(service_account_key: Optional[str] = None): service_account_key = service_account_key or DEFAULT_BIGQUERY_KEY # Prefer short-lived ADC supplied by Workload Identity Federation. This - # keeps GitHub Actions from reading any stored GCP service-account key. + # keeps workload exchange tests from reading stored service-account keys. if _configure_application_default_credentials(service_account_key): return try: # Get the GCloud secrets from AWS secrets manager and store it in the - # default file. This remains a local/deployment fallback; CI supplies - # ADC above and never reaches this path. + # default file. Workload exchange requires federated ADC above and + # never reaches this existing fallback. secrets = get_secrets(service_account_key) file_path = os.path.join(os.getcwd(), ".google_credential.json") with open(file_path, 'w') as f: diff --git a/test/testing/test_bigquery_credentials.py b/test/testing/test_bigquery_credentials.py index a87107373..b8c21b5e5 100644 --- a/test/testing/test_bigquery_credentials.py +++ b/test/testing/test_bigquery_credentials.py @@ -2,12 +2,14 @@ # All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import asyncio import os from unittest import mock import kumoapi.data_source as data_source_api import pytest +from io_utils import fsutil from kumo.connector import bigquery_connector from kumo.connector.bigquery_connector import BigQueryConnector from kumo.testing import bigquery as testing_bigquery @@ -16,6 +18,149 @@ invalid_bigquery_key_test_identity_config, ) +_CREDENTIAL_ENV_VARS = ( + 'GOOGLE_APPLICATION_CREDENTIALS', + testing_bigquery.ENV_GOOGLE_CREDENTIALS_JSON, + testing_bigquery.ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT, + testing_bigquery.ENV_GCS_KUMO_BUCKET.name, + 'GCP_PROJECT_ID', + 'GOOGLE_CLOUD_PROJECT', + 'KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED', +) + + +@pytest.fixture(autouse=True) +def credential_environment(monkeypatch): + # Register even absent variables: the helper writes them without using + # monkeypatch, and their original values must survive fixture teardown. + for name in _CREDENTIAL_ENV_VARS: + monkeypatch.setenv(name, '') + monkeypatch.delenv(name) + + +@pytest.fixture(params=['true', 'false']) +def wif_credentials(monkeypatch, request): + monkeypatch.setenv('GITHUB_ACTIONS', request.param) + monkeypatch.setenv('KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED', 'true') + get_secrets = mock.Mock(side_effect=AssertionError( + 'Workload exchange must not read a stored service-account key')) + monkeypatch.setattr(testing_bigquery, 'get_secrets', get_secrets) + yield get_secrets + get_secrets.assert_not_called() + + +@pytest.mark.parametrize('github_actions', ['true', 'false']) +@pytest.mark.parametrize('exchange_enabled', [None, 'false']) +def test_unconfigured_auth_preserves_fallback( + monkeypatch, github_actions, exchange_enabled): + monkeypatch.setenv('GITHUB_ACTIONS', github_actions) + if exchange_enabled is not None: + monkeypatch.setenv('KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED', exchange_enabled) + + assert not testing_bigquery._configure_application_default_credentials( + testing_bigquery.DEFAULT_BIGQUERY_KEY) + + +@pytest.mark.parametrize('original_value', [None, 'original']) +def test_credential_environment_restores(monkeypatch, tmp_path, original_value): + for name in _CREDENTIAL_ENV_VARS: + if original_value is not None: + monkeypatch.setenv(name, original_value) + credentials_path = str(tmp_path / 'gha-creds.json') + fsutil.write_text(credentials_path, '{"type": "external_account"}') + + with pytest.MonkeyPatch.context() as isolated: + credential_environment.__wrapped__(isolated) + isolated.setenv('KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED', 'true') + isolated.setenv('GOOGLE_APPLICATION_CREDENTIALS', credentials_path) + isolated.setenv('GOOGLE_CLOUD_PROJECT', 'example-project') + testing_bigquery.setup_bigquery_credentials() + assert os.environ[testing_bigquery.ENV_GOOGLE_CREDENTIALS_JSON] == ( + credentials_path) + + for name in _CREDENTIAL_ENV_VARS: + assert os.environ.get(name) == original_value + + +@pytest.mark.parametrize('missing_path', [False, True]) +def test_wif_requires_adc(monkeypatch, tmp_path, wif_credentials, missing_path): + if missing_path: + monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', + str(tmp_path / 'missing.json')) + else: + monkeypatch.delenv('GOOGLE_APPLICATION_CREDENTIALS', raising=False) + + with pytest.raises(RuntimeError, match='Federation ADC'): + testing_bigquery.setup_bigquery_credentials() + + +@pytest.mark.parametrize('credential_type', [ + 'service_account', 'authorized_user', 'impersonated_service_account', +]) +def test_wif_rejects_nonfederated_adc( + monkeypatch, tmp_path, wif_credentials, credential_type): + credentials_path = str(tmp_path / 'adc.json') + fsutil.write_text(credentials_path, '{"type": "' + credential_type + '"}') + monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', credentials_path) + + with pytest.raises(RuntimeError, match='external_account ADC'): + testing_bigquery.setup_bigquery_credentials() + + +def test_wif_switches_impersonation(monkeypatch, tmp_path, wif_credentials): + credentials_path = str(tmp_path / 'gha-creds.json') + fsutil.write_text(credentials_path, '{"type": "external_account"}') + monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', credentials_path) + targets = { + testing_bigquery.FAIL_TABLE_PERMISSION_BIGQUERY_KEY: ( + testing_bigquery.ENV_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT, + 'no-table@example.test'), + testing_bigquery.FAIL_PROJECT_PERMISSION_BIGQUERY_KEY: ( + testing_bigquery.ENV_BIGQUERY_NO_JOB_SERVICE_ACCOUNT, + 'no-job@example.test'), + } + for env_name, target in targets.values(): + monkeypatch.setenv(env_name, target) + + for key in (None, *targets, None): + testing_bigquery.setup_bigquery_credentials(key) + assert os.environ.get( + testing_bigquery.ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT) == ( + targets[key][1] if key else None) + assert os.environ[bigquery_connector.ENV_GOOGLE_CREDENTIALS_JSON] == ( + credentials_path) + assert fsutil.read_text(credentials_path) == '{"type": "external_account"}' + + +@pytest.mark.parametrize('asynchronous', [False, True]) +def test_wif_restores_impersonation( + monkeypatch, tmp_path, wif_credentials, asynchronous): + credentials_path = str(tmp_path / 'gha-creds.json') + fsutil.write_text(credentials_path, '{"type": "external_account"}') + monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', credentials_path) + target_env = testing_bigquery.ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT + monkeypatch.setenv(target_env, 'previous@example.test') + monkeypatch.setenv(testing_bigquery.ENV_BIGQUERY_NO_JOB_SERVICE_ACCOUNT, + 'no-job@example.test') + + def fail(): + assert os.environ[target_env] == 'no-job@example.test' + raise ValueError('test failed') + + async def async_fail(): + fail() + + wrapped = testing_bigquery.withBigQueryCredentials( + service_account_key=testing_bigquery.FAIL_PROJECT_PERMISSION_BIGQUERY_KEY + )(async_fail if asynchronous else fail) + with pytest.raises(ValueError, match='test failed'): + if asynchronous: + asyncio.run(wrapped()) + else: + wrapped() + + assert os.environ[target_env] == 'previous@example.test' + def test_bigquery_rest_identity_uses_workload_exchange(monkeypatch): service_account = ( @@ -71,18 +216,21 @@ def test_restricted_identity_uses_iam_impersonation(monkeypatch, tmp_path): testing_bigquery.ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT] == target -def test_restricted_identity_requires_target_variable(monkeypatch, tmp_path): +@pytest.mark.parametrize(('key', 'target_env'), [ + (testing_bigquery.FAIL_TABLE_PERMISSION_BIGQUERY_KEY, + testing_bigquery.ENV_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT), + (testing_bigquery.FAIL_PROJECT_PERMISSION_BIGQUERY_KEY, + testing_bigquery.ENV_BIGQUERY_NO_JOB_SERVICE_ACCOUNT), +]) +def test_restricted_identity_requires_target_variable( + monkeypatch, tmp_path, wif_credentials, key, target_env): credentials_path = tmp_path / "gha-creds.json" credentials_path.write_text('{"type": "external_account"}') monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', str(credentials_path)) - monkeypatch.delenv( - testing_bigquery.ENV_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT, - raising=False, - ) + monkeypatch.delenv(target_env, raising=False) - with pytest.raises(RuntimeError, match='GCP_BIGQUERY_NO_TABLE'): - testing_bigquery.setup_bigquery_credentials( - testing_bigquery.FAIL_TABLE_PERMISSION_BIGQUERY_KEY) + with pytest.raises(RuntimeError, match=target_env): + testing_bigquery.setup_bigquery_credentials(key) def test_connector_loads_external_account_and_impersonates(