Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/testing.yml
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ jobs:
runs-on: ubuntu-24.04
continue-on-error: true
environment:
name: azure-adls-tests
name: ${{ github.repository == 'NVIDIA/kumo-engine' && (contains(matrix.test_dir, 'test/io_utils/') || contains(matrix.test_dir, 'test/connector/') || contains(matrix.test_dir, 'test/rest/test_*.py')) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && 'gha-kumo-workflow-tests' || 'azure-adls-tests' }}
# This job uses environment-scoped credentials for tests; it does not
# deploy an application. Avoid one deployment record per matrix shard.
deployment: false
Expand All @@ -90,7 +90,7 @@ jobs:
id-token: write
env:
KUMO_RUN_ADLS_TESTS: ${{ github.repository == 'kumo-ai/kumo' && contains(matrix.test_dir, 'test/io_utils/') && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && '1' || '0' }}
KUMO_RUN_GCP_TESTS: ${{ github.repository == 'kumo-ai/kumo' && (contains(matrix.test_dir, 'test/io_utils/') || contains(matrix.test_dir, 'test/connector/') || contains(matrix.test_dir, 'test/rest/test_*.py')) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && '1' || '0' }}
KUMO_RUN_GCP_TESTS: ${{ github.repository == 'NVIDIA/kumo-engine' && (contains(matrix.test_dir, 'test/io_utils/') || contains(matrix.test_dir, 'test/connector/') || contains(matrix.test_dir, 'test/rest/test_*.py')) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && '1' || '0' }}
AZURE_STORAGE_ACCOUNT: ${{ vars.AZURE_STORAGE_ACCOUNT }}
AZURE_CONTAINER_NAME: ${{ vars.AZURE_CONTAINER_NAME }}
GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }}
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/testing_gcs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ jobs:
contents: read
id-token: write
env:
KUMO_RUN_GCP_TESTS: ${{ github.repository == 'kumo-ai/kumo' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && '1' || '0' }}
KUMO_RUN_GCP_TESTS: ${{ github.repository == 'NVIDIA/kumo-engine' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && '1' || '0' }}
GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }}
GCP_TEST_SERVICE_ACCOUNT: ${{ vars.GCP_TEST_SERVICE_ACCOUNT }}

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/testing_snowflake.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ jobs:
contents: read
id-token: write
env:
KUMO_RUN_GCP_TESTS: ${{ github.repository == 'kumo-ai/kumo' && (contains(matrix.test_dir, 'test/io_utils/fsutil') || contains(matrix.test_dir, 'test/connector') || contains(matrix.test_dir, 'test/rest')) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && '1' || '0' }}
KUMO_RUN_GCP_TESTS: ${{ github.repository == 'NVIDIA/kumo-engine' && (contains(matrix.test_dir, 'test/io_utils/fsutil') || contains(matrix.test_dir, 'test/connector') || contains(matrix.test_dir, 'test/rest')) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && '1' || '0' }}
GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }}
GCP_TEST_SERVICE_ACCOUNT: ${{ vars.GCP_TEST_SERVICE_ACCOUNT }}
GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT: ${{ vars.GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT }}
Expand Down
28 changes: 23 additions & 5 deletions kumo/testing/bigquery.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,14 @@
from collections.abc import Callable
from typing import Optional

from io_utils import logging
from io_utils import fsutil, logging
from io_utils.secrets import get_secrets
from kumo.connector.bigquery_connector import (
ENV_GCS_KUMO_BUCKET,
ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH,
ENV_GOOGLE_CREDENTIALS_JSON,
)
from service_framework.env_var import EnvVar

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -49,10 +50,27 @@ def _configure_application_default_credentials(
tests select a restricted service account through IAM impersonation rather
than loading a second private key.
"""
# GCP test jobs enable workload exchange explicitly, independently of the
# runner. Reuse that configuration to require federated test credentials.
require_wif = EnvVar('KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED', bool,
default_value=False).parse_value()
credentials_path = os.getenv('GOOGLE_APPLICATION_CREDENTIALS')
if not credentials_path or not os.path.exists(credentials_path):
if not credentials_path or not fsutil.exists(credentials_path):
if require_wif:
raise RuntimeError(
'Workload exchange BigQuery tests require Workload Identity '
'Federation ADC configured through '
'GOOGLE_APPLICATION_CREDENTIALS')
return False

if require_wif:
credentials = json.loads(fsutil.read_text(credentials_path))
if credentials.get('type') != 'external_account':
raise RuntimeError(
'Workload exchange BigQuery tests require external_account ADC; '
'stored service-account keys and user credentials are not '
'supported with workload exchange enabled')

impersonation_env = _IMPERSONATION_ENV_BY_KEY.get(service_account_key)
if impersonation_env is None:
os.environ.pop(ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT, None)
Expand All @@ -79,14 +97,14 @@ def setup_bigquery_credentials(service_account_key: Optional[str] = None):
service_account_key = service_account_key or DEFAULT_BIGQUERY_KEY

# Prefer short-lived ADC supplied by Workload Identity Federation. This
# keeps GitHub Actions from reading any stored GCP service-account key.
# keeps workload exchange tests from reading stored service-account keys.
if _configure_application_default_credentials(service_account_key):
return

try:
# Get the GCloud secrets from AWS secrets manager and store it in the
# default file. This remains a local/deployment fallback; CI supplies
# ADC above and never reaches this path.
# default file. Workload exchange requires federated ADC above and
# never reaches this existing fallback.
secrets = get_secrets(service_account_key)
file_path = os.path.join(os.getcwd(), ".google_credential.json")
with open(file_path, 'w') as f:
Expand Down
164 changes: 156 additions & 8 deletions test/testing/test_bigquery_credentials.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,14 @@
# All rights reserved.
# SPDX-License-Identifier: Apache-2.0

import asyncio
import os
from unittest import mock

import kumoapi.data_source as data_source_api
import pytest

from io_utils import fsutil
from kumo.connector import bigquery_connector
from kumo.connector.bigquery_connector import BigQueryConnector
from kumo.testing import bigquery as testing_bigquery
Expand All @@ -16,6 +18,149 @@
invalid_bigquery_key_test_identity_config,
)

_CREDENTIAL_ENV_VARS = (
'GOOGLE_APPLICATION_CREDENTIALS',
testing_bigquery.ENV_GOOGLE_CREDENTIALS_JSON,
testing_bigquery.ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT,
testing_bigquery.ENV_GCS_KUMO_BUCKET.name,
'GCP_PROJECT_ID',
'GOOGLE_CLOUD_PROJECT',
'KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED',
)


@pytest.fixture(autouse=True)
def credential_environment(monkeypatch):
# Register even absent variables: the helper writes them without using
# monkeypatch, and their original values must survive fixture teardown.
for name in _CREDENTIAL_ENV_VARS:
monkeypatch.setenv(name, '')
monkeypatch.delenv(name)


@pytest.fixture(params=['true', 'false'])
def wif_credentials(monkeypatch, request):
monkeypatch.setenv('GITHUB_ACTIONS', request.param)
monkeypatch.setenv('KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED', 'true')
get_secrets = mock.Mock(side_effect=AssertionError(
'Workload exchange must not read a stored service-account key'))
monkeypatch.setattr(testing_bigquery, 'get_secrets', get_secrets)
yield get_secrets
get_secrets.assert_not_called()


@pytest.mark.parametrize('github_actions', ['true', 'false'])
@pytest.mark.parametrize('exchange_enabled', [None, 'false'])
def test_unconfigured_auth_preserves_fallback(
monkeypatch, github_actions, exchange_enabled):
monkeypatch.setenv('GITHUB_ACTIONS', github_actions)
if exchange_enabled is not None:
monkeypatch.setenv('KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED', exchange_enabled)

assert not testing_bigquery._configure_application_default_credentials(
testing_bigquery.DEFAULT_BIGQUERY_KEY)


@pytest.mark.parametrize('original_value', [None, 'original'])
def test_credential_environment_restores(monkeypatch, tmp_path, original_value):
for name in _CREDENTIAL_ENV_VARS:
if original_value is not None:
monkeypatch.setenv(name, original_value)
credentials_path = str(tmp_path / 'gha-creds.json')
fsutil.write_text(credentials_path, '{"type": "external_account"}')

with pytest.MonkeyPatch.context() as isolated:
credential_environment.__wrapped__(isolated)
isolated.setenv('KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED', 'true')
isolated.setenv('GOOGLE_APPLICATION_CREDENTIALS', credentials_path)
isolated.setenv('GOOGLE_CLOUD_PROJECT', 'example-project')
testing_bigquery.setup_bigquery_credentials()
assert os.environ[testing_bigquery.ENV_GOOGLE_CREDENTIALS_JSON] == (
credentials_path)

for name in _CREDENTIAL_ENV_VARS:
assert os.environ.get(name) == original_value


@pytest.mark.parametrize('missing_path', [False, True])
def test_wif_requires_adc(monkeypatch, tmp_path, wif_credentials, missing_path):
if missing_path:
monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS',
str(tmp_path / 'missing.json'))
else:
monkeypatch.delenv('GOOGLE_APPLICATION_CREDENTIALS', raising=False)

with pytest.raises(RuntimeError, match='Federation ADC'):
testing_bigquery.setup_bigquery_credentials()


@pytest.mark.parametrize('credential_type', [
'service_account', 'authorized_user', 'impersonated_service_account',
])
def test_wif_rejects_nonfederated_adc(
monkeypatch, tmp_path, wif_credentials, credential_type):
credentials_path = str(tmp_path / 'adc.json')
fsutil.write_text(credentials_path, '{"type": "' + credential_type + '"}')
monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', credentials_path)

with pytest.raises(RuntimeError, match='external_account ADC'):
testing_bigquery.setup_bigquery_credentials()


def test_wif_switches_impersonation(monkeypatch, tmp_path, wif_credentials):
credentials_path = str(tmp_path / 'gha-creds.json')
fsutil.write_text(credentials_path, '{"type": "external_account"}')
monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', credentials_path)
targets = {
testing_bigquery.FAIL_TABLE_PERMISSION_BIGQUERY_KEY: (
testing_bigquery.ENV_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT,
'no-table@example.test'),
testing_bigquery.FAIL_PROJECT_PERMISSION_BIGQUERY_KEY: (
testing_bigquery.ENV_BIGQUERY_NO_JOB_SERVICE_ACCOUNT,
'no-job@example.test'),
}
for env_name, target in targets.values():
monkeypatch.setenv(env_name, target)

for key in (None, *targets, None):
testing_bigquery.setup_bigquery_credentials(key)
assert os.environ.get(
testing_bigquery.ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT) == (
targets[key][1] if key else None)
assert os.environ[bigquery_connector.ENV_GOOGLE_CREDENTIALS_JSON] == (
credentials_path)
assert fsutil.read_text(credentials_path) == '{"type": "external_account"}'


@pytest.mark.parametrize('asynchronous', [False, True])
def test_wif_restores_impersonation(
monkeypatch, tmp_path, wif_credentials, asynchronous):
credentials_path = str(tmp_path / 'gha-creds.json')
fsutil.write_text(credentials_path, '{"type": "external_account"}')
monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', credentials_path)
target_env = testing_bigquery.ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT
monkeypatch.setenv(target_env, 'previous@example.test')
monkeypatch.setenv(testing_bigquery.ENV_BIGQUERY_NO_JOB_SERVICE_ACCOUNT,
'no-job@example.test')

def fail():
assert os.environ[target_env] == 'no-job@example.test'
raise ValueError('test failed')

async def async_fail():
fail()

wrapped = testing_bigquery.withBigQueryCredentials(
service_account_key=testing_bigquery.FAIL_PROJECT_PERMISSION_BIGQUERY_KEY
)(async_fail if asynchronous else fail)
with pytest.raises(ValueError, match='test failed'):
if asynchronous:
asyncio.run(wrapped())
else:
wrapped()

assert os.environ[target_env] == 'previous@example.test'


def test_bigquery_rest_identity_uses_workload_exchange(monkeypatch):
service_account = (
Expand Down Expand Up @@ -71,18 +216,21 @@ def test_restricted_identity_uses_iam_impersonation(monkeypatch, tmp_path):
testing_bigquery.ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT] == target


def test_restricted_identity_requires_target_variable(monkeypatch, tmp_path):
@pytest.mark.parametrize(('key', 'target_env'), [
(testing_bigquery.FAIL_TABLE_PERMISSION_BIGQUERY_KEY,
testing_bigquery.ENV_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT),
(testing_bigquery.FAIL_PROJECT_PERMISSION_BIGQUERY_KEY,
testing_bigquery.ENV_BIGQUERY_NO_JOB_SERVICE_ACCOUNT),
])
def test_restricted_identity_requires_target_variable(
monkeypatch, tmp_path, wif_credentials, key, target_env):
credentials_path = tmp_path / "gha-creds.json"
credentials_path.write_text('{"type": "external_account"}')
monkeypatch.setenv('GOOGLE_APPLICATION_CREDENTIALS', str(credentials_path))
monkeypatch.delenv(
testing_bigquery.ENV_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT,
raising=False,
)
monkeypatch.delenv(target_env, raising=False)

with pytest.raises(RuntimeError, match='GCP_BIGQUERY_NO_TABLE'):
testing_bigquery.setup_bigquery_credentials(
testing_bigquery.FAIL_TABLE_PERMISSION_BIGQUERY_KEY)
with pytest.raises(RuntimeError, match=target_env):
testing_bigquery.setup_bigquery_credentials(key)


def test_connector_loads_external_account_and_impersonates(
Expand Down
Loading