Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions docs/development/testing.md
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,30 @@ Missing required credentials fail setup. The
launcher copies ADC into its temporary directory and preserves its external
token-source references, which must remain accessible for the run.

Direct local BigQuery tests also require an explicit
`GOOGLE_APPLICATION_CREDENTIALS` file. Supported local ADC types are
`service_account`, `authorized_user`, `impersonated_service_account`, and
`external_account`. Enabling `KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED` requires
`external_account` even outside a profile. BigQuery test setup does not fetch
AWS secrets, search home-directory credentials, or write credential files.
Missing or invalid inputs fail before dependent resources start.

For local restricted-permission tests using a service-account key, provide
separate key files through `GCP_BIGQUERY_NO_TABLE_CREDENTIALS` and
`GCP_BIGQUERY_NO_JOB_CREDENTIALS`. For user, impersonated, or federated ADC,
provide the two `GCP_BIGQUERY_NO_*_SERVICE_ACCOUNT` targets instead; the source
identity needs permission to impersonate them. The test identities must have
the intended restricted permissions. The helper validates the credential type
and explicit configuration; the live tests verify the actual permissions.

Use `@withBigQueryCredentials` for the default identity, or
`@withBigQueryCredentials(identity=BigQueryIdentity.NO_TABLE)` /
`BigQueryIdentity.NO_JOB` for restricted tests. These decorators only declare
requirements. Shared pytest fixtures prepare the base identity before session
resources and scope each test's identity until its dependent fixtures finish.
Explicit input files remain unchanged, and environment changes are restored
after success, failure, or skip.

The GCP-only runtime removes AWS secret fallback and uses a local artifact
directory. The GCP+AWS profile retains S3 because the training test actually
uses it. Credentials are prepared before worker startup; function-specific
Expand Down
197 changes: 88 additions & 109 deletions kumo/testing/bigquery.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,38 +2,28 @@
# All rights reserved.
# SPDX-License-Identifier: Apache-2.0

import asyncio
import functools
import json
import os
from collections.abc import Callable, Generator
from contextlib import contextmanager
from enum import Enum
from typing import Optional

import pytest

from io_utils import fsutil
from io_utils.secrets import get_secrets
from kumo.connector.bigquery_connector import (
ENV_GCS_KUMO_BUCKET,
ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH,
ENV_GOOGLE_CREDENTIALS_JSON,
)
from service_framework.env_var import EnvVar
from tools.ci_test_profiles import TestProfile, profile_for

DEFAULT_BIGQUERY_KEY = 'kumo-test-bigquery-service-account-key'
FAIL_TABLE_PERMISSION_BIGQUERY_KEY = 'kumo-test-permission-fail-bigquery-service-account-key-2'
FAIL_PROJECT_PERMISSION_BIGQUERY_KEY = 'kumo-test-permission-fail-bigquery-service-account-key'

ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT = 'GOOGLE_IMPERSONATE_SERVICE_ACCOUNT'
ENV_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT = 'GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT'
ENV_BIGQUERY_NO_JOB_SERVICE_ACCOUNT = 'GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT'
ENV_BIGQUERY_NO_TABLE_CREDENTIALS = 'GCP_BIGQUERY_NO_TABLE_CREDENTIALS'
ENV_BIGQUERY_NO_JOB_CREDENTIALS = 'GCP_BIGQUERY_NO_JOB_CREDENTIALS'

_IMPERSONATION_ENV_BY_KEY = {
FAIL_TABLE_PERMISSION_BIGQUERY_KEY: ENV_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT,
FAIL_PROJECT_PERMISSION_BIGQUERY_KEY: ENV_BIGQUERY_NO_JOB_SERVICE_ACCOUNT,
}
_CREDENTIAL_ENV_VARS = (
'GOOGLE_APPLICATION_CREDENTIALS',
ENV_GOOGLE_CREDENTIALS_JSON,
Expand All @@ -43,6 +33,12 @@
)


class BigQueryIdentity(Enum):
DEFAULT = 'default'
NO_TABLE = 'no_table'
NO_JOB = 'no_job'


class CredentialKind(Enum):
FEDERATED = 'external_account'
USER = 'authorized_user'
Expand All @@ -68,89 +64,87 @@ def credential_kind(path: str) -> CredentialKind:
'unsupported credential type') from error


def resolve_credentials(identity: BigQueryIdentity) -> tuple[str, str | None]:
"""Resolve explicit inputs without changing the environment or files."""
if not isinstance(identity, BigQueryIdentity):
raise TypeError('BigQuery credentials require a BigQueryIdentity')
path = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS')
if not path:
raise RuntimeError('BigQuery tests require explicit '
'GOOGLE_APPLICATION_CREDENTIALS')
kind = credential_kind(path)
profile = configured_profile()
require_wif = (profile is not None and 'gcp' in profile.capabilities) or EnvVar(
'KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED', bool, default_value=False).parse_value()
if require_wif and kind != CredentialKind.FEDERATED:
raise RuntimeError('Workload exchange BigQuery tests require '
'external_account ADC')
if identity == BigQueryIdentity.DEFAULT:
return path, None

if kind == CredentialKind.KEY:
variable = {
BigQueryIdentity.NO_TABLE: ENV_BIGQUERY_NO_TABLE_CREDENTIALS,
BigQueryIdentity.NO_JOB: ENV_BIGQUERY_NO_JOB_CREDENTIALS,
}[identity]
restricted_path = os.environ.get(variable)
if not restricted_path:
raise RuntimeError(f'{variable} must identify the restricted '
'BigQuery service-account credential file')
if credential_kind(restricted_path) != CredentialKind.KEY:
raise RuntimeError(f'{variable} must contain service_account '
'credentials')
return restricted_path, None

variable = {
BigQueryIdentity.NO_TABLE: ENV_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT,
BigQueryIdentity.NO_JOB: ENV_BIGQUERY_NO_JOB_SERVICE_ACCOUNT,
}[identity]
target = os.environ.get(variable)
if not target:
raise RuntimeError(f'{variable} must be configured for '
'the restricted BigQuery test identity')
return path, target


@contextmanager
def setup_bigquery_credentials(
service_account_key: Optional[str] = None,
identity: BigQueryIdentity = BigQueryIdentity.DEFAULT,
) -> Generator[None, None, None]:
"""Scope test credentials, including transitional stored-key fallback.

An ADC file's type determines whether restricted tests use impersonation.
A service-account key instead needs the existing per-identity secret. The
fallback remains until hosted workflows and local setup migrate together;
it never overwrites input files or writes shared home/repository files.
"""
key = service_account_key or DEFAULT_BIGQUERY_KEY
"""Scope explicit ADC to a test or session; input files remain read-only."""
path, target = resolve_credentials(identity)
with pytest.MonkeyPatch.context() as environment:
# Register all touched variables even when absent, including values a
# decorated test may overwrite before raising or skipping.
for name in _CREDENTIAL_ENV_VARS:
previous = os.environ.get(name)
environment.setenv(name, previous or '')
if previous is None:
environment.delenv(name)

profile = configured_profile()
profiled_gcp = profile is not None and 'gcp' in profile.capabilities
path = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS')
if profiled_gcp and not path:
raise RuntimeError('GCP test profiles require explicit '
'GOOGLE_APPLICATION_CREDENTIALS')
if not profiled_gcp:
path = path or os.environ.get(ENV_GOOGLE_CREDENTIALS_JSON)
if (not profiled_gcp and not path
and fsutil.exists(ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH)):
path = ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH
kind = credential_kind(path) if path else None
if profiled_gcp and kind != CredentialKind.FEDERATED:
raise RuntimeError('GCP test profiles require external_account ADC')
target_variable = _IMPERSONATION_ENV_BY_KEY.get(key)
target = None
if kind is not None and kind != CredentialKind.KEY and target_variable:
target = os.environ.get(target_variable)
if not target:
raise RuntimeError(f'{target_variable} must be configured for '
'the restricted BigQuery test identity')

temporary_dir = None
try:
if path is None or (kind == CredentialKind.KEY and target_variable):
try:
credentials = get_secrets(key)
except Exception:
pytest.skip('No BigQuery credentials found')
temporary_dir = fsutil.mkdtemp()
path = os.path.join(temporary_dir, 'credentials.json')
fsutil.write_text(path, json.dumps(credentials))
if credential_kind(path) != CredentialKind.KEY:
raise RuntimeError('Stored BigQuery test credentials must '
'be service_account credentials')

environment.setenv('GOOGLE_APPLICATION_CREDENTIALS', path)
environment.setenv(ENV_GOOGLE_CREDENTIALS_JSON, path)
if target:
environment.setenv(ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT,
target)
else:
environment.delenv(ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT,
raising=False)
environment.setenv(
ENV_GCS_KUMO_BUCKET.name,
os.environ.get(ENV_GCS_KUMO_BUCKET.name) or 'kumo-test-bucket')
project = (os.environ.get('GCP_PROJECT_ID')
or os.environ.get('GOOGLE_CLOUD_PROJECT'))
if project:
environment.setenv('GCP_PROJECT_ID', project)
yield
finally:
if temporary_dir is not None:
fsutil.rm(temporary_dir)
environment.setenv('GOOGLE_APPLICATION_CREDENTIALS', path)
environment.setenv(ENV_GOOGLE_CREDENTIALS_JSON, path)
if target:
environment.setenv(ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT, target)
else:
environment.delenv(ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT,
raising=False)
environment.setenv(
ENV_GCS_KUMO_BUCKET.name,
os.environ.get(ENV_GCS_KUMO_BUCKET.name) or 'kumo-test-bucket')
project = (os.environ.get('GCP_PROJECT_ID')
or os.environ.get('GOOGLE_CLOUD_PROJECT'))
if project:
environment.setenv('GCP_PROJECT_ID', project)
yield


@pytest.fixture(scope='session', autouse=True)
def configure_profile_credentials():
def configure_profile_credentials(request: pytest.FixtureRequest):
"""Prepare ADC before session resources and worker processes start."""
profile = configured_profile()
if profile is None or 'gcp' not in profile.capabilities:
selected = any(item.get_closest_marker('bigquery_credentials') is not None
and item.get_closest_marker('skip') is None
for item in request.session.items)
if not selected and (profile is None or 'gcp' not in profile.capabilities):
yield
return
with setup_bigquery_credentials():
Expand All @@ -163,39 +157,24 @@ def configure_bigquery_credentials(
configure_profile_credentials,
):
marker = request.node.get_closest_marker('bigquery_credentials')
if configured_profile() is None or marker is None:
if marker is None:
yield
return
key, = marker.args
if key is not None and not isinstance(key, str):
raise TypeError('bigquery_credentials requires a string identity key')
with setup_bigquery_credentials(key):
identity, = marker.args
if not isinstance(identity, BigQueryIdentity):
raise TypeError('bigquery_credentials requires a BigQueryIdentity')
with setup_bigquery_credentials(identity):
yield


def withBigQueryCredentials(func: Optional[Callable] = None,
service_account_key: Optional[str] = None):
def withBigQueryCredentials(
func: Callable | None = None,
*,
identity: BigQueryIdentity = BigQueryIdentity.DEFAULT,
):
"""Declare credentials without performing auth or wrapping the test body."""
def decorator(inner_func: Callable):
marked = pytest.mark.requires_credentials('gcp')(
pytest.mark.bigquery_credentials(service_account_key)(inner_func))
if configured_profile() is not None:
return marked
# Temporary unprofiled path: existing workflows retain phase-one
# behavior until all invocations use the profile launcher.
if asyncio.iscoroutinefunction(inner_func):

async def async_wrapper(*args, **kwargs):
with setup_bigquery_credentials(service_account_key):
return await inner_func(*args, **kwargs)

wrapper = async_wrapper
else:

def sync_wrapper(*args, **kwargs):
with setup_bigquery_credentials(service_account_key):
return inner_func(*args, **kwargs)

wrapper = sync_wrapper
return functools.wraps(marked)(wrapper)
return pytest.mark.requires_credentials('gcp')(
pytest.mark.bigquery_credentials(identity)(inner_func))

return decorator if func is None else decorator(func)
11 changes: 3 additions & 8 deletions test/connector/test_bigquery_connector.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,7 @@
SourceTableNotFoundError,
)
from kumo.stats.compute_stats_python import compute_col_stats_python
from kumo.testing.bigquery import (
FAIL_PROJECT_PERMISSION_BIGQUERY_KEY,
FAIL_TABLE_PERMISSION_BIGQUERY_KEY,
withBigQueryCredentials,
)
from kumo.testing.bigquery import BigQueryIdentity, withBigQueryCredentials


def _write_df_to_bigquery(connector, table_name, df, replace=False):
Expand Down Expand Up @@ -367,7 +363,7 @@ def test_set_credentials_and_open():
connector.open()


@withBigQueryCredentials(service_account_key=FAIL_TABLE_PERMISSION_BIGQUERY_KEY)
@withBigQueryCredentials(identity=BigQueryIdentity.NO_TABLE)
def test_bigquery_table_permission_fail_read_table():
connector = BigQueryConnector(
name='bigquery_fail',
Expand All @@ -385,8 +381,7 @@ def test_bigquery_table_permission_fail_read_table():
assert 'LOAN' not in message


@withBigQueryCredentials(
service_account_key=FAIL_PROJECT_PERMISSION_BIGQUERY_KEY)
@withBigQueryCredentials(identity=BigQueryIdentity.NO_JOB)
def test_bigquery_project_permission_fail_read_table():
connector = BigQueryConnector(
name='bigquery_fail',
Expand Down
Loading
Loading