Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 4 additions & 5 deletions .github/workflows/testing.yml
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ jobs:
runs-on: ubuntu-24.04
continue-on-error: true
environment:
name: azure-adls-tests
name: ${{ github.repository == 'NVIDIA/kumo-engine' && (contains(matrix.test_dir, 'test/io_utils/') || contains(matrix.test_dir, 'test/connector/') || contains(matrix.test_dir, 'test/rest/test_*.py')) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && 'gha-kumo-workflow-tests' || 'azure-adls-tests' }}
# This job uses environment-scoped credentials for tests; it does not
# deploy an application. Avoid one deployment record per matrix shard.
deployment: false
Expand All @@ -90,7 +90,7 @@ jobs:
id-token: write
env:
KUMO_RUN_ADLS_TESTS: ${{ github.repository == 'kumo-ai/kumo' && contains(matrix.test_dir, 'test/io_utils/') && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && '1' || '0' }}
KUMO_RUN_GCP_TESTS: ${{ github.repository == 'kumo-ai/kumo' && (contains(matrix.test_dir, 'test/io_utils/') || contains(matrix.test_dir, 'test/connector/') || contains(matrix.test_dir, 'test/rest/test_*.py')) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && '1' || '0' }}
KUMO_RUN_GCP_TESTS: ${{ github.repository == 'NVIDIA/kumo-engine' && (contains(matrix.test_dir, 'test/io_utils/') || contains(matrix.test_dir, 'test/connector/') || contains(matrix.test_dir, 'test/rest/test_*.py')) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && '1' || '0' }}
AZURE_STORAGE_ACCOUNT: ${{ vars.AZURE_STORAGE_ACCOUNT }}
AZURE_CONTAINER_NAME: ${{ vars.AZURE_CONTAINER_NAME }}
GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }}
Expand Down Expand Up @@ -190,10 +190,9 @@ jobs:
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
with:
files: |
ui/**
docs/**/*.md
README.md
CHANGELOG.md
tools/**
benchmark/**

- name: Configure AWS credentials
if: env.FORCE_RUN_TESTS == 'true' || steps.non-pytest-changes.outputs.only_changed != 'true'
Expand Down
35 changes: 6 additions & 29 deletions .github/workflows/testing_gcs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ on: # yamllint disable-line rule:truthy
push:
branches:
- main
paths-ignore: ["ui/**", "tools/**"]
pull_request:
merge_group:
workflow_dispatch:
Expand Down Expand Up @@ -45,12 +44,11 @@ jobs:
deployment: false
name: gcs-e2e-tests
runs-on: ubuntu-24.04
continue-on-error: true
permissions:
contents: read
id-token: write
env:
KUMO_RUN_GCP_TESTS: ${{ github.repository == 'kumo-ai/kumo' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && '1' || '0' }}
KUMO_RUN_GCP_TESTS: ${{ github.repository == 'NVIDIA/kumo-engine' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && '1' || '0' }}
GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }}
GCP_TEST_SERVICE_ACCOUNT: ${{ vars.GCP_TEST_SERVICE_ACCOUNT }}

Expand All @@ -70,63 +68,42 @@ jobs:
ref: ${{ github.sha }}
fetch-depth: 40

- name: Get GCS related changes
id: gcs-files-changed
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
with:
files: |
.github/actions/setup/action.yml
.github/workflows/testing_gcs.yml
kumo/connector/**
kumo/testing/gcs.py
io_utils/**
rest/**
setup.py
test/rest/integ/test_gcs_e2e.py
test/rest/integ/test_bigquery_diskgraph_e2e.py
test/rest/integ/test_bigquery_ingestion_diskgraph_e2e.py
test/rest/integ/test_cases/test_bigquery_e2e_fixtures.json
rest/devserver/configs/diskgraph-test.json
kumo/testing/bigquery.py
kumo/testing/rest.py

# Run every PR: shared package, fixture, and workflow changes can affect
# these integrations, not just connector and REST source paths.
- name: Set up Python
if: steps.gcs-files-changed.outputs.any_changed == 'true'
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
python-version: "3.14"
activate-environment: true
prune-cache: true

- name: Configure AWS credentials
if: steps.gcs-files-changed.outputs.any_changed == 'true'
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
with:
role-to-assume: arn:aws:iam::926922431314:role/gha-kumo-workflow-tests
aws-region: us-west-2

- name: Set up packages
if: steps.gcs-files-changed.outputs.any_changed == 'true' && !contains(matrix.test_dir, 'diskgraph')
if: ${{ !contains(matrix.test_dir, 'diskgraph') }}
uses: ./.github/actions/setup
with:
github_token: ${{ secrets.KUMO_GITHUB_BOT_TOKEN }}

- name: Set up packages (with diskgraph)
if: steps.gcs-files-changed.outputs.any_changed == 'true' && contains(matrix.test_dir, 'diskgraph')
if: contains(matrix.test_dir, 'diskgraph')
uses: ./.github/actions/setup
with:
github_token: ${{ secrets.KUMO_GITHUB_BOT_TOKEN }}

- name: Authenticate to Google Cloud with GitHub OIDC
if: steps.gcs-files-changed.outputs.any_changed == 'true' && env.KUMO_RUN_GCP_TESTS == '1'
if: env.KUMO_RUN_GCP_TESTS == '1'
Comment thread
danielmawhirter marked this conversation as resolved.
Resolved
uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3
with:
project_id: ${{ vars.GCP_PROJECT_ID }}
workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.GCP_TEST_SERVICE_ACCOUNT }}

- name: Run E2E tests
if: steps.gcs-files-changed.outputs.any_changed == 'true'
timeout-minutes: 60
run: |
set -o pipefail
Expand Down
18 changes: 0 additions & 18 deletions .github/workflows/testing_snowflake.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,12 +49,6 @@ jobs:
permissions:
contents: read
id-token: write
env:
KUMO_RUN_GCP_TESTS: ${{ github.repository == 'kumo-ai/kumo' && (contains(matrix.test_dir, 'test/io_utils/fsutil') || contains(matrix.test_dir, 'test/connector') || contains(matrix.test_dir, 'test/rest')) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && '1' || '0' }}
GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }}
GCP_TEST_SERVICE_ACCOUNT: ${{ vars.GCP_TEST_SERVICE_ACCOUNT }}
GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT: ${{ vars.GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT }}
GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT: ${{ vars.GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT }}

strategy:
matrix:
Expand Down Expand Up @@ -179,14 +173,6 @@ jobs:
with:
github_token: ${{ secrets.KUMO_GITHUB_BOT_TOKEN }}

- name: Authenticate to Google Cloud with GitHub OIDC
if: steps.data-backend-files-changed.outputs.any_changed == 'true' && steps.only-ui-files-changed.outputs.only_changed != 'true' && steps.only-rest-files-changed.outputs.only_changed != 'true' && env.KUMO_RUN_GCP_TESTS == '1'
uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3
with:
project_id: ${{ vars.GCP_PROJECT_ID }}
workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.GCP_TEST_SERVICE_ACCOUNT }}

# NOTE here we use coverage.py instead of pytest-cov to obtain JSON
# outputs that are easier to parse and manage (pytest-cov/455).
# coverage.py also has additional nice features that may be useful in the
Expand All @@ -202,8 +188,6 @@ jobs:
GLOG_logtostdout=1 python -m coverage run -m pytest "${PYTEST_CI_OUTPUT_ARGS[@]}" --durations=30 --runsnowflake ${{ matrix.test_dir }}
env:
SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }}
GCS_TEST_BUCKET: 'kumo-test-bucket'
KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED: ${{ env.KUMO_RUN_GCP_TESTS == '1' && 'true' || 'false' }}
AWS_DEFAULT_REGION: "us-west-2"
DEPLOY_ENV_CONFIG_FILE: ${{ contains(matrix.test_dir, 'diskgraph') && 'rest/devserver/configs/diskgraph-test.json' || '' }}

Expand All @@ -218,7 +202,5 @@ jobs:
FULL_TEST=1 GLOG_logtostdout=1 python -m coverage run -m pytest "${PYTEST_CI_OUTPUT_ARGS[@]}" --durations=30 --runsnowflake ${{ matrix.test_dir }}
env:
SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }}
GCS_TEST_BUCKET: 'kumo-test-bucket'
KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED: ${{ env.KUMO_RUN_GCP_TESTS == '1' && 'true' || 'false' }}
AWS_DEFAULT_REGION: "us-west-2"
DEPLOY_ENV_CONFIG_FILE: ${{ contains(matrix.test_dir, 'diskgraph') && 'rest/devserver/configs/diskgraph-test.json' || '' }}
17 changes: 15 additions & 2 deletions kumo/testing/bigquery.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,14 @@
from collections.abc import Callable
from typing import Optional

from io_utils import logging
from io_utils import fsutil, logging
from io_utils.secrets import get_secrets
from kumo.connector.bigquery_connector import (
ENV_GCS_KUMO_BUCKET,
ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH,
ENV_GOOGLE_CREDENTIALS_JSON,
)
from service_framework.env_var import EnvVar

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -49,10 +50,22 @@ def _configure_application_default_credentials(
tests select a restricted service account through IAM impersonation rather
than loading a second private key.
"""
require_wif = EnvVar('KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED', bool,
default_value=False).parse_value()
credentials_path = os.getenv('GOOGLE_APPLICATION_CREDENTIALS')
if not credentials_path or not os.path.exists(credentials_path):
if not credentials_path or not fsutil.exists(credentials_path):
if require_wif:
raise RuntimeError(
'Workload exchange BigQuery tests require federated ADC in '
'GOOGLE_APPLICATION_CREDENTIALS')
return False

if require_wif:
credentials = json.loads(fsutil.read_text(credentials_path))
if credentials.get('type') != 'external_account':
raise RuntimeError(
'Workload exchange BigQuery tests require external_account ADC')

impersonation_env = _IMPERSONATION_ENV_BY_KEY.get(service_account_key)
if impersonation_env is None:
os.environ.pop(ENV_GOOGLE_IMPERSONATE_SERVICE_ACCOUNT, None)
Expand Down
37 changes: 14 additions & 23 deletions kumo/testing/gcs.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
# SPDX-License-Identifier: Apache-2.0

import base64
import inspect
import json
import os
from collections.abc import Callable
Expand Down Expand Up @@ -42,6 +41,12 @@ def setup_gcs_credentials():

if ENV_GCS_TEST_BUCKET not in os.environ:
os.environ[ENV_GCS_TEST_BUCKET] = 'kumo-test-bucket'
# CI supplies short-lived ADC. Never replace it with a stored key, and
# reuse the selected credentials when another test requests this setup.
credentials_path = os.getenv('GOOGLE_APPLICATION_CREDENTIALS')
if credentials_path:
os.environ[ENV_GOOGLE_CREDENTIALS_JSON] = credentials_path
return
google_creds = os.getenv("GOOGLE_CREDENTIALS_STR_JSON")
if google_creds:
logger.info("Using GOOGLE_CREDENTIALS_STR_JSON environment variable")
Expand Down Expand Up @@ -108,31 +113,17 @@ def _check_gcs_credentials() -> bool:


def withGCSCredentials(func: Callable):
"""Decorator that sets up GCS credentials and skips tests if unavailable.
"""Declare credentials without performing IO during pytest collection.

This decorator:
1. Calls setup_gcs_credentials() to fetch and configure credentials
2. Skips the test if credentials are not available
3. Allows tests to run in CI with credentials but skip locally without them
The test setup hook resolves this prerequisite after skip selection and
before resource fixtures, including session workers, are initialized.
"""
return pytest.mark.gcs_credentials(func)

Args:
func: The test function to decorate

Example:
@withGCSCredentials
def test_gcs_operation():
# Test code here
pass
"""
# A collection-time mark prevents live GCS fixtures from being resolved
# when credentials are absent. A runtime wrapper skips too late: pytest
# sets up fixtures before invoking the wrapped test.
credentials_missing = not _check_gcs_credentials()
reason = "No GCS credentials found"
if inspect.isgeneratorfunction(func):
from kumo.testing._skip import skip_if
return skip_if(credentials_missing, reason, func)
return pytest.mark.skipif(credentials_missing, reason=reason)(func)
def require_gcs_credentials() -> None:
if not _check_gcs_credentials():
pytest.skip('No GCS credentials found')


def withMockedGCSFilesystem(func: Callable):
Expand Down
15 changes: 15 additions & 0 deletions test/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@
from kumo.graph.table import Table # noqa: E402
from kumo.task.task import Task # noqa: E402
from kumo.testing import withDatabricksCredentials # noqa: E402
from kumo.testing.gcs import require_gcs_credentials # noqa: E402
from kumo.testing.generator import EdgeSpec, GraphSpec, TableSpec # noqa: E402
from kumo.testing.snowflake import setup_snowflake_credentials # noqa: E402
from kumo.utils.embedding_models import embedding_model_store # noqa: E402
Expand Down Expand Up @@ -190,7 +191,21 @@ def pytest_runtest_makereport(item, call):
_flake_diagnostics.run_on_failure(item.nodeid, tc)


def pytest_runtest_setup(item: pytest.Item) -> None:
# Pytest's tryfirst skip hook runs before this normal setup hook; its
# fixture setup runs afterward. Skipped Snowflake targets never load GCP
# credentials, and selected GCS tests configure them before session workers.
if item.get_closest_marker('gcs_credentials') is not None:
require_gcs_credentials()


@pytest.fixture
def gcs_credentials() -> None:
require_gcs_credentials()


def pytest_configure(config):
config.addinivalue_line('markers', 'gcs_credentials: live GCS prerequisite')
# register nightly marker
config.addinivalue_line("markers",
("nightly(name): Used to mark long-running"
Expand Down
4 changes: 1 addition & 3 deletions test/io_utils/fsutil/test_all.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@

from io_utils import fsutil
from kumo.testing.adls import adls_test_base_uri, withADLSCredentials
from kumo.testing.gcs import withGCSCredentials
from kumo.testing.s3 import withS3Credentials


Expand All @@ -30,8 +29,7 @@ def root_s3() -> Generator[str, None, None]:


@pytest.fixture
@withGCSCredentials
def root_gcs() -> Generator[str, None, None]:
def root_gcs(gcs_credentials) -> Generator[str, None, None]:
from kumo.testing.gcs import ENV_GCS_TEST_BUCKET
bucket = os.getenv(ENV_GCS_TEST_BUCKET, 'kumo-test-bucket')
yield f"gs://{bucket}/fsutil-test/"
Expand Down
74 changes: 74 additions & 0 deletions test/scripts/test_gcp_ci_workflows.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: Apache-2.0

from types import SimpleNamespace

import pytest
import yaml

from io_utils import fsutil


def workflow(name):
return yaml.safe_load(fsutil.read_text(f'.github/workflows/{name}.yml'))


def evaluate(expression, repository, event, source_repository, target):
expression = expression.removeprefix('${{').removesuffix('}}').strip()
expression = expression.replace('&&', 'and').replace('||', 'or')
github = SimpleNamespace(
repository=repository, event_name=event,
event=SimpleNamespace(pull_request=SimpleNamespace(
head=SimpleNamespace(repo=SimpleNamespace(
full_name=source_repository)))))
return eval(expression, {'__builtins__': {}}, {
'github': github, 'matrix': SimpleNamespace(test_dir=target),
'contains': lambda value, substring: substring in value,
})


@pytest.mark.parametrize('repository', ['NVIDIA/kumo-engine', 'other/repo'])
@pytest.mark.parametrize('event,source', [
('pull_request', 'NVIDIA/kumo-engine'), ('pull_request', 'other/fork'),
('push', ''), ('merge_group', ''), ('workflow_dispatch', ''),
])
@pytest.mark.parametrize('target,gcp', [
('test/io_utils/', True), ('test/connector/', True),
('--splits 2 --group 1 test/rest/test_*.py', True),
('test/data/', False),
])
def test_general_auth_routing(repository, event, source, target, gcp):
job = workflow('testing')['jobs']['pytest']
eligible = (repository == 'NVIDIA/kumo-engine' and gcp
and (event != 'pull_request' or source == repository))
enabled = evaluate(job['env']['KUMO_RUN_GCP_TESTS'], repository,
event, source, target)
environment = evaluate(job['environment']['name'], repository,
event, source, target)

assert enabled == ('1' if eligible else '0')
assert environment == ('gha-kumo-workflow-tests' if eligible
else 'azure-adls-tests')


def test_snowflake_has_no_google():
job = workflow('testing_snowflake')['jobs']['pytest']
assert not any('google-github-actions/auth' in step.get('uses', '')
for step in job['steps'])
assert job['name'] == 'all-snowflake-tests'


def test_gcs_runs_every_change():
document = workflow('testing_gcs')
# YAML 1.1 interprets the workflow key "on" as True.
triggers = document[True]
assert 'pull_request' in triggers
assert 'paths-ignore' not in triggers['push']
job = document['jobs']['pytest-gcs']
assert not job.get('continue-on-error', False)
assert job['name'] == 'gcs-e2e-tests'
steps = job['steps']
assert not any('changed-files' in step.get('uses', '') for step in steps)
run = next(step for step in steps if step['name'] == 'Run E2E tests')
assert 'if' not in run
Loading
Loading