From da306c4433a4262d7c2fc04f597001336d683941 Mon Sep 17 00:00:00 2001 From: Kyle Montemayor Date: Wed, 20 May 2026 20:41:03 +0000 Subject: [PATCH] Migrate RESOURCE_CONFIG_PATH env var to GIGL_RESOURCE_CONFIG_URI Aligns the resource-config env var with the GIGL_* convention introduced in #642 and the existing GIGL_RESOURCE_CONFIG_URI_ENV_KEY constant. All writers (get_resource_config, launch_graph_store_local.py, Makefile's notebooks_test target) now set the new name; readers fall back to the deprecated RESOURCE_CONFIG_PATH with a one-time warning so externally-set values keep working for one release cycle. Co-Authored-By: Claude Opus 4.7 (1M context) --- Makefile | 2 +- gigl/env/constants.py | 49 +++++++++++++++++-- gigl/env/pipelines_config.py | 15 ++++-- gigl/src/common/constants/resource_config.py | 1 - .../types/pb_wrappers/gigl_resource_config.py | 7 ++- scripts/launch_graph_store_local.py | 3 +- 6 files changed, 60 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index f80eb2952..93ab75ffc 100644 --- a/Makefile +++ b/Makefile @@ -135,7 +135,7 @@ integration_test: build_cpp_extensions notebooks_test: - RESOURCE_CONFIG_PATH=${GIGL_TEST_DEFAULT_RESOURCE_CONFIG} python -m tests.config_tests.notebooks_test + GIGL_RESOURCE_CONFIG_URI=${GIGL_TEST_DEFAULT_RESOURCE_CONFIG} python -m tests.config_tests.notebooks_test mock_assets: uv run python -m gigl.src.mocking.dataset_asset_mocking_suite --resource_config_uri="deployment/configs/e2e_cicd_resource_config.yaml" --env test diff --git a/gigl/env/constants.py b/gigl/env/constants.py index 13686c64d..ce3b9da95 100644 --- a/gigl/env/constants.py +++ b/gigl/env/constants.py @@ -1,11 +1,21 @@ -"""Environment-variable keys exported by ``launch_custom``. +"""Environment-variable keys used across GiGL. -These keys are set on the subprocess env (never on the parent -``os.environ``) by ``gigl.src.common.custom_launcher.launch_custom`` so -that receiving CLIs can ``os.environ.get(...)`` their runtime context. +Most of these keys are set on subprocess env (never on the parent +``os.environ``) by ``gigl.src.common.custom_launcher.launch_custom`` so that +receiving CLIs can ``os.environ.get(...)`` their runtime context. + +``GIGL_RESOURCE_CONFIG_URI`` is also written to the parent ``os.environ`` by +``gigl.env.pipelines_config.get_resource_config`` so that downstream readers +(e.g. ``GiglResourceConfigWrapper.get_resource_config_uri``) can recover the +value within the same process. Use :func:`read_resource_config_uri_from_env` +to read it; that helper also falls back to the deprecated +``RESOURCE_CONFIG_PATH`` with a one-time warning. """ -from typing import Final +import os +from typing import Final, Optional + +from gigl.common.logger import Logger GIGL_APPLIED_TASK_IDENTIFIER_ENV_KEY: Final[str] = "GIGL_APPLIED_TASK_IDENTIFIER" GIGL_TASK_CONFIG_URI_ENV_KEY: Final[str] = "GIGL_TASK_CONFIG_URI" @@ -13,3 +23,32 @@ GIGL_CPU_DOCKER_URI_ENV_KEY: Final[str] = "GIGL_CPU_DOCKER_URI" GIGL_CUDA_DOCKER_URI_ENV_KEY: Final[str] = "GIGL_CUDA_DOCKER_URI" GIGL_COMPONENT_ENV_KEY: Final[str] = "GIGL_COMPONENT" + +_LEGACY_RESOURCE_CONFIG_ENV_KEY: Final[str] = "RESOURCE_CONFIG_PATH" +_legacy_resource_config_env_warned: bool = False +_logger = Logger() + + +def read_resource_config_uri_from_env() -> Optional[str]: + """Read the resource-config URI from the environment. + + Prefers ``GIGL_RESOURCE_CONFIG_URI``. Falls back to the deprecated + ``RESOURCE_CONFIG_PATH`` and emits a one-time warning if that path is taken. + + Returns: + The URI string if set under either name, else ``None``. + """ + global _legacy_resource_config_env_warned + value = os.environ.get(GIGL_RESOURCE_CONFIG_URI_ENV_KEY) + if value is not None: + return value + + legacy_value = os.environ.get(_LEGACY_RESOURCE_CONFIG_ENV_KEY) + if legacy_value is not None and not _legacy_resource_config_env_warned: + _logger.warning( + f"Environment variable {_LEGACY_RESOURCE_CONFIG_ENV_KEY!r} is deprecated; " + f"use {GIGL_RESOURCE_CONFIG_URI_ENV_KEY!r} instead. " + "Support for the legacy name will be removed in a future release." + ) + _legacy_resource_config_env_warned = True + return legacy_value diff --git a/gigl/env/pipelines_config.py b/gigl/env/pipelines_config.py index 867855882..a4ec4eacf 100644 --- a/gigl/env/pipelines_config.py +++ b/gigl/env/pipelines_config.py @@ -5,6 +5,10 @@ from gigl.common import Uri, UriFactory from gigl.common.logger import Logger +from gigl.env.constants import ( + GIGL_RESOURCE_CONFIG_URI_ENV_KEY, + read_resource_config_uri_from_env, +) from gigl.src.common.types.pb_wrappers.gigl_resource_config import ( GiglResourceConfigWrapper, ) @@ -55,8 +59,9 @@ def get_resource_config( Args: resource_config_uri: Optional[Uri] = None The URI of the resource config file. If None, the function will try to load the resource config from the - command-line argument --resource_config_uri or the environment variable RESOURCE_CONFIG_PATH. If these are - not set, the function will try to load the resource config from the pipeline options. + command-line argument --resource_config_uri or the environment variable GIGL_RESOURCE_CONFIG_URI (the + deprecated RESOURCE_CONFIG_PATH is still read as a fallback, with a one-time warning). If these are not + set, the function will try to load the resource config from the pipeline options. Returns: resource_config: GiglResourceConfigWrapper @@ -77,8 +82,8 @@ def get_resource_config( required=False, ) args, _ = parser.parse_known_args() - resource_config_str = args.resource_config_uri or os.getenv( - "RESOURCE_CONFIG_PATH" + resource_config_str = ( + args.resource_config_uri or read_resource_config_uri_from_env() ) if resource_config_str is None: @@ -90,7 +95,7 @@ def get_resource_config( "No resource config provided, either via command-line argument or environment variable." ) - os.environ["RESOURCE_CONFIG_PATH"] = resource_config_str + os.environ[GIGL_RESOURCE_CONFIG_URI_ENV_KEY] = resource_config_str resource_config_path = UriFactory.create_uri(uri=resource_config_str) from gigl.common.utils.proto_utils import ProtoUtils diff --git a/gigl/src/common/constants/resource_config.py b/gigl/src/common/constants/resource_config.py index 84ec3cd63..df31aace3 100644 --- a/gigl/src/common/constants/resource_config.py +++ b/gigl/src/common/constants/resource_config.py @@ -1,3 +1,2 @@ SPARK_SPLIT_GENERATOR_CONFIG = "spark_split_generator_config" DATAFLOW_SPLIT_GENERATOR_CONFIG = "dataflow_split_generator_config" -RESOURCE_CONFIG_OS_ENV = "RESOURCE_CONFIG_PATH" diff --git a/gigl/src/common/types/pb_wrappers/gigl_resource_config.py b/gigl/src/common/types/pb_wrappers/gigl_resource_config.py index 8d944c865..6a5d0b40b 100644 --- a/gigl/src/common/types/pb_wrappers/gigl_resource_config.py +++ b/gigl/src/common/types/pb_wrappers/gigl_resource_config.py @@ -1,11 +1,10 @@ import argparse -import os from dataclasses import dataclass from typing import Optional, Union -import gigl.src.common.constants.resource_config as resource_config_constants from gigl.common import GcsUri, UriFactory from gigl.common.logger import Logger +from gigl.env.constants import read_resource_config_uri_from_env from gigl.src.common.constants.components import GiGLComponents from snapchat.research.gbml.gigl_resource_config_pb2 import ( CustomLauncherConfig, @@ -108,8 +107,8 @@ def get_resource_config_uri(self) -> str: ) args, _ = parser.parse_known_args() - resource_config_path = args.resource_config_uri or os.getenv( - resource_config_constants.RESOURCE_CONFIG_OS_ENV + resource_config_path = ( + args.resource_config_uri or read_resource_config_uri_from_env() ) return str(resource_config_path) diff --git a/scripts/launch_graph_store_local.py b/scripts/launch_graph_store_local.py index 68ca0dad8..368a57dcd 100644 --- a/scripts/launch_graph_store_local.py +++ b/scripts/launch_graph_store_local.py @@ -67,6 +67,7 @@ import yaml from gigl.distributed.utils import get_free_ports +from gigl.env.constants import GIGL_RESOURCE_CONFIG_URI_ENV_KEY DEFAULT_JOB_ROOT = Path("/tmp/gigl") JOB_ID_SUFFIX_RE = re.compile(r"^(.*?)(\d+)$") @@ -735,7 +736,7 @@ def main(argv: Optional[list[str]] = None) -> int: "MASTER_ADDR": args.host, "MASTER_PORT": str(master_port), "WORLD_SIZE": str(world_size), - "RESOURCE_CONFIG_PATH": resource_config_uri, + GIGL_RESOURCE_CONFIG_URI_ENV_KEY: resource_config_uri, "COMPUTE_CLUSTER_LOCAL_WORLD_SIZE": str(args.compute_procs_per_node), "PYTHONUNBUFFERED": "1", }