reverted core_deps changes

Author: daniel-sanche

.github/workflows/unittest.yml

@@ -40,33 +40,15 @@ jobs:
       # Each weight is roughly 1 minute of expected execution time
       # Default for unset packages is 1
       PACKAGE_WEIGHTS: |
-        django-google-spanner: 2
-        gapic-generator: 3
-        google-ai-generativelanguage: 3
-        google-api-core: 2
-        google-auth: 2
-        google-cloud-bigquery: 4
-        google-cloud-bigquery-storage: 2
-        google-cloud-bigtable: 4
+        google-ai-generativelanguage: 4
+        google-auth: 5
         google-cloud-compute: 12
         google-cloud-compute-v1beta: 12
-        google-cloud-datastore: 3
-        google-cloud-dialogflow: 4
-        google-cloud-dialogflow-cx: 4
-        google-cloud-discoveryengine: 4
-        google-cloud-firestore: 3
-        google-cloud-logging: 3
-        google-cloud-monitoring: 3
-        google-cloud-ndb: 2
-        google-cloud-pubsub: 2
-        google-cloud-retail: 3
-        google-cloud-spanner: 4
-        google-cloud-storage: 4
-        google-shopping-merchant-accounts: 3
-        pandas-gbq: 2
-        proto-plus: 1
-        sqlalchemy-bigquery: 2
-        sqlalchemy-spanner: 2
+        google-cloud-dialogflow: 6
+        google-cloud-dialogflow-cx: 6
+        google-cloud-discoveryengine: 8
+        google-cloud-retail: 5
+        google-shopping-merchant-accounts: 4
     steps:
     - name: Checkout
       uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
@@ -174,52 +156,12 @@ jobs:
         path: .coverage.${{ matrix.python }}.*
         include-hidden-files: true
 
-  core-deps:
-    needs: initialize
-    if: needs.initialize.outputs.matrix != '[]' && needs.initialize.outputs.matrix != ''
-    runs-on: ubuntu-22.04
-    strategy:
-      fail-fast: true
-      matrix:
-        python: ["3.14"]
-        package_shard: ${{ fromJson(needs.initialize.outputs.matrix) }}
-    name: ${{ matrix.package_shard.is_sharded && format('core-deps handwritten ({0}, {1})', matrix.python, matrix.package_shard.name) || format('core-deps handwritten ({0})', matrix.python) }}
-    steps:
-    - name: Checkout
-      uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
-      # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base`
-      # See https://github.com/googleapis/google-cloud-python/issues/12013
-      # and https://github.com/actions/checkout#checkout-head.
-      with:
-        fetch-depth: 2
-        persist-credentials: false
-    - name: Setup Python
-      uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
-      with:
-        python-version: ${{ matrix.python }}
-        cache: 'pip'
-        allow-prereleases: true
-    - name: Install nox
-      run: |
-        pip install nox
-    - name: Run core_deps_from_source for ${{ matrix.package_shard.description }}
-      env:
-        BUILD_TYPE: presubmit
-        TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }}
-        TEST_TYPE: core_deps_from_source
-        PY_VERSION: ${{ matrix.python }}
-        PACKAGE_LIST: ${{ matrix.package_shard.packages }}
-        NOX_DEFAULT_VENV_BACKEND: "virtualenv"
-        NOXFORCEPYTHON: ${{ matrix.python }}
-      run: |
-        ci/run_conditional_tests.sh
-
   all-tests:
-    needs: [initialize, unit, core-deps]
+    needs: [initialize, unit]
     if: always()
     runs-on: ubuntu-latest
     steps:
-      - name: Check test results
+      - name: Check unit test results
         run: |
           # 1. Check initialize job
           if [[ "${{ needs.initialize.result }}" != "success" ]]; then
@@ -231,12 +173,7 @@ jobs:
             echo "Unit tests failed"
             exit 1
           fi
-          # 3. Check core dependencies test shards
-          if [[ "${{ needs.core-deps.result }}" != "success" && "${{ needs.core-deps.result }}" != "skipped" ]]; then
-            echo "Core dependencies tests failed"
-            exit 1
-          fi
-          echo "All unit and core dependencies tests passed or were skipped"
+          echo "All unit tests passed or were skipped"
 
   cover:
     if: always() && !cancelled() && needs.all-tests.result == 'success'

ci/get_package_shards.py

@@ -70,11 +70,9 @@ def get_package_weights():
     return weights
 
 
-def get_packages(handwritten_only=False):
+def get_packages():
     """Lists all package directory paths in the repository grouped by package name.
 
-    If handwritten_only is True, includes only non-GAPIC_AUTO libraries.
-
     Returns:
         dict: A dictionary mapping package_name -> list of relative directory paths.
     """
@@ -85,19 +83,8 @@ def get_packages(handwritten_only=False):
             continue
         for d in os.listdir(subdir):
             full_path = os.path.join(subdir, d) + '/'
-            if not os.path.isdir(full_path):
-                continue
-            if handwritten_only:
-                meta_file = os.path.join(full_path, ".repo-metadata.json")
-                if os.path.exists(meta_file):
-                    try:
-                        with open(meta_file) as f:
-                            data = json.load(f)
-                            if data.get("library_type") == "GAPIC_AUTO":
-                                continue
-                    except Exception:
-                        pass
-            packages_map[d].append(full_path)
+            if os.path.isdir(full_path):
+                packages_map[d].append(full_path)
     return packages_map
 
 
@@ -145,22 +132,6 @@ def get_packages_to_test():
                 if full_path not in to_test_paths[pkg_name]:
                     to_test_paths[pkg_name].append(full_path)
 
-    # Core dependency packages whose changes require testing across downstream handwritten packages
-    core_packages = {
-        "google-api-core",
-        "google-auth",
-        "google-auth-httplib2",
-        "google-auth-oauthlib",
-        "google-cloud-core",
-        "googleapis-common-protos",
-        "grpc-google-iam-v1",
-        "proto-plus",
-        "google-crc32c",
-    }
-    if any(pkg in core_packages for pkg in to_test_paths):
-        # When a core package changes, test all handwritten packages (non-GAPIC_AUTO)
-        return get_packages(handwritten_only=True)
-
     return dict(to_test_paths)
 
 

ci/run_single_test.sh

@@ -68,14 +68,6 @@ case ${TEST_TYPE} in
         nox -s prerelease_deps-3.14
         retval=$?
         ;;
-    core_deps_from_source)
-        if [[ "$(pwd)" == */preview-packages/* ]]; then
-            echo "Skipping core_deps_from_source for preview package $(pwd)"
-            exit 0
-        fi
-        nox --stop-on-first-error -s core_deps_from_source
-        retval=$?
-        ;;
     unit)
         case ${PY_VERSION} in
         "3.10")

packages/db-dtypes/noxfile.py

@@ -512,27 +512,16 @@ def core_deps_from_source(session, protobuf_implementation):
     install_unittest_dependencies(session, "-c", constraints_path)
 
     core_dependencies_from_source = [
-        "googleapis-common-protos",
-        "google-api-core",
-        "google-auth",
-        "grpc-google-iam-v1",
-        "proto-plus",
+        "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos",
+        "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core",
+        "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth",
+        "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1",
+        "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus",
     ]
 
-    deps_dir = CURRENT_DIRECTORY.parent
-    while deps_dir.name != "packages" and deps_dir.parent != deps_dir:
-        deps_dir = deps_dir.parent
-
-    local_paths = [
-        str(deps_dir / dep)
-        for dep in core_dependencies_from_source
-        if (deps_dir / dep).exists()
-    ]
-    if local_paths:
-        session.install(*local_paths, "--no-deps", "--ignore-installed")
-        print(
-            f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}"
-        )
+    for dep in core_dependencies_from_source:
+        session.install(dep, "--no-deps", "--ignore-installed")
+        print(f"Installed {dep}")
 
     tests_path = os.path.join("tests", "unit")
     session.run(

packages/gapic-generator/gapic/templates/noxfile.py.j2

@@ -6,8 +6,9 @@ import os
 import pathlib
 import re
 import shutil
-import warnings
+
 from typing import Dict, List
+import warnings
 
 import nox
 
@@ -162,7 +163,7 @@ def lint(session):
         "ruff", "format",
         "--check",
         f"--target-version=py{ALL_PYTHON[0].replace('.', '')}",
-        "--line-length=88",
+        "--line-length=88", 
         *LINT_PATHS,
     )
 
@@ -178,7 +179,7 @@ def lint(session):
 def blacken(session):
     """(Deprecated) Legacy session. Please use 'nox -s format'."""
     session.log("WARNING: The 'blacken' session is deprecated and will be removed in a future release. Please use 'nox -s format' in the future.")
-
+    
     # Just run the ruff formatter (keeping legacy behavior of only formatting, not sorting imports)
     session.install(RUFF_VERSION)
     session.run(
@@ -517,14 +518,14 @@ def prerelease_deps(session, protobuf_implementation):
     # Extract the base package name, safely ignoring version bounds and spaces
     # (e.g., "grpcio>=1.75.1" becomes "grpcio")
     parsed_deps = {
-        dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1)
+        dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) 
         for dep in prerel_deps
     }
 
     # Dynamically sort local packages vs PyPI dependencies
     local_paths = []
     pypi_deps = []
-
+    
     for dep, pkg_name in parsed_deps.items():
         if (deps_dir / pkg_name).exists():
             local_paths.append(str(deps_dir / pkg_name))
@@ -623,16 +624,13 @@ def core_deps_from_source(session, protobuf_implementation):
         "proto-plus",
     ]
 
-    # Locate the monorepo 'packages' directory containing core dependencies
-    deps_dir = next(
-        p / "packages"
-        for p in CURRENT_DIRECTORY.parents
-        if (p / "packages").is_dir()
-    )
+    deps_dir = CURRENT_DIRECTORY.parent
+    while deps_dir.name != "packages" and deps_dir.parent != deps_dir:
+        deps_dir = deps_dir.parent
 
     # Batch the pip installation to avoid sequential overhead
     dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source]
-
+    
     session.install(*dep_paths, "--no-deps", "--ignore-installed")
     print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}")
 

packages/gapic-generator/tests/integration/goldens/asset/noxfile.py

@@ -17,8 +17,9 @@
 import pathlib
 import re
 import shutil
-import warnings
+
 from typing import Dict, List
+import warnings
 
 import nox
 
@@ -615,12 +616,9 @@ def core_deps_from_source(session, protobuf_implementation):
         "proto-plus",
     ]
 
-    # Locate the monorepo 'packages' directory containing core dependencies
-    deps_dir = next(
-        p / "packages"
-        for p in CURRENT_DIRECTORY.parents
-        if (p / "packages").is_dir()
-    )
+    deps_dir = CURRENT_DIRECTORY.parent
+    while deps_dir.name != "packages" and deps_dir.parent != deps_dir:
+        deps_dir = deps_dir.parent
 
     # Batch the pip installation to avoid sequential overhead
     dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source]

packages/gapic-generator/tests/integration/goldens/credentials/noxfile.py

@@ -17,8 +17,9 @@
 import pathlib
 import re
 import shutil
-import warnings
+
 from typing import Dict, List
+import warnings
 
 import nox
 
@@ -615,12 +616,9 @@ def core_deps_from_source(session, protobuf_implementation):
         "proto-plus",
     ]
 
-    # Locate the monorepo 'packages' directory containing core dependencies
-    deps_dir = next(
-        p / "packages"
-        for p in CURRENT_DIRECTORY.parents
-        if (p / "packages").is_dir()
-    )
+    deps_dir = CURRENT_DIRECTORY.parent
+    while deps_dir.name != "packages" and deps_dir.parent != deps_dir:
+        deps_dir = deps_dir.parent
 
     # Batch the pip installation to avoid sequential overhead
     dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source]

packages/gapic-generator/tests/integration/goldens/eventarc/noxfile.py

@@ -17,8 +17,9 @@
 import pathlib
 import re
 import shutil
-import warnings
+
 from typing import Dict, List
+import warnings
 
 import nox
 
@@ -615,12 +616,9 @@ def core_deps_from_source(session, protobuf_implementation):
         "proto-plus",
     ]
 
-    # Locate the monorepo 'packages' directory containing core dependencies
-    deps_dir = next(
-        p / "packages"
-        for p in CURRENT_DIRECTORY.parents
-        if (p / "packages").is_dir()
-    )
+    deps_dir = CURRENT_DIRECTORY.parent
+    while deps_dir.name != "packages" and deps_dir.parent != deps_dir:
+        deps_dir = deps_dir.parent
 
     # Batch the pip installation to avoid sequential overhead
     dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source]

packages/gapic-generator/tests/integration/goldens/logging/noxfile.py

@@ -17,8 +17,9 @@
 import pathlib
 import re
 import shutil
-import warnings
+
 from typing import Dict, List
+import warnings
 
 import nox
 
@@ -615,12 +616,9 @@ def core_deps_from_source(session, protobuf_implementation):
         "proto-plus",
     ]
 
-    # Locate the monorepo 'packages' directory containing core dependencies
-    deps_dir = next(
-        p / "packages"
-        for p in CURRENT_DIRECTORY.parents
-        if (p / "packages").is_dir()
-    )
+    deps_dir = CURRENT_DIRECTORY.parent
+    while deps_dir.name != "packages" and deps_dir.parent != deps_dir:
+        deps_dir = deps_dir.parent
 
     # Batch the pip installation to avoid sequential overhead
     dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source]