ci: publish cytnx to PyPI on tag (stable) and master push (.devN nightly)

.github/workflows/release_pypi.yml

@@ -1,10 +1,13 @@
 name: Release to PyPI
 
+# Official `cytnx` release pipeline. Triggers only on `v*` tag pushes
+# and manual workflow_dispatch; the resulting wheels are published to
+# production PyPI in both cases. Nightly builds against master are
+# handled by release_pypi_nightly.yml and publish under a separate
+# nightly project.
+
 on:
-  pull_request:
   push:
-    branches:
-      - master
     tags:
       - "v*"
   workflow_dispatch:
@@ -20,58 +23,32 @@ jobs:
         # newer images because Homebrew dylib minos can force a higher wheel
         # deployment target and break delocate compatibility for lower targets.
         os: [ubuntu-24.04, ubuntu-24.04-arm, macos-14, macos-15-intel]
-    defaults:
-      run:
-        shell: bash -el {0}
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4.3.1
       with:
         fetch-depth: 0
         submodules: recursive
 
-    # When re-running a PR job, GitHub Actions uses the merge commit created at
-    # the time of the original run, not a fresh merge with the latest target branch.
-    # This step ensures we always test against the most recent target branch.
-    - name: Merge with latest target branch (pull_request only)
-      if: github.event_name == 'pull_request'
-      run: |
-        git fetch origin ${{ github.event.pull_request.base.ref }}
-        git merge --no-edit origin/${{ github.event.pull_request.base.ref }}
-        # Refresh submodules in case the merge moved any gitlinks (e.g. cmake_modules/morse_cmake).
-        git submodule update --init --recursive
-
-    - name: Cache Ccache Directory (pull_request)
-      if: ${{ github.event_name == 'pull_request' }}
-      uses: actions/cache@v4
-      with:
-        path: |
-          ~/.ccache
-        key: ccache-wheel-${{ runner.os }}-${{ github.event.pull_request.head.ref }}-${{ github.sha }}
-        restore-keys: |
-          ccache-wheel-${{ runner.os }}-${{ github.event.pull_request.head.ref }}-
-          ccache-wheel-${{ runner.os }}-${{ github.event.pull_request.base.ref }}-
-
-    - name: Cache Ccache Directory (push)
-      if: ${{ github.event_name == 'push' }}
-      uses: actions/cache@v4
+    - name: Cache Ccache Directory
+      uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830  # v4.3.0
       with:
         path: |
           ~/.ccache
         key: ccache-wheel-${{ runner.os }}-${{ github.ref_name }}-${{ github.sha }}
         restore-keys: |
           ccache-wheel-${{ runner.os }}-${{ github.ref_name }}-
+          ccache-wheel-${{ runner.os }}-
 
     - name: Set Ccache Directory
       run: |
-        echo "Start building---------------------------------"
         # HOST_CCACHE_DIR is consumed only by the docker bind-mount source
         # below; the in-container CCACHE_DIR is set in the Build Wheels
         # step's env: block (and forwarded via environment-pass).
         echo "HOST_CCACHE_DIR=${HOME}/.ccache" >> "$GITHUB_ENV"
 
     - name: Build Wheels
-      uses: pypa/cibuildwheel@v3.3.0
+      uses: pypa/cibuildwheel@63fd63b352a9a8bdcc24791c9dbee952ee9a8abc  # v3.3.0
       env:
         CMAKE_C_COMPILER_LAUNCHER: ccache
         CMAKE_CXX_COMPILER_LAUNCHER: ccache
@@ -94,28 +71,25 @@ jobs:
         CIBW_TEST_COMMAND: "python {project}/tools/validate_ccache_stats.py"
 
 
-    - uses: actions/upload-artifact@v4
+    - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
       with:
         name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }}
         path: ./wheelhouse/*.whl
 
-  ReleaseTestPyPI:
-    name: ReleaseWheel-TestPyPI
+  ReleasePyPI:
+    name: ReleasePyPI
     needs: BuildWheel
-    if: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) }}
     runs-on: ubuntu-24.04
     permissions:
       id-token: write
     steps:
-    - uses: actions/download-artifact@v4
+    - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
       with:
         pattern: cibw-wheels-*
         path: dist
         merge-multiple: true
 
-    - name: Publish package distributions to TestPyPI (pull_request and tag push)
-      if: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) }}
-      uses: pypa/gh-action-pypi-publish@release/v1
+    - name: Publish package distributions to PyPI (tag push or manual dispatch)
+      uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b  # release/v1
       with:
-        repository-url: https://test.pypi.org/legacy/
         packages-dir: dist

.github/workflows/release_pypi_nightly.yml

@@ -0,0 +1,112 @@
+name: Nightly Release to PyPI
+
+# Builds and publishes a `cytnx` dev wheel set to PyPI on every push to
+# master (i.e. every PR merge), so that `pip install --pre cytnx`
+# tracks the latest master while `pip install cytnx` keeps resolving
+# to the most recent tagged release. Wheels share the official `cytnx`
+# PyPI project name but use a PEP 440 dev version
+# (MAJOR.MINOR.PATCH.devYYYYMMDDHHMM) stamped into pyproject.toml by
+# tools/prepare_nightly_release.py at build time.
+
+on:
+  push:
+    branches:
+      - master
+  workflow_dispatch:
+
+jobs:
+  BuildNightlyWheel:
+    name: BuildNightlyWheel-${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-24.04, ubuntu-24.04-arm, macos-14, macos-15-intel]
+
+    steps:
+    - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4.3.1
+      with:
+        fetch-depth: 0
+        submodules: recursive
+
+    - name: Install nightly stamping helper
+      # Installs the `release-tools` PEP 735 dependency-group from
+      # pyproject.toml (currently just `tomlkit`). Using `--group`
+      # rather than `pip install .[release-tools]` keeps the source
+      # of truth for build-pipeline deps in pyproject.toml without
+      # triggering scikit-build-core to compile cytnx itself before
+      # cibuildwheel does so under its own isolated environment.
+      # `--group` requires pip 25.1+; the runner-bundled pip is
+      # upgraded first to make the requirement portable across
+      # runner image versions.
+      run: |
+        python -m pip install --upgrade pip
+        pip install --group release-tools
+
+    - name: Stamp pyproject.toml for nightly release
+      # Rewrites pyproject.toml to set a static
+      # MAJOR.MINOR.PATCH.devYYYYMMDDHHMM version and exports
+      # CYTNX_VERSION_TAG to $GITHUB_ENV so subsequent steps (and the
+      # cibuildwheel build) can append the same suffix to
+      # cytnx.__version__ via CMake.
+      run: python tools/prepare_nightly_release.py
+
+    - name: Cache Ccache Directory
+      # Share the ccache prefix with release_pypi.yml: nightlies and
+      # tagged releases build the same C++ sources, so a cache primed
+      # by either feeds the other.
+      uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830  # v4.3.0
+      with:
+        path: |
+          ~/.ccache
+        key: ccache-wheel-${{ runner.os }}-${{ github.ref_name }}-${{ github.sha }}
+        restore-keys: |
+          ccache-wheel-${{ runner.os }}-${{ github.ref_name }}-
+          ccache-wheel-${{ runner.os }}-
+
+    - name: Set Ccache Directory
+      run: |
+        # HOST_CCACHE_DIR is consumed only by the docker bind-mount source
+        # below; the in-container CCACHE_DIR is set in the Build Wheels
+        # step's env: block (and forwarded via environment-pass).
+        echo "HOST_CCACHE_DIR=${HOME}/.ccache" >> "$GITHUB_ENV"
+
+    - name: Build Wheels
+      uses: pypa/cibuildwheel@63fd63b352a9a8bdcc24791c9dbee952ee9a8abc  # v3.3.0
+      env:
+        CMAKE_C_COMPILER_LAUNCHER: ccache
+        CMAKE_CXX_COMPILER_LAUNCHER: ccache
+        CMAKE_CUDA_COMPILER_LAUNCHER: ccache
+        CCACHE_COMPILERCHECK: content
+        CCACHE_MAXSIZE: 1G
+        CCACHE_DIR: ${{ runner.os == 'Linux' && '/host_ccache' || env.HOST_CCACHE_DIR }}
+        CCACHE_BASEDIR: ${{ runner.os == 'Linux' && '/project/build' || github.workspace }}
+        CIBW_CONTAINER_ENGINE: "docker; create_args: --volume ${{ env.HOST_CCACHE_DIR }}:/host_ccache"
+        CIBW_TEST_COMMAND: "python {project}/tools/validate_ccache_stats.py"
+        # CYTNX_VERSION_TAG was exported to $GITHUB_ENV by the Stamp
+        # step above; on Linux it is forwarded into the manylinux
+        # container via [tool.cibuildwheel.linux].environment-pass.
+        CYTNX_VERSION_TAG: ${{ env.CYTNX_VERSION_TAG }}
+
+    - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
+      with:
+        name: cibw-wheels-nightly-${{ matrix.os }}-${{ strategy.job-index }}
+        path: ./wheelhouse/*.whl
+
+  PublishNightlyPyPI:
+    name: PublishNightlyPyPI
+    needs: BuildNightlyWheel
+    runs-on: ubuntu-24.04
+    permissions:
+      id-token: write
+    steps:
+    - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
+      with:
+        pattern: cibw-wheels-nightly-*
+        path: dist
+        merge-multiple: true
+
+    - name: Publish package distributions to PyPI (cytnx dev release)
+      uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b  # release/v1
+      with:
+        packages-dir: dist

CMakeLists.txt

@@ -63,9 +63,21 @@ include(version.cmake)
 set(CYTNX_VERSION
   ${CYTNX_VERSION_MAJOR}.${CYTNX_VERSION_MINOR}.${CYTNX_VERSION_PATCH}
 )
+# `CYTNX_VERSION_FULL` carries an optional PEP 440-style suffix
+# (e.g. `.dev202605311220`) supplied via the `CYTNX_VERSION_TAG`
+# environment variable. The nightly release pipeline sets this from
+# tools/prepare_nightly_release.py so that the wheel filename
+# (cytnx_nightly-X.Y.Z.devN-*.whl) and the runtime `cytnx.__version__`
+# agree. `CYTNX_VERSION` itself stays strictly numeric because
+# `project(VERSION ...)` and `set_target_properties(VERSION ...)`
+# require MAJOR.MINOR.PATCH.
+set(CYTNX_VERSION_FULL "${CYTNX_VERSION}")
+if(DEFINED ENV{CYTNX_VERSION_TAG} AND NOT "$ENV{CYTNX_VERSION_TAG}" STREQUAL "")
+  string(APPEND CYTNX_VERSION_FULL "$ENV{CYTNX_VERSION_TAG}")
+endif()
 set(CYTNX_VARIANT_INFO "")
 
-message(STATUS " Version: ${CYTNX_VERSION}")
+message(STATUS " Version: ${CYTNX_VERSION_FULL}")
 
 # create a file that contain all the link flags:
 FILE(WRITE "${CMAKE_BINARY_DIR}/linkflags.tmp" "" "")
@@ -392,7 +404,7 @@ IF(BUILD_PYTHON)
     pybind/ncon_py.cpp
   )
   target_link_libraries(pycytnx PUBLIC cytnx)
-  target_compile_definitions(pycytnx PRIVATE CYTNX_VERSION="${CYTNX_VERSION}")
+  target_compile_definitions(pycytnx PRIVATE CYTNX_VERSION="${CYTNX_VERSION_FULL}")
 
   # On macOS, Python extensions should NOT link to libpython
   # Use -undefined dynamic_lookup to resolve symbols from the running interpreter

pyproject.toml

@@ -44,9 +44,8 @@ coverage = [
 # both Python and C++ coverage. Build-time deps (scikit-build-core,
 # pybind11) come from `[build-system].requires` automatically.
 dev = [
-    "pytest",
-    "pytest-cov",
-    "gcovr",
+    "cytnx[test]",
+    "cytnx[coverage]",
 ]
 docs = [
     "sphinx>=7.4.7",
@@ -56,6 +55,17 @@ docs = [
     "furo>=2024.8.6",
 ]
 
+[dependency-groups]
+# Build / release helpers (e.g. tools/prepare_nightly_release.py,
+# which rewrites this file to stamp a nightly version). Declared as a
+# PEP 735 dependency-group rather than under
+# `[project.optional-dependencies]` because these tools are needed at
+# *build pipeline* time only, not at install or run time of cytnx
+# itself. `pip install --group release-tools` installs the listed
+# packages without invoking scikit-build-core to compile the project,
+# which `pip install .[release-tools]` would have forced.
+release-tools = ["tomlkit"]
+
 [project.urls]
 Documentation = "https://cytnx-dev.github.io/Cytnx/"
 Repository = "https://github.com/Cytnx-dev/Cytnx.git"
@@ -93,7 +103,7 @@ before-build = "python ./tools/cibuildwheel_before_build.py"
 [tool.cibuildwheel.linux]
 before-all = "bash ./tools/cibuildwheel_before_all.sh"
 environment = { CMAKE_INCLUDE_PATH = "/usr/include/openblas", CCACHE_DEBUG = "true", CCACHE_DEBUGLEVEL = "1", CCACHE_LOGFILE = "/tmp/cytnx-ccache.log" }
-environment-pass = ["CMAKE_C_COMPILER_LAUNCHER", "CMAKE_CXX_COMPILER_LAUNCHER", "CMAKE_CUDA_COMPILER_LAUNCHER", "CCACHE_COMPILERCHECK", "CCACHE_MAXSIZE", "CCACHE_DIR", "CCACHE_BASEDIR", "CCACHE_DEBUG", "CCACHE_DEBUGLEVEL", "CCACHE_LOGFILE"]
+environment-pass = ["CMAKE_C_COMPILER_LAUNCHER", "CMAKE_CXX_COMPILER_LAUNCHER", "CMAKE_CUDA_COMPILER_LAUNCHER", "CCACHE_COMPILERCHECK", "CCACHE_MAXSIZE", "CCACHE_DIR", "CCACHE_BASEDIR", "CCACHE_DEBUG", "CCACHE_DEBUGLEVEL", "CCACHE_LOGFILE", "CYTNX_VERSION_TAG"]
 
 [tool.cibuildwheel.macos]
 before-all = "bash ./tools/cibuildwheel_before_all_macos.sh"