diff --git a/.github/workflows/release_pypi.yml b/.github/workflows/release_pypi.yml index fdcb8749b..5f2db2311 100644 --- a/.github/workflows/release_pypi.yml +++ b/.github/workflows/release_pypi.yml @@ -1,10 +1,13 @@ name: Release to PyPI +# Official `cytnx` release pipeline. Triggers only on `v*` tag pushes +# and manual workflow_dispatch; the resulting wheels are published to +# production PyPI in both cases. Nightly builds against master are +# handled by release_pypi_nightly.yml and publish under a separate +# nightly project. + on: - pull_request: push: - branches: - - master tags: - "v*" workflow_dispatch: @@ -20,58 +23,32 @@ jobs: # newer images because Homebrew dylib minos can force a higher wheel # deployment target and break delocate compatibility for lower targets. os: [ubuntu-24.04, ubuntu-24.04-arm, macos-14, macos-15-intel] - defaults: - run: - shell: bash -el {0} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: fetch-depth: 0 submodules: recursive - # When re-running a PR job, GitHub Actions uses the merge commit created at - # the time of the original run, not a fresh merge with the latest target branch. - # This step ensures we always test against the most recent target branch. - - name: Merge with latest target branch (pull_request only) - if: github.event_name == 'pull_request' - run: | - git fetch origin ${{ github.event.pull_request.base.ref }} - git merge --no-edit origin/${{ github.event.pull_request.base.ref }} - # Refresh submodules in case the merge moved any gitlinks (e.g. cmake_modules/morse_cmake). - git submodule update --init --recursive - - - name: Cache Ccache Directory (pull_request) - if: ${{ github.event_name == 'pull_request' }} - uses: actions/cache@v4 - with: - path: | - ~/.ccache - key: ccache-wheel-${{ runner.os }}-${{ github.event.pull_request.head.ref }}-${{ github.sha }} - restore-keys: | - ccache-wheel-${{ runner.os }}-${{ github.event.pull_request.head.ref }}- - ccache-wheel-${{ runner.os }}-${{ github.event.pull_request.base.ref }}- - - - name: Cache Ccache Directory (push) - if: ${{ github.event_name == 'push' }} - uses: actions/cache@v4 + - name: Cache Ccache Directory + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | ~/.ccache key: ccache-wheel-${{ runner.os }}-${{ github.ref_name }}-${{ github.sha }} restore-keys: | ccache-wheel-${{ runner.os }}-${{ github.ref_name }}- + ccache-wheel-${{ runner.os }}- - name: Set Ccache Directory run: | - echo "Start building---------------------------------" # HOST_CCACHE_DIR is consumed only by the docker bind-mount source # below; the in-container CCACHE_DIR is set in the Build Wheels # step's env: block (and forwarded via environment-pass). echo "HOST_CCACHE_DIR=${HOME}/.ccache" >> "$GITHUB_ENV" - name: Build Wheels - uses: pypa/cibuildwheel@v3.3.0 + uses: pypa/cibuildwheel@63fd63b352a9a8bdcc24791c9dbee952ee9a8abc # v3.3.0 env: CMAKE_C_COMPILER_LAUNCHER: ccache CMAKE_CXX_COMPILER_LAUNCHER: ccache @@ -94,28 +71,25 @@ jobs: CIBW_TEST_COMMAND: "python {project}/tools/validate_ccache_stats.py" - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} path: ./wheelhouse/*.whl - ReleaseTestPyPI: - name: ReleaseWheel-TestPyPI + ReleasePyPI: + name: ReleasePyPI needs: BuildWheel - if: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) }} runs-on: ubuntu-24.04 permissions: id-token: write steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: pattern: cibw-wheels-* path: dist merge-multiple: true - - name: Publish package distributions to TestPyPI (pull_request and tag push) - if: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) }} - uses: pypa/gh-action-pypi-publish@release/v1 + - name: Publish package distributions to PyPI (tag push or manual dispatch) + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 with: - repository-url: https://test.pypi.org/legacy/ packages-dir: dist diff --git a/.github/workflows/release_pypi_nightly.yml b/.github/workflows/release_pypi_nightly.yml new file mode 100644 index 000000000..7151a4f3c --- /dev/null +++ b/.github/workflows/release_pypi_nightly.yml @@ -0,0 +1,112 @@ +name: Nightly Release to PyPI + +# Builds and publishes a `cytnx` dev wheel set to PyPI on every push to +# master (i.e. every PR merge), so that `pip install --pre cytnx` +# tracks the latest master while `pip install cytnx` keeps resolving +# to the most recent tagged release. Wheels share the official `cytnx` +# PyPI project name but use a PEP 440 dev version +# (MAJOR.MINOR.PATCH.devYYYYMMDDHHMM) stamped into pyproject.toml by +# tools/prepare_nightly_release.py at build time. + +on: + push: + branches: + - master + workflow_dispatch: + +jobs: + BuildNightlyWheel: + name: BuildNightlyWheel-${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, ubuntu-24.04-arm, macos-14, macos-15-intel] + + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + fetch-depth: 0 + submodules: recursive + + - name: Install nightly stamping helper + # Installs the `release-tools` PEP 735 dependency-group from + # pyproject.toml (currently just `tomlkit`). Using `--group` + # rather than `pip install .[release-tools]` keeps the source + # of truth for build-pipeline deps in pyproject.toml without + # triggering scikit-build-core to compile cytnx itself before + # cibuildwheel does so under its own isolated environment. + # `--group` requires pip 25.1+; the runner-bundled pip is + # upgraded first to make the requirement portable across + # runner image versions. + run: | + python -m pip install --upgrade pip + pip install --group release-tools + + - name: Stamp pyproject.toml for nightly release + # Rewrites pyproject.toml to set a static + # MAJOR.MINOR.PATCH.devYYYYMMDDHHMM version and exports + # CYTNX_VERSION_TAG to $GITHUB_ENV so subsequent steps (and the + # cibuildwheel build) can append the same suffix to + # cytnx.__version__ via CMake. + run: python tools/prepare_nightly_release.py + + - name: Cache Ccache Directory + # Share the ccache prefix with release_pypi.yml: nightlies and + # tagged releases build the same C++ sources, so a cache primed + # by either feeds the other. + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.ccache + key: ccache-wheel-${{ runner.os }}-${{ github.ref_name }}-${{ github.sha }} + restore-keys: | + ccache-wheel-${{ runner.os }}-${{ github.ref_name }}- + ccache-wheel-${{ runner.os }}- + + - name: Set Ccache Directory + run: | + # HOST_CCACHE_DIR is consumed only by the docker bind-mount source + # below; the in-container CCACHE_DIR is set in the Build Wheels + # step's env: block (and forwarded via environment-pass). + echo "HOST_CCACHE_DIR=${HOME}/.ccache" >> "$GITHUB_ENV" + + - name: Build Wheels + uses: pypa/cibuildwheel@63fd63b352a9a8bdcc24791c9dbee952ee9a8abc # v3.3.0 + env: + CMAKE_C_COMPILER_LAUNCHER: ccache + CMAKE_CXX_COMPILER_LAUNCHER: ccache + CMAKE_CUDA_COMPILER_LAUNCHER: ccache + CCACHE_COMPILERCHECK: content + CCACHE_MAXSIZE: 1G + CCACHE_DIR: ${{ runner.os == 'Linux' && '/host_ccache' || env.HOST_CCACHE_DIR }} + CCACHE_BASEDIR: ${{ runner.os == 'Linux' && '/project/build' || github.workspace }} + CIBW_CONTAINER_ENGINE: "docker; create_args: --volume ${{ env.HOST_CCACHE_DIR }}:/host_ccache" + CIBW_TEST_COMMAND: "python {project}/tools/validate_ccache_stats.py" + # CYTNX_VERSION_TAG was exported to $GITHUB_ENV by the Stamp + # step above; on Linux it is forwarded into the manylinux + # container via [tool.cibuildwheel.linux].environment-pass. + CYTNX_VERSION_TAG: ${{ env.CYTNX_VERSION_TAG }} + + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: cibw-wheels-nightly-${{ matrix.os }}-${{ strategy.job-index }} + path: ./wheelhouse/*.whl + + PublishNightlyPyPI: + name: PublishNightlyPyPI + needs: BuildNightlyWheel + runs-on: ubuntu-24.04 + permissions: + id-token: write + steps: + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: cibw-wheels-nightly-* + path: dist + merge-multiple: true + + - name: Publish package distributions to PyPI (cytnx dev release) + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 + with: + packages-dir: dist diff --git a/CMakeLists.txt b/CMakeLists.txt index 40baa8243..4df530b45 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -63,9 +63,21 @@ include(version.cmake) set(CYTNX_VERSION ${CYTNX_VERSION_MAJOR}.${CYTNX_VERSION_MINOR}.${CYTNX_VERSION_PATCH} ) +# `CYTNX_VERSION_FULL` carries an optional PEP 440-style suffix +# (e.g. `.dev202605311220`) supplied via the `CYTNX_VERSION_TAG` +# environment variable. The nightly release pipeline sets this from +# tools/prepare_nightly_release.py so that the wheel filename +# (cytnx_nightly-X.Y.Z.devN-*.whl) and the runtime `cytnx.__version__` +# agree. `CYTNX_VERSION` itself stays strictly numeric because +# `project(VERSION ...)` and `set_target_properties(VERSION ...)` +# require MAJOR.MINOR.PATCH. +set(CYTNX_VERSION_FULL "${CYTNX_VERSION}") +if(DEFINED ENV{CYTNX_VERSION_TAG} AND NOT "$ENV{CYTNX_VERSION_TAG}" STREQUAL "") + string(APPEND CYTNX_VERSION_FULL "$ENV{CYTNX_VERSION_TAG}") +endif() set(CYTNX_VARIANT_INFO "") -message(STATUS " Version: ${CYTNX_VERSION}") +message(STATUS " Version: ${CYTNX_VERSION_FULL}") # create a file that contain all the link flags: FILE(WRITE "${CMAKE_BINARY_DIR}/linkflags.tmp" "" "") @@ -392,7 +404,7 @@ IF(BUILD_PYTHON) pybind/ncon_py.cpp ) target_link_libraries(pycytnx PUBLIC cytnx) - target_compile_definitions(pycytnx PRIVATE CYTNX_VERSION="${CYTNX_VERSION}") + target_compile_definitions(pycytnx PRIVATE CYTNX_VERSION="${CYTNX_VERSION_FULL}") # On macOS, Python extensions should NOT link to libpython # Use -undefined dynamic_lookup to resolve symbols from the running interpreter diff --git a/pyproject.toml b/pyproject.toml index 45de6d9dd..7ad52e015 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,9 +44,8 @@ coverage = [ # both Python and C++ coverage. Build-time deps (scikit-build-core, # pybind11) come from `[build-system].requires` automatically. dev = [ - "pytest", - "pytest-cov", - "gcovr", + "cytnx[test]", + "cytnx[coverage]", ] docs = [ "sphinx>=7.4.7", @@ -56,6 +55,17 @@ docs = [ "furo>=2024.8.6", ] +[dependency-groups] +# Build / release helpers (e.g. tools/prepare_nightly_release.py, +# which rewrites this file to stamp a nightly version). Declared as a +# PEP 735 dependency-group rather than under +# `[project.optional-dependencies]` because these tools are needed at +# *build pipeline* time only, not at install or run time of cytnx +# itself. `pip install --group release-tools` installs the listed +# packages without invoking scikit-build-core to compile the project, +# which `pip install .[release-tools]` would have forced. +release-tools = ["tomlkit"] + [project.urls] Documentation = "https://cytnx-dev.github.io/Cytnx/" Repository = "https://github.com/Cytnx-dev/Cytnx.git" @@ -93,7 +103,7 @@ before-build = "python ./tools/cibuildwheel_before_build.py" [tool.cibuildwheel.linux] before-all = "bash ./tools/cibuildwheel_before_all.sh" environment = { CMAKE_INCLUDE_PATH = "/usr/include/openblas", CCACHE_DEBUG = "true", CCACHE_DEBUGLEVEL = "1", CCACHE_LOGFILE = "/tmp/cytnx-ccache.log" } -environment-pass = ["CMAKE_C_COMPILER_LAUNCHER", "CMAKE_CXX_COMPILER_LAUNCHER", "CMAKE_CUDA_COMPILER_LAUNCHER", "CCACHE_COMPILERCHECK", "CCACHE_MAXSIZE", "CCACHE_DIR", "CCACHE_BASEDIR", "CCACHE_DEBUG", "CCACHE_DEBUGLEVEL", "CCACHE_LOGFILE"] +environment-pass = ["CMAKE_C_COMPILER_LAUNCHER", "CMAKE_CXX_COMPILER_LAUNCHER", "CMAKE_CUDA_COMPILER_LAUNCHER", "CCACHE_COMPILERCHECK", "CCACHE_MAXSIZE", "CCACHE_DIR", "CCACHE_BASEDIR", "CCACHE_DEBUG", "CCACHE_DEBUGLEVEL", "CCACHE_LOGFILE", "CYTNX_VERSION_TAG"] [tool.cibuildwheel.macos] before-all = "bash ./tools/cibuildwheel_before_all_macos.sh" diff --git a/tools/prepare_nightly_release.py b/tools/prepare_nightly_release.py new file mode 100644 index 000000000..6f357648c --- /dev/null +++ b/tools/prepare_nightly_release.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Stamp pyproject.toml and emit the build version tag for a nightly release. + +The script + + 1. reads `MAJOR.MINOR.PATCH` from version.cmake using the same regex + that the `[tool.scikit-build.metadata.version]` block of + pyproject.toml uses, so there is only one source of truth for how + the version is parsed; + 2. derives a PEP 440 dev version of the form + `MAJOR.MINOR.PATCH.devYYYYMMDDHHMM` (UTC stamp); + 3. rewrites pyproject.toml in place so cibuildwheel produces wheels + for the `cytnx` PyPI project with that static dev version + (`pip install --pre cytnx` will pick up the nightly; `pip install + cytnx` continues to install the latest stable, mirroring the + numpy / scipy convention); and + 4. appends `CYTNX_VERSION_TAG=.devYYYYMMDDHHMM` to `$GITHUB_ENV` so + the surrounding CI job can forward the same tag into CMake. The + C++ compile definition `CYTNX_VERSION` (which becomes + `cytnx.__version__`) is built from the numeric CMake version plus + this tag, keeping `cytnx.__version__` aligned with the wheel + filename. + +This is intended to run inside a fresh CI checkout before cibuildwheel. +It mutates the working tree and is not idempotent. + +Requires `tomlkit` (declared in pyproject.toml's `release-tools` +optional-dependencies group) so the rewrite preserves comments and +formatting on round-trip. +""" + +import datetime +import os +import pathlib +import re +import sys + +import tomlkit + +REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent +PYPROJECT = REPO_ROOT / "pyproject.toml" +VERSION_CMAKE = REPO_ROOT / "version.cmake" + + +def load_version_regex(doc: tomlkit.TOMLDocument) -> re.Pattern[str]: + pattern = doc["tool"]["scikit-build"]["metadata"]["version"]["regex"] + return re.compile(str(pattern).strip()) + + +def read_base_version(version_re: re.Pattern[str]) -> str: + text = VERSION_CMAKE.read_text() + match = version_re.search(text) + if not match: + sys.exit(f"could not parse MAJOR/MINOR/PATCH from {VERSION_CMAKE}") + return f"{match['major']}.{match['minor']}.{match['patch']}" + + +def build_dev_tag() -> str: + stamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d%H%M") + return f".dev{stamp}" + + +def rewrite_pyproject(doc: tomlkit.TOMLDocument, version: str) -> None: + project = doc["project"] + + dynamic = list(project.get("dynamic", [])) + if "version" not in dynamic: + sys.exit('expected "version" in [project].dynamic in pyproject.toml') + dynamic.remove("version") + if dynamic: + project["dynamic"] = dynamic + else: + del project["dynamic"] + project["version"] = version + + skb_metadata = doc["tool"]["scikit-build"]["metadata"] + if "version" not in skb_metadata: + sys.exit("expected [tool.scikit-build.metadata.version] in pyproject.toml") + del skb_metadata["version"] + + PYPROJECT.write_text(tomlkit.dumps(doc)) + + +def emit_github_env(tag: str) -> None: + github_env = os.environ.get("GITHUB_ENV") + if not github_env: + # Outside Actions; print so the operator can pass it manually. + print(f"CYTNX_VERSION_TAG={tag}") + return + with open(github_env, "a", encoding="utf-8") as f: + f.write(f"CYTNX_VERSION_TAG={tag}\n") + + +def main() -> None: + doc = tomlkit.parse(PYPROJECT.read_text()) + version_re = load_version_regex(doc) + base = read_base_version(version_re) + tag = build_dev_tag() + version = f"{base}{tag}" + + rewrite_pyproject(doc, version) + emit_github_env(tag) + + print(f"stamped pyproject.toml: cytnx=={version}") + + +if __name__ == "__main__": + main()